feat: add Materiel.net, Cdiscount and PCComponentes via headless browser

Google/Bing/DuckDuckGo/Qwant were tested as a way to find prices on the
3 remaining requested merchants and all four block scraping immediately
(visible captcha or JS-only anti-bot challenge on the very first
automated request) - not usable as an intermediary. Instead, a headless
browser was used to scrape these 3 merchants directly, since their
blocking (JS-only search, empty body, 403) turned out to be bypassable
with real browser rendering rather than needing to get past a captcha:

- Materiel.net: search is entirely client-side (Stimulus/Turbo), a plain
  GET on /recherche/?query=... always returns the empty "no results"
  template. Fixed by actually filling the search input and clicking
  submit in a real page.
- Cdiscount: returns a fully empty response body to plain HTTP fetches
  (no error, just nothing) but renders fine in a real browser.
- PCComponentes: blocks plain HTTP requests with a 403, but a headless
  browser with a normal user agent goes through.

Implementation:
- New playwright-core dependency, pointed at a system-installed
  Chromium binary via CHROMIUM_EXECUTABLE_PATH (set to /usr/bin/chromium
  here) instead of downloading Playwright's own ~300MB browser
- New headless_browser_service.ts: lazily launches a single shared
  Chromium instance reused across lookups (avoids paying browser launch
  cost per request), with a withPage() helper that always closes its
  browser context afterwards. A new start/browser.ts preload registers
  an app.terminating() hook so the browser process is closed on shutdown
  instead of leaking a child process.
- fetchMaterielNetPrice/fetchCdiscountPrice/fetchPcComponentesPrice
  follow the same model-token matching strategy as the existing
  cheerio-based merchants (LDLC/Grosbill/TopAchat): search on a short
  brand+series query, then pick the result whose label/attribute
  contains the extracted model number, to avoid misreporting an
  unrelated product's price
- PCComponentes exposes clean data-product-price/data-product-category
  attributes directly on result links, so full pre-built "Sobremesa" PCs
  that happen to mention the searched CPU are excluded rather than
  reported as its price
- Fixed two bugs found while testing with a Ryzen 5 9600X (a case with
  multiple search results, unlike the Ryzen 3 3200G single-redirect case
  used earlier):
  - Materiel.net: a search results list page also has .o-product__price
    elements (one per card), so checking for their mere presence isn't
    enough to detect the "redirected to a single product" case; now
    disambiguated by URL pattern (/produit/ vs /recherche/)
  - New parseFlattenedFragmentedPriceText(): Playwright's innerText()
    flattens "<whole>€<sup>cents</sup>" prices to plain strings like
    "284€95" or "2 049€95", which the existing parser misread as
    28495/204995. Cheerio-based scrapers (LDLC/Grosbill) already handled
    this correctly via DOM structure (parseFragmentedPrice); this adds
    the equivalent for already-flattened text.
- findCheapestOffers() now queries all 7 merchants in parallel
- Extended the MerchantOffer type/favicon domain map on the frontend for
  the 3 new merchants

All 30 existing tests still pass unchanged (they don't exercise live
network scraping, only the surrounding auth/validation/caching logic).

.env.example

@@ -19,3 +19,9 @@ DB_PORT=3306
 DB_USER=admin
 DB_PASSWORD=password
 DB_DATABASE=pc_builderv2
+
+# Merchant price comparison (headless browser scraping via playwright-core)
+# Path to a system Chromium/Chrome binary. Leave unset to let Playwright
+# try its own bundled browser (must be installed separately with
+# `npx playwright install chromium` if so).
+CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium

adonisrc.ts

@@ -73,6 +73,7 @@ export default defineConfig({
     () => import('#start/routes'),
     () => import('#start/kernel'),
     () => import('#start/validator'),
+    () => import('#start/browser'),
   ],
 
   /*

app/services/headless_browser_service.ts

@@ -0,0 +1,69 @@
+import { chromium, type Browser } from 'playwright-core'
+import env from '#start/env'
+import logger from '@adonisjs/core/services/logger'
+
+const USER_AGENT =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+
+let browserPromise: Promise<Browser> | null = null
+
+/**
+ * Lazily launches a single shared headless Chromium instance, reused
+ * across requests instead of paying the ~1-2s browser startup cost on
+ * every merchant lookup. Points at a system-installed Chromium/Chrome
+ * binary via CHROMIUM_EXECUTABLE_PATH (see .env.example) rather than
+ * bundling Playwright's own ~300MB download.
+ */
+function getBrowser(): Promise<Browser> {
+  if (!browserPromise) {
+    browserPromise = chromium
+      .launch({
+        executablePath: env.get('CHROMIUM_EXECUTABLE_PATH') || undefined,
+        headless: true,
+        args: ['--no-sandbox', '--disable-setuid-sandbox'],
+      })
+      .catch((error) => {
+        browserPromise = null
+        throw error
+      })
+  }
+  return browserPromise
+}
+
+/**
+ * Runs `fn` with a fresh browser page/context, always closing the context
+ * afterwards (success or failure) so pages don't accumulate across many
+ * lookups. The underlying browser process itself stays warm between calls.
+ * Returns null (rather than throwing) if the browser fails to launch or the
+ * callback throws, since merchant lookups are best-effort.
+ */
+async function withPage<T>(
+  fn: (page: import('playwright-core').Page) => Promise<T>
+): Promise<T | null> {
+  try {
+    const browser = await getBrowser()
+    const context = await browser.newContext({ userAgent: USER_AGENT, locale: 'fr-FR' })
+    try {
+      const page = await context.newPage()
+      return await fn(page)
+    } finally {
+      await context.close()
+    }
+  } catch (error) {
+    logger.warn({ err: error }, 'Headless browser lookup failed')
+    return null
+  }
+}
+
+/**
+ * Closes the shared browser instance, if running. Intended for graceful
+ * shutdown hooks / tests, not needed during normal request handling.
+ */
+async function closeBrowser(): Promise<void> {
+  if (!browserPromise) return
+  const browser = await browserPromise.catch(() => null)
+  browserPromise = null
+  await browser?.close()
+}
+
+export { withPage, closeBrowser }

app/services/merchant_price_service.ts

@@ -1,11 +1,19 @@
 import * as cheerio from 'cheerio'
 import logger from '@adonisjs/core/services/logger'
+import { withPage } from '#services/headless_browser_service'
 
 const USER_AGENT =
   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
 
 export interface MerchantOffer {
-  merchant: 'Amazon' | 'LDLC' | 'Grosbill' | 'TopAchat'
+  merchant:
+    | 'Amazon'
+    | 'LDLC'
+    | 'Grosbill'
+    | 'TopAchat'
+    | 'Materiel.net'
+    | 'Cdiscount'
+    | 'PCComponentes'
   price: number
   url: string
 }
@@ -41,11 +49,31 @@ function parsePriceText(text: string): number | null {
 }
 
 /**
- * Some merchants (LDLC, Grosbill) render prices split across sibling/child
+ * Some merchants (LDLC, Grosbill, Materiel.net) render prices split across
- * elements instead of plain text, e.g. `<div class="price">64€<sup>95</sup></div>`
+ * sibling/child elements instead of plain text, e.g.
- * renders as "64€95" once flattened to text, which parsePriceText would
+ * `<div class="price">64€<sup>95</sup></div>` renders as "64€95" once
- * misread as 6495. This reads the integer part and the `<sup>` cents part
+ * flattened to text, which parsePriceText would misread as 6495 (or, worse,
- * separately and recombines them properly.
+ * "2 049€95" as 204995). This detects that "<whole>€<cents>" shape in
+ * flattened text and recombines it as <whole>.<cents> instead.
+ */
+function parseFlattenedFragmentedPriceText(text: string): number | null {
+  const match = text
+    .replace(/\u00a0/g, ' ')
+    .trim()
+    .match(/^([\d\s]+)\s*€\s*(\d{1,2})$/)
+  if (!match) return null
+  const whole = match[1].replace(/\s/g, '')
+  const cents = match[2].padEnd(2, '0').slice(0, 2)
+  const value = Number.parseFloat(`${whole}.${cents}`)
+  return Number.isNaN(value) ? null : value
+}
+
+/**
+ * Same fragmented-price shape as parseFlattenedFragmentedPriceText, but
+ * operating on a Cheerio element with a real `<sup>` child instead of
+ * already-flattened text — used when scraping static HTML with cheerio
+ * (LDLC, Grosbill) rather than reading `.innerText()` from a live browser
+ * page (Materiel.net).
  *
  * LDLC additionally nests a duplicate `.price` wrapper
  * (`<div class="price"><div class="price">64€<sup>95</sup></div></div>`),
@@ -253,6 +281,157 @@ async function fetchTopAchatPrice(productName: string): Promise<MerchantOffer |
   return { merchant: 'TopAchat', price, url: link ? new URL(link, url).toString() : url }
 }
 
+/**
+ * Searches Materiel.net by simulating the real search form submission in a
+ * headless browser. A plain GET on /recherche/?query=... always returns an
+ * empty "no results" template — the search only actually runs client-side
+ * (Stimulus/Turbo), so this is one of three merchants where a real browser
+ * is required rather than a plain HTTP fetch.
+ *
+ * When there's a single match, Materiel.net redirects straight to the
+ * product page. Otherwise it lists cards matched by model token.
+ */
+async function fetchMaterielNetPrice(productName: string): Promise<MerchantOffer | null> {
+  const query = shortSearchQuery(productName)
+  const modelToken = extractModelToken(productName)
+
+  return withPage(async (page) => {
+    await page.goto('https://www.materiel.net/', { waitUntil: 'networkidle', timeout: 20000 })
+    await page.fill('#search_search_text', query)
+    await page.click('button[type="submit"]')
+    await page.waitForLoadState('networkidle', { timeout: 20000 })
+
+    // Case 1: redirected straight to a single product page (URL pattern
+    // distinguishes this from Case 2 — a list page also has
+    // .o-product__price elements, once per card, so checking for its mere
+    // presence isn't enough to tell the two cases apart).
+    if (/\/produit\//.test(page.url())) {
+      const title = await page.title()
+      if (!modelToken || title.toLowerCase().includes(modelToken.toLowerCase())) {
+        const text = await page
+          .locator('.o-product__price')
+          .first()
+          .innerText()
+          .catch(() => '')
+        const price = parseFlattenedFragmentedPriceText(text) ?? parsePriceText(text)
+        if (price !== null) {
+          return { merchant: 'Materiel.net' as const, price, url: page.url() }
+        }
+      }
+      return null
+    }
+
+    // Case 2: a list of results.
+    const cards = page.locator('li.c-products-list__item[data-id]')
+    const count = await cards.count()
+    for (let i = 0; i < count; i++) {
+      const card = cards.nth(i)
+      const label = await card
+        .locator('.c-product__meta h2.c-product__title')
+        .innerText()
+        .catch(() => '')
+      if (modelToken && !label.toLowerCase().includes(modelToken.toLowerCase())) continue
+
+      const priceText = await card
+        .locator('.o-product__price')
+        .first()
+        .innerText()
+        .catch(() => '')
+      const price = parseFlattenedFragmentedPriceText(priceText) ?? parsePriceText(priceText)
+      if (price === null) continue
+
+      const link = await card
+        .locator('.c-product__meta a.c-product__link')
+        .first()
+        .getAttribute('href')
+      return { merchant: 'Materiel.net' as const, price, url: link ?? page.url() }
+    }
+
+    return null
+  })
+}
+
+/**
+ * Searches Cdiscount by product name via a headless browser. Cdiscount's
+ * search-results page returns an empty body to plain HTTP fetches (no
+ * error, just nothing), which is consistent with the listing being
+ * rendered client-side and/or gated behind bot-detection that only lets
+ * real browsers through.
+ */
+async function fetchCdiscountPrice(productName: string): Promise<MerchantOffer | null> {
+  const query = shortSearchQuery(productName)
+  const modelToken = extractModelToken(productName)
+  const url = `https://www.cdiscount.com/search/10/${encodeURIComponent(query)}.html`
+
+  return withPage(async (page) => {
+    await page.goto(url, { waitUntil: 'networkidle', timeout: 20000 })
+
+    const cards = page.locator('div[data-e2e="offer-wrapper"]')
+    const count = await cards.count()
+    for (let i = 0; i < count; i++) {
+      const card = cards.nth(i)
+      const label = await card
+        .locator('[data-e2e="lplr-title"]')
+        .innerText()
+        .catch(() => '')
+      if (modelToken && !label.toLowerCase().includes(modelToken.toLowerCase())) continue
+
+      const priceText = await card
+        .locator('[data-e2e="lplr-price"]')
+        .first()
+        .innerText()
+        .catch(() => '')
+      const price = parsePriceText(priceText)
+      if (price === null) continue
+
+      const link = await card.locator('a').first().getAttribute('href')
+      const offerUrl = link ? new URL(link, 'https://www.cdiscount.com').toString() : url
+      return { merchant: 'Cdiscount' as const, price, url: offerUrl }
+    }
+
+    return null
+  })
+}
+
+/**
+ * Searches PCComponentes.com by product name via a headless browser
+ * (blocked with a plain HTTP 403 otherwise). Product cards expose clean,
+ * pre-parsed data on the <a> element itself (data-product-price as a plain
+ * number, data-product-category to distinguish standalone components from
+ * full "Sobremesa" pre-built PCs), so full-PC listings that happen to
+ * mention the same CPU are excluded rather than misreported as its price.
+ */
+async function fetchPcComponentesPrice(productName: string): Promise<MerchantOffer | null> {
+  const query = shortSearchQuery(productName)
+  const modelToken = extractModelToken(productName)
+  const url = `https://www.pccomponentes.com/buscar/?query=${encodeURIComponent(query)}`
+
+  return withPage(async (page) => {
+    await page.goto(url, { waitUntil: 'networkidle', timeout: 20000 })
+
+    const cards = page.locator('#search-results-product-grid > a[data-product-id]')
+    const count = await cards.count()
+    for (let i = 0; i < count; i++) {
+      const card = cards.nth(i)
+      const category = await card.getAttribute('data-product-category')
+      if (category === 'Sobremesa') continue // full pre-built PC, not the component itself
+
+      const name = (await card.getAttribute('data-product-name')) ?? ''
+      if (modelToken && !name.toLowerCase().includes(modelToken.toLowerCase())) continue
+
+      const priceAttr = await card.getAttribute('data-product-price')
+      const price = priceAttr ? Number.parseFloat(priceAttr) : null
+      if (price === null || Number.isNaN(price)) continue
+
+      const href = await card.getAttribute('href')
+      const offerUrl = href ? new URL(href, 'https://www.pccomponentes.com').toString() : url
+      return { merchant: 'PCComponentes' as const, price, url: offerUrl }
+    }
+
+    return null
+  })
+}
+
 /**
  * Fetches offers from every supported merchant in parallel and returns
  * them sorted by ascending price. Merchants that fail (blocked, no match,
@@ -268,6 +447,9 @@ async function findCheapestOffers(
     fetchLdlcPrice(productName),
     fetchGrosbillPrice(productName),
     fetchTopAchatPrice(productName),
+    fetchMaterielNetPrice(productName),
+    fetchCdiscountPrice(productName),
+    fetchPcComponentesPrice(productName),
   ]
 
   const results = await Promise.allSettled(lookups)
@@ -288,4 +470,7 @@ export {
   fetchLdlcPrice,
   fetchGrosbillPrice,
   fetchTopAchatPrice,
+  fetchMaterielNetPrice,
+  fetchCdiscountPrice,
+  fetchPcComponentesPrice,
 }

inertia/pages/types/UITypes.ts

@@ -62,7 +62,14 @@ export type ScrapedPart = {
 }
 
 export type MerchantOffer = {
-  merchant: 'Amazon' | 'LDLC' | 'Grosbill' | 'TopAchat'
+  merchant:
+    | 'Amazon'
+    | 'LDLC'
+    | 'Grosbill'
+    | 'TopAchat'
+    | 'Materiel.net'
+    | 'Cdiscount'
+    | 'PCComponentes'
   price: number
   url: string
 }

inertia/pages/widgets/NewPartSearch.vue

@@ -78,10 +78,13 @@ function cheapestOffer(part: ScrapedPart): MerchantOffer | null {
 
 function merchantDomain(merchant: MerchantOffer['merchant']): string {
   const domains: Record<MerchantOffer['merchant'], string> = {
-    Amazon: 'amazon.fr',
+    'Amazon': 'amazon.fr',
-    LDLC: 'ldlc.com',
+    'LDLC': 'ldlc.com',
-    Grosbill: 'grosbill.com',
+    'Grosbill': 'grosbill.com',
-    TopAchat: 'topachat.com',
+    'TopAchat': 'topachat.com',
+    'Materiel.net': 'materiel.net',
+    'Cdiscount': 'cdiscount.com',
+    'PCComponentes': 'pccomponentes.com',
   }
   return domains[merchant]
 }

package-lock.json

@@ -29,6 +29,7 @@
         "edge.js": "^6.5.1",
         "luxon": "^3.7.2",
         "mysql2": "^3.22.4",
+        "playwright-core": "^1.61.1",
         "reflect-metadata": "^0.2.2",
         "uuid": "^14.0.1",
         "vue": "^3.5.35",
@@ -8097,6 +8098,18 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/playwright-core": {
+      "version": "1.61.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz",
+      "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==",
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/pluralize": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz",

package.json

@@ -85,6 +85,7 @@
     "edge.js": "^6.5.1",
     "luxon": "^3.7.2",
     "mysql2": "^3.22.4",
+    "playwright-core": "^1.61.1",
     "reflect-metadata": "^0.2.2",
     "uuid": "^14.0.1",
     "vue": "^3.5.35",

start/browser.ts

@@ -0,0 +1,19 @@
+/*
+|--------------------------------------------------------------------------
+| Headless browser lifecycle
+|--------------------------------------------------------------------------
+|
+| The merchant price comparison feature (app/services/merchant_price_service.ts)
+| launches a single shared headless Chromium instance on demand (see
+| app/services/headless_browser_service.ts). This makes sure that process
+| is closed when the app shuts down, instead of leaking a Chromium child
+| process.
+|
+*/
+
+import app from '@adonisjs/core/services/app'
+
+app.terminating(async () => {
+  const { closeBrowser } = await import('#services/headless_browser_service')
+  await closeBrowser()
+})

start/env.ts

@@ -24,4 +24,7 @@ export default await Env.create(new URL('../', import.meta.url), {
 
   // Session
   SESSION_DRIVER: Env.schema.enum(['cookie', 'memory', 'database'] as const),
+
+  // Merchant price comparison (headless browser scraping)
+  CHROMIUM_EXECUTABLE_PATH: Env.schema.string.optional(),
 })