diff --git a/.env.example b/.env.example index 2545b8d..1c3abde 100644 --- a/.env.example +++ b/.env.example @@ -18,4 +18,10 @@ DB_HOST=127.0.0.1 DB_PORT=3306 DB_USER=admin DB_PASSWORD=password -DB_DATABASE=pc_builderv2 \ No newline at end of file +DB_DATABASE=pc_builderv2 + +# Merchant price comparison (headless browser scraping via playwright-core) +# Path to a system Chromium/Chrome binary. Leave unset to let Playwright +# try its own bundled browser (must be installed separately with +# `npx playwright install chromium` if so). +CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium \ No newline at end of file diff --git a/adonisrc.ts b/adonisrc.ts index c4406c7..f2af799 100644 --- a/adonisrc.ts +++ b/adonisrc.ts @@ -73,6 +73,7 @@ export default defineConfig({ () => import('#start/routes'), () => import('#start/kernel'), () => import('#start/validator'), + () => import('#start/browser'), ], /* diff --git a/app/services/headless_browser_service.ts b/app/services/headless_browser_service.ts new file mode 100644 index 0000000..861b9c8 --- /dev/null +++ b/app/services/headless_browser_service.ts @@ -0,0 +1,69 @@ +import { chromium, type Browser } from 'playwright-core' +import env from '#start/env' +import logger from '@adonisjs/core/services/logger' + +const USER_AGENT = + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + +let browserPromise: Promise | null = null + +/** + * Lazily launches a single shared headless Chromium instance, reused + * across requests instead of paying the ~1-2s browser startup cost on + * every merchant lookup. Points at a system-installed Chromium/Chrome + * binary via CHROMIUM_EXECUTABLE_PATH (see .env.example) rather than + * bundling Playwright's own ~300MB download. + */ +function getBrowser(): Promise { + if (!browserPromise) { + browserPromise = chromium + .launch({ + executablePath: env.get('CHROMIUM_EXECUTABLE_PATH') || undefined, + headless: true, + args: ['--no-sandbox', '--disable-setuid-sandbox'], + }) + .catch((error) => { + browserPromise = null + throw error + }) + } + return browserPromise +} + +/** + * Runs `fn` with a fresh browser page/context, always closing the context + * afterwards (success or failure) so pages don't accumulate across many + * lookups. The underlying browser process itself stays warm between calls. + * Returns null (rather than throwing) if the browser fails to launch or the + * callback throws, since merchant lookups are best-effort. + */ +async function withPage( + fn: (page: import('playwright-core').Page) => Promise +): Promise { + try { + const browser = await getBrowser() + const context = await browser.newContext({ userAgent: USER_AGENT, locale: 'fr-FR' }) + try { + const page = await context.newPage() + return await fn(page) + } finally { + await context.close() + } + } catch (error) { + logger.warn({ err: error }, 'Headless browser lookup failed') + return null + } +} + +/** + * Closes the shared browser instance, if running. Intended for graceful + * shutdown hooks / tests, not needed during normal request handling. + */ +async function closeBrowser(): Promise { + if (!browserPromise) return + const browser = await browserPromise.catch(() => null) + browserPromise = null + await browser?.close() +} + +export { withPage, closeBrowser } diff --git a/app/services/merchant_price_service.ts b/app/services/merchant_price_service.ts index fe20261..b81d88f 100644 --- a/app/services/merchant_price_service.ts +++ b/app/services/merchant_price_service.ts @@ -1,11 +1,19 @@ import * as cheerio from 'cheerio' import logger from '@adonisjs/core/services/logger' +import { withPage } from '#services/headless_browser_service' const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' export interface MerchantOffer { - merchant: 'Amazon' | 'LDLC' | 'Grosbill' | 'TopAchat' + merchant: + | 'Amazon' + | 'LDLC' + | 'Grosbill' + | 'TopAchat' + | 'Materiel.net' + | 'Cdiscount' + | 'PCComponentes' price: number url: string } @@ -41,11 +49,31 @@ function parsePriceText(text: string): number | null { } /** - * Some merchants (LDLC, Grosbill) render prices split across sibling/child - * elements instead of plain text, e.g. `
64€95
` - * renders as "64€95" once flattened to text, which parsePriceText would - * misread as 6495. This reads the integer part and the `` cents part - * separately and recombines them properly. + * Some merchants (LDLC, Grosbill, Materiel.net) render prices split across + * sibling/child elements instead of plain text, e.g. + * `
64€95
` renders as "64€95" once + * flattened to text, which parsePriceText would misread as 6495 (or, worse, + * "2 049€95" as 204995). This detects that "€" shape in + * flattened text and recombines it as . instead. + */ +function parseFlattenedFragmentedPriceText(text: string): number | null { + const match = text + .replace(/\u00a0/g, ' ') + .trim() + .match(/^([\d\s]+)\s*€\s*(\d{1,2})$/) + if (!match) return null + const whole = match[1].replace(/\s/g, '') + const cents = match[2].padEnd(2, '0').slice(0, 2) + const value = Number.parseFloat(`${whole}.${cents}`) + return Number.isNaN(value) ? null : value +} + +/** + * Same fragmented-price shape as parseFlattenedFragmentedPriceText, but + * operating on a Cheerio element with a real `` child instead of + * already-flattened text — used when scraping static HTML with cheerio + * (LDLC, Grosbill) rather than reading `.innerText()` from a live browser + * page (Materiel.net). * * LDLC additionally nests a duplicate `.price` wrapper * (`
64€95
`), @@ -253,6 +281,157 @@ async function fetchTopAchatPrice(productName: string): Promise { + const query = shortSearchQuery(productName) + const modelToken = extractModelToken(productName) + + return withPage(async (page) => { + await page.goto('https://www.materiel.net/', { waitUntil: 'networkidle', timeout: 20000 }) + await page.fill('#search_search_text', query) + await page.click('button[type="submit"]') + await page.waitForLoadState('networkidle', { timeout: 20000 }) + + // Case 1: redirected straight to a single product page (URL pattern + // distinguishes this from Case 2 — a list page also has + // .o-product__price elements, once per card, so checking for its mere + // presence isn't enough to tell the two cases apart). + if (/\/produit\//.test(page.url())) { + const title = await page.title() + if (!modelToken || title.toLowerCase().includes(modelToken.toLowerCase())) { + const text = await page + .locator('.o-product__price') + .first() + .innerText() + .catch(() => '') + const price = parseFlattenedFragmentedPriceText(text) ?? parsePriceText(text) + if (price !== null) { + return { merchant: 'Materiel.net' as const, price, url: page.url() } + } + } + return null + } + + // Case 2: a list of results. + const cards = page.locator('li.c-products-list__item[data-id]') + const count = await cards.count() + for (let i = 0; i < count; i++) { + const card = cards.nth(i) + const label = await card + .locator('.c-product__meta h2.c-product__title') + .innerText() + .catch(() => '') + if (modelToken && !label.toLowerCase().includes(modelToken.toLowerCase())) continue + + const priceText = await card + .locator('.o-product__price') + .first() + .innerText() + .catch(() => '') + const price = parseFlattenedFragmentedPriceText(priceText) ?? parsePriceText(priceText) + if (price === null) continue + + const link = await card + .locator('.c-product__meta a.c-product__link') + .first() + .getAttribute('href') + return { merchant: 'Materiel.net' as const, price, url: link ?? page.url() } + } + + return null + }) +} + +/** + * Searches Cdiscount by product name via a headless browser. Cdiscount's + * search-results page returns an empty body to plain HTTP fetches (no + * error, just nothing), which is consistent with the listing being + * rendered client-side and/or gated behind bot-detection that only lets + * real browsers through. + */ +async function fetchCdiscountPrice(productName: string): Promise { + const query = shortSearchQuery(productName) + const modelToken = extractModelToken(productName) + const url = `https://www.cdiscount.com/search/10/${encodeURIComponent(query)}.html` + + return withPage(async (page) => { + await page.goto(url, { waitUntil: 'networkidle', timeout: 20000 }) + + const cards = page.locator('div[data-e2e="offer-wrapper"]') + const count = await cards.count() + for (let i = 0; i < count; i++) { + const card = cards.nth(i) + const label = await card + .locator('[data-e2e="lplr-title"]') + .innerText() + .catch(() => '') + if (modelToken && !label.toLowerCase().includes(modelToken.toLowerCase())) continue + + const priceText = await card + .locator('[data-e2e="lplr-price"]') + .first() + .innerText() + .catch(() => '') + const price = parsePriceText(priceText) + if (price === null) continue + + const link = await card.locator('a').first().getAttribute('href') + const offerUrl = link ? new URL(link, 'https://www.cdiscount.com').toString() : url + return { merchant: 'Cdiscount' as const, price, url: offerUrl } + } + + return null + }) +} + +/** + * Searches PCComponentes.com by product name via a headless browser + * (blocked with a plain HTTP 403 otherwise). Product cards expose clean, + * pre-parsed data on the element itself (data-product-price as a plain + * number, data-product-category to distinguish standalone components from + * full "Sobremesa" pre-built PCs), so full-PC listings that happen to + * mention the same CPU are excluded rather than misreported as its price. + */ +async function fetchPcComponentesPrice(productName: string): Promise { + const query = shortSearchQuery(productName) + const modelToken = extractModelToken(productName) + const url = `https://www.pccomponentes.com/buscar/?query=${encodeURIComponent(query)}` + + return withPage(async (page) => { + await page.goto(url, { waitUntil: 'networkidle', timeout: 20000 }) + + const cards = page.locator('#search-results-product-grid > a[data-product-id]') + const count = await cards.count() + for (let i = 0; i < count; i++) { + const card = cards.nth(i) + const category = await card.getAttribute('data-product-category') + if (category === 'Sobremesa') continue // full pre-built PC, not the component itself + + const name = (await card.getAttribute('data-product-name')) ?? '' + if (modelToken && !name.toLowerCase().includes(modelToken.toLowerCase())) continue + + const priceAttr = await card.getAttribute('data-product-price') + const price = priceAttr ? Number.parseFloat(priceAttr) : null + if (price === null || Number.isNaN(price)) continue + + const href = await card.getAttribute('href') + const offerUrl = href ? new URL(href, 'https://www.pccomponentes.com').toString() : url + return { merchant: 'PCComponentes' as const, price, url: offerUrl } + } + + return null + }) +} + /** * Fetches offers from every supported merchant in parallel and returns * them sorted by ascending price. Merchants that fail (blocked, no match, @@ -268,6 +447,9 @@ async function findCheapestOffers( fetchLdlcPrice(productName), fetchGrosbillPrice(productName), fetchTopAchatPrice(productName), + fetchMaterielNetPrice(productName), + fetchCdiscountPrice(productName), + fetchPcComponentesPrice(productName), ] const results = await Promise.allSettled(lookups) @@ -288,4 +470,7 @@ export { fetchLdlcPrice, fetchGrosbillPrice, fetchTopAchatPrice, + fetchMaterielNetPrice, + fetchCdiscountPrice, + fetchPcComponentesPrice, } diff --git a/inertia/pages/types/UITypes.ts b/inertia/pages/types/UITypes.ts index c0fd205..771c94d 100644 --- a/inertia/pages/types/UITypes.ts +++ b/inertia/pages/types/UITypes.ts @@ -62,7 +62,14 @@ export type ScrapedPart = { } export type MerchantOffer = { - merchant: 'Amazon' | 'LDLC' | 'Grosbill' | 'TopAchat' + merchant: + | 'Amazon' + | 'LDLC' + | 'Grosbill' + | 'TopAchat' + | 'Materiel.net' + | 'Cdiscount' + | 'PCComponentes' price: number url: string } diff --git a/inertia/pages/widgets/NewPartSearch.vue b/inertia/pages/widgets/NewPartSearch.vue index 7872dd9..f788899 100644 --- a/inertia/pages/widgets/NewPartSearch.vue +++ b/inertia/pages/widgets/NewPartSearch.vue @@ -78,10 +78,13 @@ function cheapestOffer(part: ScrapedPart): MerchantOffer | null { function merchantDomain(merchant: MerchantOffer['merchant']): string { const domains: Record = { - Amazon: 'amazon.fr', - LDLC: 'ldlc.com', - Grosbill: 'grosbill.com', - TopAchat: 'topachat.com', + 'Amazon': 'amazon.fr', + 'LDLC': 'ldlc.com', + 'Grosbill': 'grosbill.com', + 'TopAchat': 'topachat.com', + 'Materiel.net': 'materiel.net', + 'Cdiscount': 'cdiscount.com', + 'PCComponentes': 'pccomponentes.com', } return domains[merchant] } diff --git a/package-lock.json b/package-lock.json index 0f1389f..db8a728 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "edge.js": "^6.5.1", "luxon": "^3.7.2", "mysql2": "^3.22.4", + "playwright-core": "^1.61.1", "reflect-metadata": "^0.2.2", "uuid": "^14.0.1", "vue": "^3.5.35", @@ -8097,6 +8098,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/pluralize": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", diff --git a/package.json b/package.json index 78a20d7..2faa41c 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "edge.js": "^6.5.1", "luxon": "^3.7.2", "mysql2": "^3.22.4", + "playwright-core": "^1.61.1", "reflect-metadata": "^0.2.2", "uuid": "^14.0.1", "vue": "^3.5.35", diff --git a/start/browser.ts b/start/browser.ts new file mode 100644 index 0000000..efed28a --- /dev/null +++ b/start/browser.ts @@ -0,0 +1,19 @@ +/* +|-------------------------------------------------------------------------- +| Headless browser lifecycle +|-------------------------------------------------------------------------- +| +| The merchant price comparison feature (app/services/merchant_price_service.ts) +| launches a single shared headless Chromium instance on demand (see +| app/services/headless_browser_service.ts). This makes sure that process +| is closed when the app shuts down, instead of leaking a Chromium child +| process. +| +*/ + +import app from '@adonisjs/core/services/app' + +app.terminating(async () => { + const { closeBrowser } = await import('#services/headless_browser_service') + await closeBrowser() +}) diff --git a/start/env.ts b/start/env.ts index 64a1959..594c624 100644 --- a/start/env.ts +++ b/start/env.ts @@ -24,4 +24,7 @@ export default await Env.create(new URL('../', import.meta.url), { // Session SESSION_DRIVER: Env.schema.enum(['cookie', 'memory', 'database'] as const), + + // Merchant price comparison (headless browser scraping) + CHROMIUM_EXECUTABLE_PATH: Env.schema.string.optional(), })