Kevin L'Hereec
a7788e593e
Google/Bing/DuckDuckGo/Qwant were tested as a way to find prices on the
3 remaining requested merchants and all four block scraping immediately
(visible captcha or JS-only anti-bot challenge on the very first
automated request) - not usable as an intermediary. Instead, a headless
browser was used to scrape these 3 merchants directly, since their
blocking (JS-only search, empty body, 403) turned out to be bypassable
with real browser rendering rather than needing to get past a captcha:
- Materiel.net: search is entirely client-side (Stimulus/Turbo), a plain
GET on /recherche/?query=... always returns the empty "no results"
template. Fixed by actually filling the search input and clicking
submit in a real page.
- Cdiscount: returns a fully empty response body to plain HTTP fetches
(no error, just nothing) but renders fine in a real browser.
- PCComponentes: blocks plain HTTP requests with a 403, but a headless
browser with a normal user agent goes through.
Implementation:
- New playwright-core dependency, pointed at a system-installed
Chromium binary via CHROMIUM_EXECUTABLE_PATH (set to /usr/bin/chromium
here) instead of downloading Playwright's own ~300MB browser
- New headless_browser_service.ts: lazily launches a single shared
Chromium instance reused across lookups (avoids paying browser launch
cost per request), with a withPage() helper that always closes its
browser context afterwards. A new start/browser.ts preload registers
an app.terminating() hook so the browser process is closed on shutdown
instead of leaking a child process.
- fetchMaterielNetPrice/fetchCdiscountPrice/fetchPcComponentesPrice
follow the same model-token matching strategy as the existing
cheerio-based merchants (LDLC/Grosbill/TopAchat): search on a short
brand+series query, then pick the result whose label/attribute
contains the extracted model number, to avoid misreporting an
unrelated product's price
- PCComponentes exposes clean data-product-price/data-product-category
attributes directly on result links, so full pre-built "Sobremesa" PCs
that happen to mention the searched CPU are excluded rather than
reported as its price
- Fixed two bugs found while testing with a Ryzen 5 9600X (a case with
multiple search results, unlike the Ryzen 3 3200G single-redirect case
used earlier):
- Materiel.net: a search results list page also has .o-product__price
elements (one per card), so checking for their mere presence isn't
enough to detect the "redirected to a single product" case; now
disambiguated by URL pattern (/produit/ vs /recherche/)
- New parseFlattenedFragmentedPriceText(): Playwright's innerText()
flattens "<whole>€<sup>cents</sup>" prices to plain strings like
"284€95" or "2 049€95", which the existing parser misread as
28495/204995. Cheerio-based scrapers (LDLC/Grosbill) already handled
this correctly via DOM structure (parseFragmentedPrice); this adds
the equivalent for already-flattened text.
- findCheapestOffers() now queries all 7 merchants in parallel
- Extended the MerchantOffer type/favicon domain map on the frontend for
the 3 new merchants
All 30 existing tests still pass unchanged (they don't exercise live
network scraping, only the surrounding auth/validation/caching logic).
103 lines
3.0 KiB
JSON
103 lines
3.0 KiB
JSON
{
|
|
"name": "pc-builder",
|
|
"version": "0.0.0",
|
|
"private": true,
|
|
"type": "module",
|
|
"license": "UNLICENSED",
|
|
"engines": {
|
|
"node": ">=24.0.0"
|
|
},
|
|
"scripts": {
|
|
"start": "node bin/server.js",
|
|
"build": "node ace build",
|
|
"dev": "node ace serve --hmr",
|
|
"test": "node ace test",
|
|
"lint": "eslint .",
|
|
"format": "prettier --write .",
|
|
"typecheck": "tsc --noEmit && vue-tsc --noEmit --project inertia/tsconfig.json"
|
|
},
|
|
"imports": {
|
|
"#controllers/*": "./app/controllers/*.js",
|
|
"#exceptions/*": "./app/exceptions/*.js",
|
|
"#models/*": "./app/models/*.js",
|
|
"#mails/*": "./app/mails/*.js",
|
|
"#services/*": "./app/services/*.js",
|
|
"#listeners/*": "./app/listeners/*.js",
|
|
"#events/*": "./app/events/*.js",
|
|
"#generated/*": "./.adonisjs/server/*.js",
|
|
"#middleware/*": "./app/middleware/*.js",
|
|
"#transformers/*": "./app/transformers/*.js",
|
|
"#types/*": "./types/*.js",
|
|
"#validators/*": "./app/validators/*.js",
|
|
"#providers/*": "./providers/*.js",
|
|
"#policies/*": "./app/policies/*.js",
|
|
"#abilities/*": "./app/abilities/*.js",
|
|
"#database/*": "./database/*.js",
|
|
"#tests/*": "./tests/*.js",
|
|
"#start/*": "./start/*.js",
|
|
"#config/*": "./config/*.js"
|
|
},
|
|
"devDependencies": {
|
|
"@adonisjs/assembler": "^8.4.0",
|
|
"@adonisjs/eslint-config": "^3.1.0",
|
|
"@adonisjs/prettier-config": "^1.5.0",
|
|
"@adonisjs/tsconfig": "^2.0.0",
|
|
"@japa/api-client": "^3.2.1",
|
|
"@japa/assert": "^4.2.0",
|
|
"@japa/browser-client": "^2.3.0",
|
|
"@japa/plugin-adonisjs": "^5.2.0",
|
|
"@japa/runner": "^5.3.0",
|
|
"@poppinss/ts-exec": "^1.4.4",
|
|
"@types/luxon": "^3.7.1",
|
|
"@types/node": "~25.9.1",
|
|
"@vitejs/plugin-vue": "^6.0.7",
|
|
"autoprefixer": "^10.5.2",
|
|
"eslint": "^10.4.0",
|
|
"eslint-plugin-vue": "^10.9.1",
|
|
"hot-hook": "^1.0.0",
|
|
"pino-pretty": "^13.1.3",
|
|
"postcss": "^8.5.15",
|
|
"prettier": "^3.8.3",
|
|
"tailwindcss": "^3.4.19",
|
|
"typescript": "~6.0.3",
|
|
"vite": "^7.3.3",
|
|
"vue-tsc": "^3.3.2",
|
|
"youch": "^4.1.1"
|
|
},
|
|
"dependencies": {
|
|
"@adonisjs/auth": "^10.1.0",
|
|
"@adonisjs/core": "^7.3.3",
|
|
"@adonisjs/cors": "^3.0.0",
|
|
"@adonisjs/inertia": "^4.2.0",
|
|
"@adonisjs/lucid": "^22.4.2",
|
|
"@adonisjs/session": "^8.1.0",
|
|
"@adonisjs/shield": "^9.0.0",
|
|
"@adonisjs/static": "^2.0.1",
|
|
"@adonisjs/vite": "^5.1.1",
|
|
"@inertiajs/vue3": "^2.3.24",
|
|
"@tuyau/core": "^1.2.2",
|
|
"@types/uuid": "^10.0.0",
|
|
"@vinejs/vine": "^4.4.0",
|
|
"@vue/server-renderer": "^3.5.35",
|
|
"@vuepic/vue-datepicker": "^13.0.0",
|
|
"better-sqlite3": "^12.10.0",
|
|
"cheerio": "^1.2.0",
|
|
"edge.js": "^6.5.1",
|
|
"luxon": "^3.7.2",
|
|
"mysql2": "^3.22.4",
|
|
"playwright-core": "^1.61.1",
|
|
"reflect-metadata": "^0.2.2",
|
|
"uuid": "^14.0.1",
|
|
"vue": "^3.5.35",
|
|
"vue-sonner": "^2.0.9",
|
|
"vue3-carousel": "^0.17.0"
|
|
},
|
|
"hotHook": {
|
|
"boundaries": [
|
|
"./app/controllers/**/*.ts",
|
|
"./app/middleware/*.ts"
|
|
]
|
|
},
|
|
"prettier": "@adonisjs/prettier-config"
|
|
}
|