Google/Bing/DuckDuckGo/Qwant were tested as a way to find prices on the
3 remaining requested merchants and all four block scraping immediately
(visible captcha or JS-only anti-bot challenge on the very first
automated request) - not usable as an intermediary. Instead, a headless
browser was used to scrape these 3 merchants directly, since their
blocking (JS-only search, empty body, 403) turned out to be bypassable
with real browser rendering rather than needing to get past a captcha:
- Materiel.net: search is entirely client-side (Stimulus/Turbo), a plain
GET on /recherche/?query=... always returns the empty "no results"
template. Fixed by actually filling the search input and clicking
submit in a real page.
- Cdiscount: returns a fully empty response body to plain HTTP fetches
(no error, just nothing) but renders fine in a real browser.
- PCComponentes: blocks plain HTTP requests with a 403, but a headless
browser with a normal user agent goes through.
Implementation:
- New playwright-core dependency, pointed at a system-installed
Chromium binary via CHROMIUM_EXECUTABLE_PATH (set to /usr/bin/chromium
here) instead of downloading Playwright's own ~300MB browser
- New headless_browser_service.ts: lazily launches a single shared
Chromium instance reused across lookups (avoids paying browser launch
cost per request), with a withPage() helper that always closes its
browser context afterwards. A new start/browser.ts preload registers
an app.terminating() hook so the browser process is closed on shutdown
instead of leaking a child process.
- fetchMaterielNetPrice/fetchCdiscountPrice/fetchPcComponentesPrice
follow the same model-token matching strategy as the existing
cheerio-based merchants (LDLC/Grosbill/TopAchat): search on a short
brand+series query, then pick the result whose label/attribute
contains the extracted model number, to avoid misreporting an
unrelated product's price
- PCComponentes exposes clean data-product-price/data-product-category
attributes directly on result links, so full pre-built "Sobremesa" PCs
that happen to mention the searched CPU are excluded rather than
reported as its price
- Fixed two bugs found while testing with a Ryzen 5 9600X (a case with
multiple search results, unlike the Ryzen 3 3200G single-redirect case
used earlier):
- Materiel.net: a search results list page also has .o-product__price
elements (one per card), so checking for their mere presence isn't
enough to detect the "redirected to a single product" case; now
disambiguated by URL pattern (/produit/ vs /recherche/)
- New parseFlattenedFragmentedPriceText(): Playwright's innerText()
flattens "<whole>€<sup>cents</sup>" prices to plain strings like
"284€95" or "2 049€95", which the existing parser misread as
28495/204995. Cheerio-based scrapers (LDLC/Grosbill) already handled
this correctly via DOM structure (parseFragmentedPrice); this adds
the equivalent for already-flattened text.
- findCheapestOffers() now queries all 7 merchants in parallel
- Extended the MerchantOffer type/favicon domain map on the frontend for
the 3 new merchants
All 30 existing tests still pass unchanged (they don't exercise live
network scraping, only the surrounding auth/validation/caching logic).
176 lines
5.3 KiB
Vue
176 lines
5.3 KiB
Vue
<script setup lang="ts">
|
|
import { reactive, ref, watch } from 'vue'
|
|
import type { MerchantOffer, ScrapedPart } from '~/pages/types/UITypes'
|
|
|
|
const emit = defineEmits<{
|
|
import: [part: ScrapedPart, offer: MerchantOffer | null]
|
|
}>()
|
|
|
|
function faviconUrl(source: string): string {
|
|
return `https://www.google.com/s2/favicons?domain=${encodeURIComponent(source)}&sz=32`
|
|
}
|
|
|
|
const query = ref('')
|
|
const results = ref<ScrapedPart[]>([])
|
|
const isLoading = ref(false)
|
|
const hasSearched = ref(false)
|
|
let debounceTimer: ReturnType<typeof setTimeout> | null = null
|
|
|
|
// Per-part price comparison state, keyed by scraped part id.
|
|
const comparing = reactive<Record<number, boolean>>({})
|
|
const offers = reactive<Record<number, MerchantOffer[]>>({})
|
|
const compareErrors = reactive<Record<number, boolean>>({})
|
|
|
|
async function search() {
|
|
if (!query.value.trim()) {
|
|
results.value = []
|
|
hasSearched.value = false
|
|
return
|
|
}
|
|
|
|
isLoading.value = true
|
|
try {
|
|
const response = await fetch(`/scraped-parts?q=${encodeURIComponent(query.value.trim())}`, {
|
|
headers: { Accept: 'application/json' },
|
|
})
|
|
if (response.ok) {
|
|
results.value = await response.json()
|
|
} else {
|
|
results.value = []
|
|
}
|
|
} finally {
|
|
isLoading.value = false
|
|
hasSearched.value = true
|
|
}
|
|
}
|
|
|
|
watch(query, () => {
|
|
if (debounceTimer) clearTimeout(debounceTimer)
|
|
debounceTimer = setTimeout(search, 400)
|
|
})
|
|
|
|
async function comparePrices(part: ScrapedPart) {
|
|
comparing[part.id] = true
|
|
compareErrors[part.id] = false
|
|
try {
|
|
const response = await fetch(`/scraped-parts/compare?id=${part.id}`, {
|
|
headers: { Accept: 'application/json' },
|
|
})
|
|
if (!response.ok) {
|
|
compareErrors[part.id] = true
|
|
return
|
|
}
|
|
const data = await response.json()
|
|
offers[part.id] = data.offers ?? []
|
|
if (offers[part.id].length === 0) {
|
|
compareErrors[part.id] = true
|
|
}
|
|
} catch {
|
|
compareErrors[part.id] = true
|
|
} finally {
|
|
comparing[part.id] = false
|
|
}
|
|
}
|
|
|
|
function cheapestOffer(part: ScrapedPart): MerchantOffer | null {
|
|
return offers[part.id]?.[0] ?? null
|
|
}
|
|
|
|
function merchantDomain(merchant: MerchantOffer['merchant']): string {
|
|
const domains: Record<MerchantOffer['merchant'], string> = {
|
|
'Amazon': 'amazon.fr',
|
|
'LDLC': 'ldlc.com',
|
|
'Grosbill': 'grosbill.com',
|
|
'TopAchat': 'topachat.com',
|
|
'Materiel.net': 'materiel.net',
|
|
'Cdiscount': 'cdiscount.com',
|
|
'PCComponentes': 'pccomponentes.com',
|
|
}
|
|
return domains[merchant]
|
|
}
|
|
|
|
function importPart(part: ScrapedPart) {
|
|
emit('import', part, cheapestOffer(part))
|
|
}
|
|
</script>
|
|
|
|
<template>
|
|
<div class="border border-gray-200 rounded-lg p-4 bg-gray-50">
|
|
<label class="block text-gray-700 text-sm font-bold mb-2">
|
|
Search a new part (pc-builder.io)
|
|
</label>
|
|
<input
|
|
v-model="query"
|
|
type="text"
|
|
placeholder="e.g. Ryzen 5 7600X, RTX 4070..."
|
|
class="w-full border-gray-200 focus:border-indigo-600 focus:ring focus:ring-indigo-500/40 px-3 py-2 border rounded text-gray-700"
|
|
/>
|
|
|
|
<p v-if="isLoading" class="text-sm text-gray-500 mt-2">Searching…</p>
|
|
<p v-else-if="hasSearched && results.length === 0" class="text-sm text-gray-500 mt-2">
|
|
No matching part found.
|
|
</p>
|
|
|
|
<ul v-if="results.length > 0" class="mt-3 space-y-2 max-h-96 overflow-y-auto">
|
|
<li
|
|
v-for="part in results"
|
|
:key="part.id"
|
|
class="bg-white border border-gray-200 rounded p-2"
|
|
>
|
|
<div class="flex items-center gap-3">
|
|
<img
|
|
v-if="part.imageUrl"
|
|
:src="part.imageUrl"
|
|
alt=""
|
|
class="w-10 h-10 object-contain shrink-0"
|
|
/>
|
|
<div class="flex-1 min-w-0">
|
|
<p class="text-sm text-gray-800 truncate" :title="part.name">{{ part.name }}</p>
|
|
<p class="text-xs text-gray-500">{{ part.type.name }}</p>
|
|
</div>
|
|
<button
|
|
type="button"
|
|
class="px-3 py-1 text-sm bg-indigo-600 text-white rounded hover:bg-indigo-700 transition shrink-0"
|
|
@click="importPart(part)"
|
|
>
|
|
Import
|
|
</button>
|
|
</div>
|
|
|
|
<div class="mt-2 pl-13 flex items-center gap-2 flex-wrap">
|
|
<button
|
|
v-if="!offers[part.id]"
|
|
type="button"
|
|
:disabled="comparing[part.id]"
|
|
class="text-xs px-2 py-1 border border-gray-300 rounded text-gray-600 hover:bg-gray-100 transition disabled:opacity-50"
|
|
@click="comparePrices(part)"
|
|
>
|
|
{{ comparing[part.id] ? 'Checking prices…' : 'Compare prices' }}
|
|
</button>
|
|
|
|
<template v-else-if="cheapestOffer(part)">
|
|
<img
|
|
:src="faviconUrl(merchantDomain(cheapestOffer(part)!.merchant))"
|
|
alt=""
|
|
class="w-4 h-4 shrink-0"
|
|
/>
|
|
<a
|
|
:href="cheapestOffer(part)!.url"
|
|
target="_blank"
|
|
rel="noopener noreferrer"
|
|
class="text-xs text-indigo-600 hover:underline"
|
|
>
|
|
{{ cheapestOffer(part)!.merchant }}: €{{ cheapestOffer(part)!.price.toFixed(2) }}
|
|
(cheapest)
|
|
</a>
|
|
</template>
|
|
|
|
<span v-if="compareErrors[part.id]" class="text-xs text-gray-400">
|
|
No live price found, using catalog price €{{ part.price.toFixed(2) }}
|
|
</span>
|
|
</div>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</template>
|