feat: search and import new parts scraped from pc-builder.io
Idealo and PCPartPicker were evaluated and ruled out: - idealo.fr search pages return 503 (Cloudflare) - the internal Idealo GraphQL endpoint used by a public scraper (app.idealo.de/app-backend/api) is protected by Akamai Bot Manager and silently returns empty data even with mobile app headers - PCPartPicker's product list/search relies on a session-bound CSRF token refreshed per page load, against a scraping-hostile ToS pc-builder.io category pages are server-rendered, include name/price/image for ~100 products per category, and have no bot protection, so the feature is built on top of that source only. - New scraped_parts cache table (type_id, name, price, image_url, source, source_url, source_ref unique per source) populated by a batch job rather than scraped live per search, since only ~100 first results per category are available without reverse-engineering pc-builder.io's DataTables/CSRF pagination flow - part_scraper_service.ts: maps our ComponentType names to pc-builder.io category slugs (SSD -> storage, since pc-builder.io has no separate HDD category) and parses each category page with cheerio, skipping rows without a resolved price (generic 'search on Amazon' links) - node ace scrape:parts: refreshes the cache for every mapped category (upsert on source+source_ref), runnable manually or later via cron - GET /scraped-parts (authenticated): search the cache by name (LOWER() LIKE, avoids a MySQL utf8_bin/utf8mb4 collation mismatch from Lucid's whereLike) and optional typeId filter - NewPartSearch.vue: debounced search widget wired into components/create.vue; clicking 'Import' pre-fills the existing component creation form (name, price, type, purchase date = today) instead of creating the Component directly, so the user still reviews and saves it like any manually entered part - 3 functional tests covering search by name, filter by typeId, and unauthenticated access being redirected to login - Initial scrape run: 989 parts across 10 categories
This commit is contained in:
1 parent
da71d33450
commit
5854ff8ed1
19 files changed
+757
-9
No files matched your search
@@ -0,0 +1,142 @@
|
||||
import * as cheerio from 'cheerio'
|
||||
import { DateTime } from 'luxon'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import ComponentType from '#models/component_type'
|
||||
import ScrapedPart from '#models/scraped_part'
|
||||
|
||||
const SOURCE = 'pc-builder.io'
|
||||
const BASE_URL = 'https://fr.pc-builder.io'
|
||||
|
||||
/**
|
||||
* Maps our own ComponentType names to the corresponding pc-builder.io
|
||||
* category slug. Categories not listed here are not scraped.
|
||||
*
|
||||
* Note: pc-builder.io only has a single "storage" category covering both
|
||||
* SSDs and HDDs (no separate slugs exist), so it is only mapped to our
|
||||
* "SSD" type to avoid two types racing to own the same scraped rows
|
||||
* (source_ref is unique per source, independent of type_id).
|
||||
*/
|
||||
const CATEGORY_SLUGS: Record<string, string> = {
|
||||
'Processor': 'cpu',
|
||||
'Motherboard': 'motherboard',
|
||||
'Graphic Card': 'gpu',
|
||||
'Memory': 'ram',
|
||||
'SSD': 'storage',
|
||||
'Cooler': 'cpu-cooler',
|
||||
'Power Supply': 'power-supply',
|
||||
'Case': 'case',
|
||||
'Fan': 'case-fan',
|
||||
'WiFi-Card': 'wireless-network-adapter',
|
||||
}
|
||||
|
||||
interface ScrapedRow {
|
||||
name: string
|
||||
price: number
|
||||
imageUrl: string | null
|
||||
sourceUrl: string
|
||||
sourceRef: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a pc-builder.io category page and parses the (server-rendered)
|
||||
* product table into a list of rows. Only the first ~100 products per
|
||||
* category are available without simulating the site's DataTables/CSRF
|
||||
* flow, which is an accepted limitation for this cache.
|
||||
*/
|
||||
async function fetchCategoryPage(slug: string): Promise<ScrapedRow[]> {
|
||||
const url = `${BASE_URL}/${slug}`
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; PcBuilderPartFinder/1.0)',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`)
|
||||
}
|
||||
|
||||
const html = await response.text()
|
||||
const $ = cheerio.load(html)
|
||||
const rows: ScrapedRow[] = []
|
||||
|
||||
$('#products-table tbody tr').each((_, el) => {
|
||||
const row = $(el)
|
||||
const nameLink = row.find('td.product a').first()
|
||||
const name = nameLink.text().trim()
|
||||
const sourceUrl = nameLink.attr('href')
|
||||
const img = row.find('td.product img.product-image')
|
||||
const imageUrl = img.attr('data-src') || img.attr('src') || null
|
||||
const priceText = row.find('td.add-btn .price-box p').first().text().trim()
|
||||
const addButton = row.find('a.add-to-build')
|
||||
const sourceRef = addButton.attr('data-id')
|
||||
|
||||
if (!name || !sourceUrl || !sourceRef || !priceText) {
|
||||
// Skip rows without a resolved price (e.g. "search on Amazon" links
|
||||
// with no fixed offer) or missing identifiers.
|
||||
return
|
||||
}
|
||||
|
||||
const price = Number.parseFloat(priceText.replace(/[^\d.,]/g, '').replace(',', '.'))
|
||||
if (Number.isNaN(price)) {
|
||||
return
|
||||
}
|
||||
|
||||
rows.push({ name, price, imageUrl, sourceUrl, sourceRef })
|
||||
})
|
||||
|
||||
return rows
|
||||
}
|
||||
|
||||
/**
|
||||
* Scrapes a single component type's mapped category and upserts the
|
||||
* resulting rows into the scraped_parts cache table.
|
||||
*/
|
||||
async function scrapeComponentType(componentType: ComponentType): Promise<number> {
|
||||
const slug = CATEGORY_SLUGS[componentType.name]
|
||||
if (!slug) {
|
||||
logger.warn({ type: componentType.name }, 'No pc-builder.io category mapped, skipping')
|
||||
return 0
|
||||
}
|
||||
|
||||
const rows = await fetchCategoryPage(slug)
|
||||
const scrapedAt = DateTime.now()
|
||||
|
||||
for (const row of rows) {
|
||||
await ScrapedPart.updateOrCreate(
|
||||
{ source: SOURCE, sourceRef: row.sourceRef },
|
||||
{
|
||||
typeId: componentType.id,
|
||||
name: row.name,
|
||||
price: row.price,
|
||||
imageUrl: row.imageUrl,
|
||||
sourceUrl: row.sourceUrl,
|
||||
scrapedAt,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
return rows.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Scrapes every mapped ComponentType category from pc-builder.io and
|
||||
* refreshes the scraped_parts cache. Returns a per-type summary.
|
||||
*/
|
||||
async function scrapeAllCategories(): Promise<Record<string, number>> {
|
||||
const componentTypes = await ComponentType.all()
|
||||
const summary: Record<string, number> = {}
|
||||
|
||||
for (const componentType of componentTypes) {
|
||||
if (!CATEGORY_SLUGS[componentType.name]) continue
|
||||
try {
|
||||
summary[componentType.name] = await scrapeComponentType(componentType)
|
||||
} catch (error) {
|
||||
logger.error({ type: componentType.name, err: error }, 'Failed to scrape category')
|
||||
summary[componentType.name] = 0
|
||||
}
|
||||
}
|
||||
|
||||
return summary
|
||||
}
|
||||
|
||||
export { scrapeAllCategories, scrapeComponentType, CATEGORY_SLUGS }
|
||||
Reference in new issue
Block a user