From 5854ff8ed1bf393f09076f88802c94e1a118e20e Mon Sep 17 00:00:00 2001 From: Kevin L'Hereec Date: Thu, 23 Jul 2026 12:10:36 +0200 Subject: [PATCH] feat: search and import new parts scraped from pc-builder.io Idealo and PCPartPicker were evaluated and ruled out: - idealo.fr search pages return 503 (Cloudflare) - the internal Idealo GraphQL endpoint used by a public scraper (app.idealo.de/app-backend/api) is protected by Akamai Bot Manager and silently returns empty data even with mobile app headers - PCPartPicker's product list/search relies on a session-bound CSRF token refreshed per page load, against a scraping-hostile ToS pc-builder.io category pages are server-rendered, include name/price/image for ~100 products per category, and have no bot protection, so the feature is built on top of that source only. - New scraped_parts cache table (type_id, name, price, image_url, source, source_url, source_ref unique per source) populated by a batch job rather than scraped live per search, since only ~100 first results per category are available without reverse-engineering pc-builder.io's DataTables/CSRF pagination flow - part_scraper_service.ts: maps our ComponentType names to pc-builder.io category slugs (SSD -> storage, since pc-builder.io has no separate HDD category) and parses each category page with cheerio, skipping rows without a resolved price (generic 'search on Amazon' links) - node ace scrape:parts: refreshes the cache for every mapped category (upsert on source+source_ref), runnable manually or later via cron - GET /scraped-parts (authenticated): search the cache by name (LOWER() LIKE, avoids a MySQL utf8_bin/utf8mb4 collation mismatch from Lucid's whereLike) and optional typeId filter - NewPartSearch.vue: debounced search widget wired into components/create.vue; clicking 'Import' pre-fills the existing component creation form (name, price, type, purchase date = today) instead of creating the Component directly, so the user still reviews and saves it like any manually entered part - 3 functional tests covering search by name, filter by typeId, and unauthenticated access being redirected to login - Initial scrape run: 989 parts across 10 categories --- .adonisjs/client/registry/index.ts | 6 + .adonisjs/client/registry/schema.d.ts | 12 + .adonisjs/client/registry/tree.d.ts | 3 + .adonisjs/server/controllers.ts | 1 + .adonisjs/server/pages.d.ts | 1 + .adonisjs/server/routes.d.ts | 3 + app/controllers/scraped_parts_controller.ts | 34 +++ app/models/scraped_part.ts | 11 + app/services/part_scraper_service.ts | 142 +++++++++ commands/scrape_parts.ts | 28 ++ ...784800932189_create_scraped_parts_table.ts | 28 ++ database/schema.ts | 27 ++ inertia/pages/components/create.vue | 21 +- inertia/pages/types/UITypes.ts | 10 + inertia/pages/widgets/NewPartSearch.vue | 87 ++++++ package-lock.json | 276 +++++++++++++++++- package.json | 1 + start/routes.ts | 5 + tests/functional/scraped_parts.spec.ts | 70 +++++ 19 files changed, 757 insertions(+), 9 deletions(-) create mode 100644 app/controllers/scraped_parts_controller.ts create mode 100644 app/models/scraped_part.ts create mode 100644 app/services/part_scraper_service.ts create mode 100644 commands/scrape_parts.ts create mode 100644 database/migrations/1784800932189_create_scraped_parts_table.ts create mode 100644 inertia/pages/widgets/NewPartSearch.vue create mode 100644 tests/functional/scraped_parts.spec.ts diff --git a/.adonisjs/client/registry/index.ts b/.adonisjs/client/registry/index.ts index b5e9771..b46a549 100644 --- a/.adonisjs/client/registry/index.ts +++ b/.adonisjs/client/registry/index.ts @@ -144,6 +144,12 @@ const routes = { tokens: [{"old":"/components/:id","type":0,"val":"components","end":""},{"old":"/components/:id","type":1,"val":"id","end":""}], types: placeholder as Registry['components.destroy']['types'], }, + 'scraped_parts.index': { + methods: ["GET","HEAD"], + pattern: '/scraped-parts', + tokens: [{"old":"/scraped-parts","type":0,"val":"scraped-parts","end":""}], + types: placeholder as Registry['scraped_parts.index']['types'], + }, } as const satisfies Record export { routes } diff --git a/.adonisjs/client/registry/schema.d.ts b/.adonisjs/client/registry/schema.d.ts index 2c8df0c..f08684d 100644 --- a/.adonisjs/client/registry/schema.d.ts +++ b/.adonisjs/client/registry/schema.d.ts @@ -283,4 +283,16 @@ export interface Registry { errorResponse: ExtractErrorResponse>> } } + 'scraped_parts.index': { + methods: ["GET","HEAD"] + pattern: '/scraped-parts' + types: { + body: {} + paramsTuple: [] + params: {} + query: {} + response: ExtractResponse>> + errorResponse: ExtractErrorResponse>> + } + } } diff --git a/.adonisjs/client/registry/tree.d.ts b/.adonisjs/client/registry/tree.d.ts index 03c555d..d071ee3 100644 --- a/.adonisjs/client/registry/tree.d.ts +++ b/.adonisjs/client/registry/tree.d.ts @@ -37,4 +37,7 @@ export interface ApiDefinition { update: typeof routes['components.update'] destroy: typeof routes['components.destroy'] } + scrapedParts: { + index: typeof routes['scraped_parts.index'] + } } diff --git a/.adonisjs/server/controllers.ts b/.adonisjs/server/controllers.ts index 371cf81..08adea2 100644 --- a/.adonisjs/server/controllers.ts +++ b/.adonisjs/server/controllers.ts @@ -9,5 +9,6 @@ export const controllers = { Computers: () => import('#controllers/computers_controller'), NewAccount: () => import('#controllers/new_account_controller'), PublicComputer: () => import('#controllers/public_computer_controller'), + ScrapedParts: () => import('#controllers/scraped_parts_controller'), Session: () => import('#controllers/session_controller'), } diff --git a/.adonisjs/server/pages.d.ts b/.adonisjs/server/pages.d.ts index 89463a0..9dfe3eb 100644 --- a/.adonisjs/server/pages.d.ts +++ b/.adonisjs/server/pages.d.ts @@ -37,6 +37,7 @@ declare module '@adonisjs/inertia/types' { 'widgets/CustomSelect': ExtractProps<(typeof import('../../inertia/pages/widgets/CustomSelect.vue'))['default']> 'widgets/DeleteConfirmationModal': ExtractProps<(typeof import('../../inertia/pages/widgets/DeleteConfirmationModal.vue'))['default']> 'widgets/FilterDropdown': ExtractProps<(typeof import('../../inertia/pages/widgets/FilterDropdown.vue'))['default']> + 'widgets/NewPartSearch': ExtractProps<(typeof import('../../inertia/pages/widgets/NewPartSearch.vue'))['default']> 'widgets/SimpleGallery': ExtractProps<(typeof import('../../inertia/pages/widgets/SimpleGallery.vue'))['default']> 'widgets/TableComponent': ExtractProps<(typeof import('../../inertia/pages/widgets/TableComponent.vue'))['default']> 'widgets/ValidationError': ExtractProps<(typeof import('../../inertia/pages/widgets/ValidationError.vue'))['default']> diff --git a/.adonisjs/server/routes.d.ts b/.adonisjs/server/routes.d.ts index b704291..8f4fb9d 100644 --- a/.adonisjs/server/routes.d.ts +++ b/.adonisjs/server/routes.d.ts @@ -27,6 +27,7 @@ export type ScannedRoutes = { 'components.edit': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'components.update': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'components.destroy': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'scraped_parts.index': { paramsTuple?: []; params?: {} } } GET: { 'home': { paramsTuple?: []; params?: {} } @@ -41,6 +42,7 @@ export type ScannedRoutes = { 'components.create': { paramsTuple?: []; params?: {} } 'components.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'components.edit': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'scraped_parts.index': { paramsTuple?: []; params?: {} } } HEAD: { 'home': { paramsTuple?: []; params?: {} } @@ -55,6 +57,7 @@ export type ScannedRoutes = { 'components.create': { paramsTuple?: []; params?: {} } 'components.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'components.edit': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'scraped_parts.index': { paramsTuple?: []; params?: {} } } POST: { 'new_account.store': { paramsTuple?: []; params?: {} } diff --git a/app/controllers/scraped_parts_controller.ts b/app/controllers/scraped_parts_controller.ts new file mode 100644 index 0000000..fe6059d --- /dev/null +++ b/app/controllers/scraped_parts_controller.ts @@ -0,0 +1,34 @@ +import type { HttpContext } from '@adonisjs/core/http' +import vine from '@vinejs/vine' +import ScrapedPart from '#models/scraped_part' + +const searchValidator = vine.compile( + vine.object({ + q: vine.string().trim().minLength(1).optional(), + typeId: vine.number().min(1).optional(), + }) +) + +export default class ScrapedPartsController { + /** + * Search the new-parts cache (scraped from pc-builder.io) by name and/or + * component type. Read-only, authenticated (the search UI lives inside + * the authenticated "create component" flow). + */ + async index({ request, auth, response }: HttpContext) { + await auth.authenticate() + const { q, typeId } = await request.validateUsing(searchValidator) + + const query = ScrapedPart.query().preload('type').orderBy('name') + + if (q) { + query.whereRaw('LOWER(name) LIKE ?', [`%${q.toLowerCase()}%`]) + } + if (typeId) { + query.where('typeId', typeId) + } + + const parts = await query.limit(30) + return response.json(parts.map((part) => part.serialize())) + } +} diff --git a/app/models/scraped_part.ts b/app/models/scraped_part.ts new file mode 100644 index 0000000..f67b434 --- /dev/null +++ b/app/models/scraped_part.ts @@ -0,0 +1,11 @@ +import { belongsTo } from '@adonisjs/lucid/orm' +import type { BelongsTo } from '@adonisjs/lucid/types/relations' +import { ScrapedPartSchema } from '#database/schema' +import ComponentType from '#models/component_type' + +export default class ScrapedPart extends ScrapedPartSchema { + @belongsTo(() => ComponentType, { + foreignKey: 'typeId', + }) + declare type: BelongsTo +} diff --git a/app/services/part_scraper_service.ts b/app/services/part_scraper_service.ts new file mode 100644 index 0000000..aaa7c48 --- /dev/null +++ b/app/services/part_scraper_service.ts @@ -0,0 +1,142 @@ +import * as cheerio from 'cheerio' +import { DateTime } from 'luxon' +import logger from '@adonisjs/core/services/logger' +import ComponentType from '#models/component_type' +import ScrapedPart from '#models/scraped_part' + +const SOURCE = 'pc-builder.io' +const BASE_URL = 'https://fr.pc-builder.io' + +/** + * Maps our own ComponentType names to the corresponding pc-builder.io + * category slug. Categories not listed here are not scraped. + * + * Note: pc-builder.io only has a single "storage" category covering both + * SSDs and HDDs (no separate slugs exist), so it is only mapped to our + * "SSD" type to avoid two types racing to own the same scraped rows + * (source_ref is unique per source, independent of type_id). + */ +const CATEGORY_SLUGS: Record = { + 'Processor': 'cpu', + 'Motherboard': 'motherboard', + 'Graphic Card': 'gpu', + 'Memory': 'ram', + 'SSD': 'storage', + 'Cooler': 'cpu-cooler', + 'Power Supply': 'power-supply', + 'Case': 'case', + 'Fan': 'case-fan', + 'WiFi-Card': 'wireless-network-adapter', +} + +interface ScrapedRow { + name: string + price: number + imageUrl: string | null + sourceUrl: string + sourceRef: string +} + +/** + * Fetches a pc-builder.io category page and parses the (server-rendered) + * product table into a list of rows. Only the first ~100 products per + * category are available without simulating the site's DataTables/CSRF + * flow, which is an accepted limitation for this cache. + */ +async function fetchCategoryPage(slug: string): Promise { + const url = `${BASE_URL}/${slug}` + const response = await fetch(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (compatible; PcBuilderPartFinder/1.0)', + }, + }) + + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`) + } + + const html = await response.text() + const $ = cheerio.load(html) + const rows: ScrapedRow[] = [] + + $('#products-table tbody tr').each((_, el) => { + const row = $(el) + const nameLink = row.find('td.product a').first() + const name = nameLink.text().trim() + const sourceUrl = nameLink.attr('href') + const img = row.find('td.product img.product-image') + const imageUrl = img.attr('data-src') || img.attr('src') || null + const priceText = row.find('td.add-btn .price-box p').first().text().trim() + const addButton = row.find('a.add-to-build') + const sourceRef = addButton.attr('data-id') + + if (!name || !sourceUrl || !sourceRef || !priceText) { + // Skip rows without a resolved price (e.g. "search on Amazon" links + // with no fixed offer) or missing identifiers. + return + } + + const price = Number.parseFloat(priceText.replace(/[^\d.,]/g, '').replace(',', '.')) + if (Number.isNaN(price)) { + return + } + + rows.push({ name, price, imageUrl, sourceUrl, sourceRef }) + }) + + return rows +} + +/** + * Scrapes a single component type's mapped category and upserts the + * resulting rows into the scraped_parts cache table. + */ +async function scrapeComponentType(componentType: ComponentType): Promise { + const slug = CATEGORY_SLUGS[componentType.name] + if (!slug) { + logger.warn({ type: componentType.name }, 'No pc-builder.io category mapped, skipping') + return 0 + } + + const rows = await fetchCategoryPage(slug) + const scrapedAt = DateTime.now() + + for (const row of rows) { + await ScrapedPart.updateOrCreate( + { source: SOURCE, sourceRef: row.sourceRef }, + { + typeId: componentType.id, + name: row.name, + price: row.price, + imageUrl: row.imageUrl, + sourceUrl: row.sourceUrl, + scrapedAt, + } + ) + } + + return rows.length +} + +/** + * Scrapes every mapped ComponentType category from pc-builder.io and + * refreshes the scraped_parts cache. Returns a per-type summary. + */ +async function scrapeAllCategories(): Promise> { + const componentTypes = await ComponentType.all() + const summary: Record = {} + + for (const componentType of componentTypes) { + if (!CATEGORY_SLUGS[componentType.name]) continue + try { + summary[componentType.name] = await scrapeComponentType(componentType) + } catch (error) { + logger.error({ type: componentType.name, err: error }, 'Failed to scrape category') + summary[componentType.name] = 0 + } + } + + return summary +} + +export { scrapeAllCategories, scrapeComponentType, CATEGORY_SLUGS } diff --git a/commands/scrape_parts.ts b/commands/scrape_parts.ts new file mode 100644 index 0000000..af7824d --- /dev/null +++ b/commands/scrape_parts.ts @@ -0,0 +1,28 @@ +import { BaseCommand } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' + +export default class ScrapeParts extends BaseCommand { + static commandName = 'scrape:parts' + static description = 'Refresh the new-parts cache by scraping pc-builder.io category pages' + + static options: CommandOptions = { + startApp: true, + } + + async run() { + const { scrapeAllCategories } = await import('#services/part_scraper_service') + + this.logger.info('Scraping pc-builder.io categories...') + const summary = await scrapeAllCategories() + + let total = 0 + for (const [type, count] of Object.entries(summary)) { + this.logger.info(` ${type}: ${count} parts`) + total += count + } + + this.logger.success( + `Done. Upserted ${total} parts across ${Object.keys(summary).length} categories.` + ) + } +} diff --git a/database/migrations/1784800932189_create_scraped_parts_table.ts b/database/migrations/1784800932189_create_scraped_parts_table.ts new file mode 100644 index 0000000..eeaddce --- /dev/null +++ b/database/migrations/1784800932189_create_scraped_parts_table.ts @@ -0,0 +1,28 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'scraped_parts' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.increments('id') + table.integer('type_id').unsigned().references('id').inTable('component_types').notNullable() + table.string('name').notNullable() + table.float('price', 8, 2).notNullable() + table.string('image_url').nullable() + table.string('source').notNullable().defaultTo('pc-builder.io') + table.string('source_url').notNullable() + table.string('source_ref').notNullable() + table.timestamp('scraped_at').notNullable() + table.timestamp('created_at') + table.timestamp('updated_at') + + table.unique(['source', 'source_ref']) + table.index(['type_id']) + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/database/schema.ts b/database/schema.ts index 128cd37..2608b7f 100644 --- a/database/schema.ts +++ b/database/schema.ts @@ -100,6 +100,33 @@ export class ComputerSchema extends BaseModel { declare userId: number | null } +export class ScrapedPartSchema extends BaseModel { + static $columns = ['createdAt', 'id', 'imageUrl', 'name', 'price', 'scrapedAt', 'source', 'sourceRef', 'sourceUrl', 'typeId', 'updatedAt'] as const + $columns = ScrapedPartSchema.$columns + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime | null + @column({ isPrimary: true }) + declare id: number + @column() + declare imageUrl: string | null + @column() + declare name: string + @column() + declare price: number + @column.dateTime() + declare scrapedAt: DateTime + @column() + declare source: string + @column() + declare sourceRef: string + @column() + declare sourceUrl: string + @column() + declare typeId: number + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime | null +} + export class UserSchema extends BaseModel { static $columns = ['createdAt', 'email', 'id', 'name', 'password', 'updatedAt'] as const $columns = UserSchema.$columns diff --git a/inertia/pages/components/create.vue b/inertia/pages/components/create.vue index e6fbafc..cd5a22b 100644 --- a/inertia/pages/components/create.vue +++ b/inertia/pages/components/create.vue @@ -3,17 +3,25 @@ import { reactive } from 'vue' import { Link } from '@adonisjs/inertia/vue' import { useForm } from '@inertiajs/vue3' import CustomSelect from '~/pages/widgets/CustomSelect.vue' -import type { ComponentType } from '~/pages/types/UITypes' +import NewPartSearch from '~/pages/widgets/NewPartSearch.vue' +import type { ComponentType, ScrapedPart } from '~/pages/types/UITypes' defineProps<{ types: ComponentType[]; errors?: Record }>() const form = reactive({ name: '', price: 0, - typeId: null, + typeId: null as number | null, purchaseDate: new Date().toISOString().split('T')[0], quantity: 1, }) +function importPart(part: ScrapedPart) { + form.name = part.name + form.price = part.price + form.typeId = part.type.id + form.purchaseDate = new Date().toISOString().split('T')[0] +} + function submitForm() { const inertiaForm = useForm(form) inertiaForm.post('/components') @@ -23,6 +31,9 @@ function submitForm() {