feat: search and import new parts scraped from pc-builder.io

Idealo and PCPartPicker were evaluated and ruled out:
- idealo.fr search pages return 503 (Cloudflare)
- the internal Idealo GraphQL endpoint used by a public scraper
  (app.idealo.de/app-backend/api) is protected by Akamai Bot Manager and
  silently returns empty data even with mobile app headers
- PCPartPicker's product list/search relies on a session-bound CSRF token
  refreshed per page load, against a scraping-hostile ToS

pc-builder.io category pages are server-rendered, include name/price/image
for ~100 products per category, and have no bot protection, so the feature
is built on top of that source only.

- New scraped_parts cache table (type_id, name, price, image_url, source,
  source_url, source_ref unique per source) populated by a batch job
  rather than scraped live per search, since only ~100 first results per
  category are available without reverse-engineering pc-builder.io's
  DataTables/CSRF pagination flow
- part_scraper_service.ts: maps our ComponentType names to pc-builder.io
  category slugs (SSD -> storage, since pc-builder.io has no separate HDD
  category) and parses each category page with cheerio, skipping rows
  without a resolved price (generic 'search on Amazon' links)
- node ace scrape:parts: refreshes the cache for every mapped category
  (upsert on source+source_ref), runnable manually or later via cron
- GET /scraped-parts (authenticated): search the cache by name (LOWER()
  LIKE, avoids a MySQL utf8_bin/utf8mb4 collation mismatch from Lucid's
  whereLike) and optional typeId filter
- NewPartSearch.vue: debounced search widget wired into
  components/create.vue; clicking 'Import' pre-fills the existing
  component creation form (name, price, type, purchase date = today)
  instead of creating the Component directly, so the user still reviews
  and saves it like any manually entered part
- 3 functional tests covering search by name, filter by typeId, and
  unauthenticated access being redirected to login
- Initial scrape run: 989 parts across 10 categories

.adonisjs/client/registry/index.ts

@@ -144,6 +144,12 @@ const routes = {
     tokens: [{"old":"/components/:id","type":0,"val":"components","end":""},{"old":"/components/:id","type":1,"val":"id","end":""}],
     types: placeholder as Registry['components.destroy']['types'],
   },
+  'scraped_parts.index': {
+    methods: ["GET","HEAD"],
+    pattern: '/scraped-parts',
+    tokens: [{"old":"/scraped-parts","type":0,"val":"scraped-parts","end":""}],
+    types: placeholder as Registry['scraped_parts.index']['types'],
+  },
 } as const satisfies Record<string, AdonisEndpoint>
 
 export { routes }

.adonisjs/client/registry/schema.d.ts

@@ -283,4 +283,16 @@ export interface Registry {
       errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/components_controller').default['destroy']>>>
     }
   }
+  'scraped_parts.index': {
+    methods: ["GET","HEAD"]
+    pattern: '/scraped-parts'
+    types: {
+      body: {}
+      paramsTuple: []
+      params: {}
+      query: {}
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/scraped_parts_controller').default['index']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/scraped_parts_controller').default['index']>>>
+    }
+  }
 }

.adonisjs/client/registry/tree.d.ts

@@ -37,4 +37,7 @@ export interface ApiDefinition {
     update: typeof routes['components.update']
     destroy: typeof routes['components.destroy']
   }
+  scrapedParts: {
+    index: typeof routes['scraped_parts.index']
+  }
 }

.adonisjs/server/controllers.ts

@@ -9,5 +9,6 @@ export const controllers = {
   Computers: () => import('#controllers/computers_controller'),
   NewAccount: () => import('#controllers/new_account_controller'),
   PublicComputer: () => import('#controllers/public_computer_controller'),
+  ScrapedParts: () => import('#controllers/scraped_parts_controller'),
   Session: () => import('#controllers/session_controller'),
 }

.adonisjs/server/pages.d.ts

@@ -37,6 +37,7 @@ declare module '@adonisjs/inertia/types' {
     'widgets/CustomSelect': ExtractProps<(typeof import('../../inertia/pages/widgets/CustomSelect.vue'))['default']>
     'widgets/DeleteConfirmationModal': ExtractProps<(typeof import('../../inertia/pages/widgets/DeleteConfirmationModal.vue'))['default']>
     'widgets/FilterDropdown': ExtractProps<(typeof import('../../inertia/pages/widgets/FilterDropdown.vue'))['default']>
+    'widgets/NewPartSearch': ExtractProps<(typeof import('../../inertia/pages/widgets/NewPartSearch.vue'))['default']>
     'widgets/SimpleGallery': ExtractProps<(typeof import('../../inertia/pages/widgets/SimpleGallery.vue'))['default']>
     'widgets/TableComponent': ExtractProps<(typeof import('../../inertia/pages/widgets/TableComponent.vue'))['default']>
     'widgets/ValidationError': ExtractProps<(typeof import('../../inertia/pages/widgets/ValidationError.vue'))['default']>

.adonisjs/server/routes.d.ts

@@ -27,6 +27,7 @@ export type ScannedRoutes = {
     'components.edit': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'components.update': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'components.destroy': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'scraped_parts.index': { paramsTuple?: []; params?: {} }
   }
   GET: {
     'home': { paramsTuple?: []; params?: {} }
@@ -41,6 +42,7 @@ export type ScannedRoutes = {
     'components.create': { paramsTuple?: []; params?: {} }
     'components.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'components.edit': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'scraped_parts.index': { paramsTuple?: []; params?: {} }
   }
   HEAD: {
     'home': { paramsTuple?: []; params?: {} }
@@ -55,6 +57,7 @@ export type ScannedRoutes = {
     'components.create': { paramsTuple?: []; params?: {} }
     'components.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'components.edit': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'scraped_parts.index': { paramsTuple?: []; params?: {} }
   }
   POST: {
     'new_account.store': { paramsTuple?: []; params?: {} }

app/controllers/scraped_parts_controller.ts

@@ -0,0 +1,34 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import vine from '@vinejs/vine'
+import ScrapedPart from '#models/scraped_part'
+
+const searchValidator = vine.compile(
+  vine.object({
+    q: vine.string().trim().minLength(1).optional(),
+    typeId: vine.number().min(1).optional(),
+  })
+)
+
+export default class ScrapedPartsController {
+  /**
+   * Search the new-parts cache (scraped from pc-builder.io) by name and/or
+   * component type. Read-only, authenticated (the search UI lives inside
+   * the authenticated "create component" flow).
+   */
+  async index({ request, auth, response }: HttpContext) {
+    await auth.authenticate()
+    const { q, typeId } = await request.validateUsing(searchValidator)
+
+    const query = ScrapedPart.query().preload('type').orderBy('name')
+
+    if (q) {
+      query.whereRaw('LOWER(name) LIKE ?', [`%${q.toLowerCase()}%`])
+    }
+    if (typeId) {
+      query.where('typeId', typeId)
+    }
+
+    const parts = await query.limit(30)
+    return response.json(parts.map((part) => part.serialize()))
+  }
+}

app/models/scraped_part.ts

@@ -0,0 +1,11 @@
+import { belongsTo } from '@adonisjs/lucid/orm'
+import type { BelongsTo } from '@adonisjs/lucid/types/relations'
+import { ScrapedPartSchema } from '#database/schema'
+import ComponentType from '#models/component_type'
+
+export default class ScrapedPart extends ScrapedPartSchema {
+  @belongsTo(() => ComponentType, {
+    foreignKey: 'typeId',
+  })
+  declare type: BelongsTo<typeof ComponentType>
+}

app/services/part_scraper_service.ts

@@ -0,0 +1,142 @@
+import * as cheerio from 'cheerio'
+import { DateTime } from 'luxon'
+import logger from '@adonisjs/core/services/logger'
+import ComponentType from '#models/component_type'
+import ScrapedPart from '#models/scraped_part'
+
+const SOURCE = 'pc-builder.io'
+const BASE_URL = 'https://fr.pc-builder.io'
+
+/**
+ * Maps our own ComponentType names to the corresponding pc-builder.io
+ * category slug. Categories not listed here are not scraped.
+ *
+ * Note: pc-builder.io only has a single "storage" category covering both
+ * SSDs and HDDs (no separate slugs exist), so it is only mapped to our
+ * "SSD" type to avoid two types racing to own the same scraped rows
+ * (source_ref is unique per source, independent of type_id).
+ */
+const CATEGORY_SLUGS: Record<string, string> = {
+  'Processor': 'cpu',
+  'Motherboard': 'motherboard',
+  'Graphic Card': 'gpu',
+  'Memory': 'ram',
+  'SSD': 'storage',
+  'Cooler': 'cpu-cooler',
+  'Power Supply': 'power-supply',
+  'Case': 'case',
+  'Fan': 'case-fan',
+  'WiFi-Card': 'wireless-network-adapter',
+}
+
+interface ScrapedRow {
+  name: string
+  price: number
+  imageUrl: string | null
+  sourceUrl: string
+  sourceRef: string
+}
+
+/**
+ * Fetches a pc-builder.io category page and parses the (server-rendered)
+ * product table into a list of rows. Only the first ~100 products per
+ * category are available without simulating the site's DataTables/CSRF
+ * flow, which is an accepted limitation for this cache.
+ */
+async function fetchCategoryPage(slug: string): Promise<ScrapedRow[]> {
+  const url = `${BASE_URL}/${slug}`
+  const response = await fetch(url, {
+    headers: {
+      'User-Agent': 'Mozilla/5.0 (compatible; PcBuilderPartFinder/1.0)',
+    },
+  })
+
+  if (!response.ok) {
+    throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`)
+  }
+
+  const html = await response.text()
+  const $ = cheerio.load(html)
+  const rows: ScrapedRow[] = []
+
+  $('#products-table tbody tr').each((_, el) => {
+    const row = $(el)
+    const nameLink = row.find('td.product a').first()
+    const name = nameLink.text().trim()
+    const sourceUrl = nameLink.attr('href')
+    const img = row.find('td.product img.product-image')
+    const imageUrl = img.attr('data-src') || img.attr('src') || null
+    const priceText = row.find('td.add-btn .price-box p').first().text().trim()
+    const addButton = row.find('a.add-to-build')
+    const sourceRef = addButton.attr('data-id')
+
+    if (!name || !sourceUrl || !sourceRef || !priceText) {
+      // Skip rows without a resolved price (e.g. "search on Amazon" links
+      // with no fixed offer) or missing identifiers.
+      return
+    }
+
+    const price = Number.parseFloat(priceText.replace(/[^\d.,]/g, '').replace(',', '.'))
+    if (Number.isNaN(price)) {
+      return
+    }
+
+    rows.push({ name, price, imageUrl, sourceUrl, sourceRef })
+  })
+
+  return rows
+}
+
+/**
+ * Scrapes a single component type's mapped category and upserts the
+ * resulting rows into the scraped_parts cache table.
+ */
+async function scrapeComponentType(componentType: ComponentType): Promise<number> {
+  const slug = CATEGORY_SLUGS[componentType.name]
+  if (!slug) {
+    logger.warn({ type: componentType.name }, 'No pc-builder.io category mapped, skipping')
+    return 0
+  }
+
+  const rows = await fetchCategoryPage(slug)
+  const scrapedAt = DateTime.now()
+
+  for (const row of rows) {
+    await ScrapedPart.updateOrCreate(
+      { source: SOURCE, sourceRef: row.sourceRef },
+      {
+        typeId: componentType.id,
+        name: row.name,
+        price: row.price,
+        imageUrl: row.imageUrl,
+        sourceUrl: row.sourceUrl,
+        scrapedAt,
+      }
+    )
+  }
+
+  return rows.length
+}
+
+/**
+ * Scrapes every mapped ComponentType category from pc-builder.io and
+ * refreshes the scraped_parts cache. Returns a per-type summary.
+ */
+async function scrapeAllCategories(): Promise<Record<string, number>> {
+  const componentTypes = await ComponentType.all()
+  const summary: Record<string, number> = {}
+
+  for (const componentType of componentTypes) {
+    if (!CATEGORY_SLUGS[componentType.name]) continue
+    try {
+      summary[componentType.name] = await scrapeComponentType(componentType)
+    } catch (error) {
+      logger.error({ type: componentType.name, err: error }, 'Failed to scrape category')
+      summary[componentType.name] = 0
+    }
+  }
+
+  return summary
+}
+
+export { scrapeAllCategories, scrapeComponentType, CATEGORY_SLUGS }

commands/scrape_parts.ts

@@ -0,0 +1,28 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+
+export default class ScrapeParts extends BaseCommand {
+  static commandName = 'scrape:parts'
+  static description = 'Refresh the new-parts cache by scraping pc-builder.io category pages'
+
+  static options: CommandOptions = {
+    startApp: true,
+  }
+
+  async run() {
+    const { scrapeAllCategories } = await import('#services/part_scraper_service')
+
+    this.logger.info('Scraping pc-builder.io categories...')
+    const summary = await scrapeAllCategories()
+
+    let total = 0
+    for (const [type, count] of Object.entries(summary)) {
+      this.logger.info(`  ${type}: ${count} parts`)
+      total += count
+    }
+
+    this.logger.success(
+      `Done. Upserted ${total} parts across ${Object.keys(summary).length} categories.`
+    )
+  }
+}

database/migrations/1784800932189_create_scraped_parts_table.ts

@@ -0,0 +1,28 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+export default class extends BaseSchema {
+  protected tableName = 'scraped_parts'
+
+  async up() {
+    this.schema.createTable(this.tableName, (table) => {
+      table.increments('id')
+      table.integer('type_id').unsigned().references('id').inTable('component_types').notNullable()
+      table.string('name').notNullable()
+      table.float('price', 8, 2).notNullable()
+      table.string('image_url').nullable()
+      table.string('source').notNullable().defaultTo('pc-builder.io')
+      table.string('source_url').notNullable()
+      table.string('source_ref').notNullable()
+      table.timestamp('scraped_at').notNullable()
+      table.timestamp('created_at')
+      table.timestamp('updated_at')
+
+      table.unique(['source', 'source_ref'])
+      table.index(['type_id'])
+    })
+  }
+
+  async down() {
+    this.schema.dropTable(this.tableName)
+  }
+}

database/schema.ts

@@ -100,6 +100,33 @@ export class ComputerSchema extends BaseModel {
   declare userId: number | null
 }
 
+export class ScrapedPartSchema extends BaseModel {
+  static $columns = ['createdAt', 'id', 'imageUrl', 'name', 'price', 'scrapedAt', 'source', 'sourceRef', 'sourceUrl', 'typeId', 'updatedAt'] as const
+  $columns = ScrapedPartSchema.$columns
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime | null
+  @column({ isPrimary: true })
+  declare id: number
+  @column()
+  declare imageUrl: string | null
+  @column()
+  declare name: string
+  @column()
+  declare price: number
+  @column.dateTime()
+  declare scrapedAt: DateTime
+  @column()
+  declare source: string
+  @column()
+  declare sourceRef: string
+  @column()
+  declare sourceUrl: string
+  @column()
+  declare typeId: number
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime | null
+}
+
 export class UserSchema extends BaseModel {
   static $columns = ['createdAt', 'email', 'id', 'name', 'password', 'updatedAt'] as const
   $columns = UserSchema.$columns

inertia/pages/components/create.vue

@@ -3,17 +3,25 @@ import { reactive } from 'vue'
 import { Link } from '@adonisjs/inertia/vue'
 import { useForm } from '@inertiajs/vue3'
 import CustomSelect from '~/pages/widgets/CustomSelect.vue'
-import type { ComponentType } from '~/pages/types/UITypes'
+import NewPartSearch from '~/pages/widgets/NewPartSearch.vue'
+import type { ComponentType, ScrapedPart } from '~/pages/types/UITypes'
 
 defineProps<{ types: ComponentType[]; errors?: Record<string, string> }>()
 const form = reactive({
   name: '',
   price: 0,
-  typeId: null,
+  typeId: null as number | null,
   purchaseDate: new Date().toISOString().split('T')[0],
   quantity: 1,
 })
 
+function importPart(part: ScrapedPart) {
+  form.name = part.name
+  form.price = part.price
+  form.typeId = part.type.id
+  form.purchaseDate = new Date().toISOString().split('T')[0]
+}
+
 function submitForm() {
   const inertiaForm = useForm(form)
   inertiaForm.post('/components')
@@ -23,6 +31,9 @@ function submitForm() {
 <template>
   <div class="max-w-2xl mx-auto p-8 bg-white shadow-lg rounded-lg">
     <h1 class="text-2xl font-semibold mb-8 text-gray-800">Create New Component</h1>
+
+    <NewPartSearch class="mb-8" @import="importPart" />
+
     <form @submit.prevent="submitForm" class="space-y-6">
       <div>
         <label class="block text-gray-700 text-sm font-bold mb-2" for="name">Name</label>
@@ -48,11 +59,7 @@ function submitForm() {
 
       <div>
         <label class="block text-gray-700 text-sm font-bold mb-2" for="type">Type</label>
-        <CustomSelect
-          v-model="form.typeId"
-          :options="types"
-          placeholder="Choose a type"
-        />
+        <CustomSelect v-model="form.typeId" :options="types" placeholder="Choose a type" />
       </div>
 
       <div>

inertia/pages/types/UITypes.ts

@@ -50,6 +50,16 @@ export type Component = {
   type: ComponentType
 }
 
+export type ScrapedPart = {
+  id: number
+  name: string
+  price: number
+  imageUrl: string | null
+  source: string
+  sourceUrl: string
+  type: ComponentType
+}
+
 export interface ServerImage {
   id: number
   link: string

inertia/pages/widgets/NewPartSearch.vue

@@ -0,0 +1,87 @@
+<script setup lang="ts">
+import { ref, watch } from 'vue'
+import type { ScrapedPart } from '~/pages/types/UITypes'
+
+const emit = defineEmits<{
+  import: [part: ScrapedPart]
+}>()
+
+const query = ref('')
+const results = ref<ScrapedPart[]>([])
+const isLoading = ref(false)
+const hasSearched = ref(false)
+let debounceTimer: ReturnType<typeof setTimeout> | null = null
+
+async function search() {
+  if (!query.value.trim()) {
+    results.value = []
+    hasSearched.value = false
+    return
+  }
+
+  isLoading.value = true
+  try {
+    const response = await fetch(`/scraped-parts?q=${encodeURIComponent(query.value.trim())}`, {
+      headers: { Accept: 'application/json' },
+    })
+    if (response.ok) {
+      results.value = await response.json()
+    } else {
+      results.value = []
+    }
+  } finally {
+    isLoading.value = false
+    hasSearched.value = true
+  }
+}
+
+watch(query, () => {
+  if (debounceTimer) clearTimeout(debounceTimer)
+  debounceTimer = setTimeout(search, 400)
+})
+</script>
+
+<template>
+  <div class="border border-gray-200 rounded-lg p-4 bg-gray-50">
+    <label class="block text-gray-700 text-sm font-bold mb-2">
+      Search a new part (pc-builder.io)
+    </label>
+    <input
+      v-model="query"
+      type="text"
+      placeholder="e.g. Ryzen 5 7600X, RTX 4070..."
+      class="w-full border-gray-200 focus:border-indigo-600 focus:ring focus:ring-indigo-500/40 px-3 py-2 border rounded text-gray-700"
+    />
+
+    <p v-if="isLoading" class="text-sm text-gray-500 mt-2">Searching…</p>
+    <p v-else-if="hasSearched && results.length === 0" class="text-sm text-gray-500 mt-2">
+      No matching part found.
+    </p>
+
+    <ul v-if="results.length > 0" class="mt-3 space-y-2 max-h-72 overflow-y-auto">
+      <li
+        v-for="part in results"
+        :key="part.id"
+        class="flex items-center gap-3 bg-white border border-gray-200 rounded p-2"
+      >
+        <img
+          v-if="part.imageUrl"
+          :src="part.imageUrl"
+          alt=""
+          class="w-10 h-10 object-contain shrink-0"
+        />
+        <div class="flex-1 min-w-0">
+          <p class="text-sm text-gray-800 truncate" :title="part.name">{{ part.name }}</p>
+          <p class="text-xs text-gray-500">{{ part.type.name }} · €{{ part.price.toFixed(2) }}</p>
+        </div>
+        <button
+          type="button"
+          class="px-3 py-1 text-sm bg-indigo-600 text-white rounded hover:bg-indigo-700 transition shrink-0"
+          @click="emit('import', part)"
+        >
+          Import
+        </button>
+      </li>
+    </ul>
+  </div>
+</template>

package-lock.json

@@ -25,6 +25,7 @@
         "@vue/server-renderer": "^3.5.35",
         "@vuepic/vue-datepicker": "^13.0.0",
         "better-sqlite3": "^12.10.0",
+        "cheerio": "^1.2.0",
         "edge.js": "^6.5.1",
         "luxon": "^3.7.2",
         "mysql2": "^3.22.4",
@@ -4158,7 +4159,6 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
       "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
-      "dev": true,
       "license": "ISC"
     },
     "node_modules/brace-expansion": {
@@ -4422,6 +4422,48 @@
         "node": ">=16"
       }
     },
+    "node_modules/cheerio": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz",
+      "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==",
+      "license": "MIT",
+      "dependencies": {
+        "cheerio-select": "^2.1.0",
+        "dom-serializer": "^2.0.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.2.2",
+        "encoding-sniffer": "^0.2.1",
+        "htmlparser2": "^10.1.0",
+        "parse5": "^7.3.0",
+        "parse5-htmlparser2-tree-adapter": "^7.1.0",
+        "parse5-parser-stream": "^7.1.2",
+        "undici": "^7.19.0",
+        "whatwg-mimetype": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=20.18.1"
+      },
+      "funding": {
+        "url": "https://github.com/cheeriojs/cheerio?sponsor=1"
+      }
+    },
+    "node_modules/cheerio-select": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
+      "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "boolbase": "^1.0.0",
+        "css-select": "^5.1.0",
+        "css-what": "^6.1.0",
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.0.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/fb55"
+      }
+    },
     "node_modules/chokidar": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz",
@@ -4717,6 +4759,34 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/css-select": {
+      "version": "5.2.2",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz",
+      "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "boolbase": "^1.0.0",
+        "css-what": "^6.1.0",
+        "domhandler": "^5.0.2",
+        "domutils": "^3.0.1",
+        "nth-check": "^2.0.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/fb55"
+      }
+    },
+    "node_modules/css-what": {
+      "version": "6.2.2",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz",
+      "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">= 6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/fb55"
+      }
+    },
     "node_modules/cssesc": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz",
@@ -4947,6 +5017,73 @@
       "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==",
       "license": "MIT"
     },
+    "node_modules/dom-serializer": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+      "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
+      "license": "MIT",
+      "dependencies": {
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.2",
+        "entities": "^4.2.0"
+      },
+      "funding": {
+        "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
+      }
+    },
+    "node_modules/dom-serializer/node_modules/entities": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
+      }
+    },
+    "node_modules/domelementtype": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+      "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fb55"
+        }
+      ],
+      "license": "BSD-2-Clause"
+    },
+    "node_modules/domhandler": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+      "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "domelementtype": "^2.3.0"
+      },
+      "engines": {
+        "node": ">= 4"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/domhandler?sponsor=1"
+      }
+    },
+    "node_modules/domutils": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
+      "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "dom-serializer": "^2.0.0",
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/domutils?sponsor=1"
+      }
+    },
     "node_modules/dunder-proto": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -5058,6 +5195,31 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/encoding-sniffer": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz",
+      "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==",
+      "license": "MIT",
+      "dependencies": {
+        "iconv-lite": "^0.6.3",
+        "whatwg-encoding": "^3.1.1"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/encoding-sniffer?sponsor=1"
+      }
+    },
+    "node_modules/encoding-sniffer/node_modules/iconv-lite": {
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+      "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+      "license": "MIT",
+      "dependencies": {
+        "safer-buffer": ">= 2.1.2 < 3.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/end-of-stream": {
       "version": "1.4.5",
       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",
@@ -6269,6 +6431,25 @@
       ],
       "license": "MIT"
     },
+    "node_modules/htmlparser2": {
+      "version": "10.1.0",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz",
+      "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==",
+      "funding": [
+        "https://github.com/fb55/htmlparser2?sponsor=1",
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fb55"
+        }
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.2.2",
+        "entities": "^7.0.1"
+      }
+    },
     "node_modules/http-errors": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
@@ -7473,7 +7654,6 @@
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
       "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
-      "dev": true,
       "license": "BSD-2-Clause",
       "dependencies": {
         "boolbase": "^1.0.0"
@@ -7703,6 +7883,55 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/parse5": {
+      "version": "7.3.0",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
+      "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
+      "license": "MIT",
+      "dependencies": {
+        "entities": "^6.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/inikulin/parse5?sponsor=1"
+      }
+    },
+    "node_modules/parse5-htmlparser2-tree-adapter": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz",
+      "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==",
+      "license": "MIT",
+      "dependencies": {
+        "domhandler": "^5.0.3",
+        "parse5": "^7.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/inikulin/parse5?sponsor=1"
+      }
+    },
+    "node_modules/parse5-parser-stream": {
+      "version": "7.1.2",
+      "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz",
+      "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==",
+      "license": "MIT",
+      "dependencies": {
+        "parse5": "^7.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/inikulin/parse5?sponsor=1"
+      }
+    },
+    "node_modules/parse5/node_modules/entities": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
+      "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
+      }
+    },
     "node_modules/parseurl": {
       "version": "1.3.3",
       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -9841,6 +10070,15 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/undici": {
+      "version": "7.28.0",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz",
+      "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=20.18.1"
+      }
+    },
     "node_modules/undici-types": {
       "version": "7.24.6",
       "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz",
@@ -10140,6 +10378,40 @@
         "vue": "^3.5.0"
       }
     },
+    "node_modules/whatwg-encoding": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
+      "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
+      "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation",
+      "license": "MIT",
+      "dependencies": {
+        "iconv-lite": "0.6.3"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/whatwg-encoding/node_modules/iconv-lite": {
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+      "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+      "license": "MIT",
+      "dependencies": {
+        "safer-buffer": ">= 2.1.2 < 3.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/whatwg-mimetype": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
+      "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

package.json

@@ -81,6 +81,7 @@
     "@vue/server-renderer": "^3.5.35",
     "@vuepic/vue-datepicker": "^13.0.0",
     "better-sqlite3": "^12.10.0",
+    "cheerio": "^1.2.0",
     "edge.js": "^6.5.1",
     "luxon": "^3.7.2",
     "mysql2": "^3.22.4",

start/routes.ts

@@ -66,3 +66,8 @@ router
   })
   .prefix('/components')
   .use(middleware.auth())
+
+router
+  .get('/scraped-parts', [controllers.ScrapedParts, 'index'])
+  .as('scraped_parts.index')
+  .use(middleware.auth())

tests/functional/scraped_parts.spec.ts

@@ -0,0 +1,70 @@
+import { test } from '@japa/runner'
+import { DateTime } from 'luxon'
+import User from '#models/user'
+import ComponentType from '#models/component_type'
+import ScrapedPart from '#models/scraped_part'
+
+async function createUser(email: string) {
+  return User.create({
+    name: 'Test User',
+    email,
+    password: 'secret@123A',
+  })
+}
+
+async function createType(name: string, unique = true) {
+  return ComponentType.firstOrCreate({ name }, { name, unique })
+}
+
+async function createScrapedPart(typeId: number, name: string, sourceRef: string) {
+  return ScrapedPart.create({
+    typeId,
+    name,
+    price: 199.99,
+    imageUrl: 'https://fr.pc-builder.io/images/cpu/example.jpg',
+    source: 'pc-builder.io',
+    sourceUrl: `https://fr.pc-builder.io/cpu/${sourceRef}`,
+    sourceRef,
+    scrapedAt: DateTime.now(),
+  })
+}
+
+test.group('Scraped parts search', () => {
+  test('an authenticated user can search scraped parts by name', async ({ client, assert }) => {
+    const user = await createUser('scraped-search-a@example.com')
+    const type = await createType('Processor-scraped-1')
+    await createScrapedPart(type.id, 'ZZZ-Test-Ryzen-Unique 5 7600X', 'amd-ryzen-5-7600x-p1311')
+    await createScrapedPart(type.id, 'ZZZ-Test-Intel-Unique i5-13600K', 'intel-i5-13600k-p9999')
+
+    const response = await client.get('/scraped-parts?q=ZZZ-Test-Ryzen-Unique').loginAs(user)
+
+    response.assertStatus(200)
+    const body = response.body()
+    assert.lengthOf(body, 1)
+    assert.equal(body[0].name, 'ZZZ-Test-Ryzen-Unique 5 7600X')
+    assert.equal(body[0].type.id, type.id)
+  })
+
+  test('search can be filtered by typeId', async ({ client, assert }) => {
+    const user = await createUser('scraped-search-b@example.com')
+    const cpuType = await createType('Processor-scraped-2')
+    const gpuType = await createType('Graphic Card-scraped-2')
+    await createScrapedPart(cpuType.id, 'ZZZ-Test-AMD Ryzen 7 7800X3D', 'amd-ryzen-7-7800x3d-p1')
+    await createScrapedPart(gpuType.id, 'ZZZ-Test-AMD Radeon RX 7800', 'amd-radeon-rx-7800-p2')
+
+    const response = await client
+      .get(`/scraped-parts?q=ZZZ-Test-AMD&typeId=${gpuType.id}`)
+      .loginAs(user)
+
+    response.assertStatus(200)
+    const body = response.body()
+    assert.lengthOf(body, 1)
+    assert.equal(body[0].name, 'ZZZ-Test-AMD Radeon RX 7800')
+  })
+
+  test('an unauthenticated request is redirected to login', async ({ client }) => {
+    const response = await client.get('/scraped-parts?q=ryzen')
+
+    response.assertRedirectsTo('/auth/login')
+  })
+})