feat: search and import new parts scraped from pc-builder.io
Idealo and PCPartPicker were evaluated and ruled out: - idealo.fr search pages return 503 (Cloudflare) - the internal Idealo GraphQL endpoint used by a public scraper (app.idealo.de/app-backend/api) is protected by Akamai Bot Manager and silently returns empty data even with mobile app headers - PCPartPicker's product list/search relies on a session-bound CSRF token refreshed per page load, against a scraping-hostile ToS pc-builder.io category pages are server-rendered, include name/price/image for ~100 products per category, and have no bot protection, so the feature is built on top of that source only. - New scraped_parts cache table (type_id, name, price, image_url, source, source_url, source_ref unique per source) populated by a batch job rather than scraped live per search, since only ~100 first results per category are available without reverse-engineering pc-builder.io's DataTables/CSRF pagination flow - part_scraper_service.ts: maps our ComponentType names to pc-builder.io category slugs (SSD -> storage, since pc-builder.io has no separate HDD category) and parses each category page with cheerio, skipping rows without a resolved price (generic 'search on Amazon' links) - node ace scrape:parts: refreshes the cache for every mapped category (upsert on source+source_ref), runnable manually or later via cron - GET /scraped-parts (authenticated): search the cache by name (LOWER() LIKE, avoids a MySQL utf8_bin/utf8mb4 collation mismatch from Lucid's whereLike) and optional typeId filter - NewPartSearch.vue: debounced search widget wired into components/create.vue; clicking 'Import' pre-fills the existing component creation form (name, price, type, purchase date = today) instead of creating the Component directly, so the user still reviews and saves it like any manually entered part - 3 functional tests covering search by name, filter by typeId, and unauthenticated access being redirected to login - Initial scrape run: 989 parts across 10 categories
This commit is contained in:
1 parent
da71d33450
commit
5854ff8ed1
19 files changed
+757
-9
No files matched your search
@@ -0,0 +1,70 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { DateTime } from 'luxon'
|
||||
import User from '#models/user'
|
||||
import ComponentType from '#models/component_type'
|
||||
import ScrapedPart from '#models/scraped_part'
|
||||
|
||||
async function createUser(email: string) {
|
||||
return User.create({
|
||||
name: 'Test User',
|
||||
email,
|
||||
password: 'secret@123A',
|
||||
})
|
||||
}
|
||||
|
||||
async function createType(name: string, unique = true) {
|
||||
return ComponentType.firstOrCreate({ name }, { name, unique })
|
||||
}
|
||||
|
||||
async function createScrapedPart(typeId: number, name: string, sourceRef: string) {
|
||||
return ScrapedPart.create({
|
||||
typeId,
|
||||
name,
|
||||
price: 199.99,
|
||||
imageUrl: 'https://fr.pc-builder.io/images/cpu/example.jpg',
|
||||
source: 'pc-builder.io',
|
||||
sourceUrl: `https://fr.pc-builder.io/cpu/${sourceRef}`,
|
||||
sourceRef,
|
||||
scrapedAt: DateTime.now(),
|
||||
})
|
||||
}
|
||||
|
||||
test.group('Scraped parts search', () => {
|
||||
test('an authenticated user can search scraped parts by name', async ({ client, assert }) => {
|
||||
const user = await createUser('scraped-search-a@example.com')
|
||||
const type = await createType('Processor-scraped-1')
|
||||
await createScrapedPart(type.id, 'ZZZ-Test-Ryzen-Unique 5 7600X', 'amd-ryzen-5-7600x-p1311')
|
||||
await createScrapedPart(type.id, 'ZZZ-Test-Intel-Unique i5-13600K', 'intel-i5-13600k-p9999')
|
||||
|
||||
const response = await client.get('/scraped-parts?q=ZZZ-Test-Ryzen-Unique').loginAs(user)
|
||||
|
||||
response.assertStatus(200)
|
||||
const body = response.body()
|
||||
assert.lengthOf(body, 1)
|
||||
assert.equal(body[0].name, 'ZZZ-Test-Ryzen-Unique 5 7600X')
|
||||
assert.equal(body[0].type.id, type.id)
|
||||
})
|
||||
|
||||
test('search can be filtered by typeId', async ({ client, assert }) => {
|
||||
const user = await createUser('scraped-search-b@example.com')
|
||||
const cpuType = await createType('Processor-scraped-2')
|
||||
const gpuType = await createType('Graphic Card-scraped-2')
|
||||
await createScrapedPart(cpuType.id, 'ZZZ-Test-AMD Ryzen 7 7800X3D', 'amd-ryzen-7-7800x3d-p1')
|
||||
await createScrapedPart(gpuType.id, 'ZZZ-Test-AMD Radeon RX 7800', 'amd-radeon-rx-7800-p2')
|
||||
|
||||
const response = await client
|
||||
.get(`/scraped-parts?q=ZZZ-Test-AMD&typeId=${gpuType.id}`)
|
||||
.loginAs(user)
|
||||
|
||||
response.assertStatus(200)
|
||||
const body = response.body()
|
||||
assert.lengthOf(body, 1)
|
||||
assert.equal(body[0].name, 'ZZZ-Test-AMD Radeon RX 7800')
|
||||
})
|
||||
|
||||
test('an unauthenticated request is redirected to login', async ({ client }) => {
|
||||
const response = await client.get('/scraped-parts?q=ryzen')
|
||||
|
||||
response.assertRedirectsTo('/auth/login')
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user