- part_scraper_service.ts: new CACHE_TTL_HOURS (24), isCategoryStale() checks the most recent scrapedAt for a given ComponentType's mapped category, ensureFreshCategory() re-scrapes it synchronously if missing or stale (errors are logged, not thrown, so search still falls back to whatever is already cached) - ScrapedPartsController#index: when a typeId filter is provided, the corresponding category cache is refreshed eagerly if stale before querying results. Free-text searches without a typeId skip this to avoid re-scraping every mapped category on every keystroke - 'node ace scrape:parts' still refreshes everything at once on demand - 4 new tests covering staleness detection (missing cache, <24h, >24h) and that unmapped types never trigger a scrape/network call
126 lines
4.2 KiB
TypeScript
126 lines
4.2 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import { DateTime } from 'luxon'
|
|
import User from '#models/user'
|
|
import ComponentType from '#models/component_type'
|
|
import ScrapedPart from '#models/scraped_part'
|
|
import {
|
|
isCategoryStale,
|
|
ensureFreshCategory,
|
|
CACHE_TTL_HOURS,
|
|
} from '#services/part_scraper_service'
|
|
|
|
async function createUser(email: string) {
|
|
return User.create({
|
|
name: 'Test User',
|
|
email,
|
|
password: 'secret@123A',
|
|
})
|
|
}
|
|
|
|
async function createType(name: string, unique = true) {
|
|
return ComponentType.firstOrCreate({ name }, { name, unique })
|
|
}
|
|
|
|
async function createScrapedPart(typeId: number, name: string, sourceRef: string) {
|
|
return ScrapedPart.create({
|
|
typeId,
|
|
name,
|
|
price: 199.99,
|
|
imageUrl: 'https://fr.pc-builder.io/images/cpu/example.jpg',
|
|
source: 'pc-builder.io',
|
|
sourceUrl: `https://fr.pc-builder.io/cpu/${sourceRef}`,
|
|
sourceRef,
|
|
scrapedAt: DateTime.now(),
|
|
})
|
|
}
|
|
|
|
test.group('Scraped parts search', () => {
|
|
test('an authenticated user can search scraped parts by name', async ({ client, assert }) => {
|
|
const user = await createUser('scraped-search-a@example.com')
|
|
const type = await createType('Processor-scraped-1')
|
|
await createScrapedPart(type.id, 'ZZZ-Test-Ryzen-Unique 5 7600X', 'amd-ryzen-5-7600x-p1311')
|
|
await createScrapedPart(type.id, 'ZZZ-Test-Intel-Unique i5-13600K', 'intel-i5-13600k-p9999')
|
|
|
|
const response = await client.get('/scraped-parts?q=ZZZ-Test-Ryzen-Unique').loginAs(user)
|
|
|
|
response.assertStatus(200)
|
|
const body = response.body()
|
|
assert.lengthOf(body, 1)
|
|
assert.equal(body[0].name, 'ZZZ-Test-Ryzen-Unique 5 7600X')
|
|
assert.equal(body[0].type.id, type.id)
|
|
})
|
|
|
|
test('search can be filtered by typeId', async ({ client, assert }) => {
|
|
const user = await createUser('scraped-search-b@example.com')
|
|
const cpuType = await createType('Processor-scraped-2')
|
|
const gpuType = await createType('Graphic Card-scraped-2')
|
|
await createScrapedPart(cpuType.id, 'ZZZ-Test-AMD Ryzen 7 7800X3D', 'amd-ryzen-7-7800x3d-p1')
|
|
await createScrapedPart(gpuType.id, 'ZZZ-Test-AMD Radeon RX 7800', 'amd-radeon-rx-7800-p2')
|
|
|
|
const response = await client
|
|
.get(`/scraped-parts?q=ZZZ-Test-AMD&typeId=${gpuType.id}`)
|
|
.loginAs(user)
|
|
|
|
response.assertStatus(200)
|
|
const body = response.body()
|
|
assert.lengthOf(body, 1)
|
|
assert.equal(body[0].name, 'ZZZ-Test-AMD Radeon RX 7800')
|
|
})
|
|
|
|
test('an unauthenticated request is redirected to login', async ({ client }) => {
|
|
const response = await client.get('/scraped-parts?q=ryzen')
|
|
|
|
response.assertRedirectsTo('/auth/login')
|
|
})
|
|
})
|
|
|
|
test.group('Scraped parts 24h cache freshness', () => {
|
|
test('a category with no cached parts is considered stale', async ({ assert }) => {
|
|
const type = await createType('Processor-freshness-1')
|
|
|
|
assert.isTrue(await isCategoryStale(type))
|
|
})
|
|
|
|
test('a category scraped within the last 24h is not stale', async ({ assert }) => {
|
|
const type = await createType('Processor-freshness-2')
|
|
await ScrapedPart.create({
|
|
typeId: type.id,
|
|
name: 'ZZZ-Test-Fresh Part',
|
|
price: 100,
|
|
imageUrl: null,
|
|
source: 'pc-builder.io',
|
|
sourceUrl: 'https://fr.pc-builder.io/cpu/fresh-part',
|
|
sourceRef: 'fresh-part-ref',
|
|
scrapedAt: DateTime.now().minus({ hours: 1 }),
|
|
})
|
|
|
|
assert.isFalse(await isCategoryStale(type))
|
|
})
|
|
|
|
test('a category scraped more than 24h ago is stale', async ({ assert }) => {
|
|
const type = await createType('Processor-freshness-3')
|
|
await ScrapedPart.create({
|
|
typeId: type.id,
|
|
name: 'ZZZ-Test-Stale Part',
|
|
price: 100,
|
|
imageUrl: null,
|
|
source: 'pc-builder.io',
|
|
sourceUrl: 'https://fr.pc-builder.io/cpu/stale-part',
|
|
sourceRef: 'stale-part-ref',
|
|
scrapedAt: DateTime.now().minus({ hours: CACHE_TTL_HOURS + 1 }),
|
|
})
|
|
|
|
assert.isTrue(await isCategoryStale(type))
|
|
})
|
|
|
|
test('ensureFreshCategory does not scrape unmapped types (no network call)', async ({
|
|
assert,
|
|
}) => {
|
|
// "Other" is a real, unmapped ComponentType (see CATEGORY_SLUGS): this
|
|
// must resolve without making any outbound HTTP request.
|
|
const type = await createType('Other-freshness-unmapped', false)
|
|
|
|
await assert.doesNotReject(() => ensureFreshCategory(type))
|
|
})
|
|
})
|