feat: add 24h cache freshness with auto-refresh on search
Build & Deploy / build (push) Failing after 5s
Build & Deploy / docker (push) Skipped

- part_scraper_service.ts: new CACHE_TTL_HOURS (24), isCategoryStale()
  checks the most recent scrapedAt for a given ComponentType's mapped
  category, ensureFreshCategory() re-scrapes it synchronously if missing
  or stale (errors are logged, not thrown, so search still falls back to
  whatever is already cached)
- ScrapedPartsController#index: when a typeId filter is provided, the
  corresponding category cache is refreshed eagerly if stale before
  querying results. Free-text searches without a typeId skip this to
  avoid re-scraping every mapped category on every keystroke -
  'node ace scrape:parts' still refreshes everything at once on demand
- 4 new tests covering staleness detection (missing cache, <24h, >24h)
  and that unmapped types never trigger a scrape/network call
This commit is contained in:
Kevin L'Hereec committed 2026-07-23 13:58:37 +02:00
1 parent 5854ff8ed1
commit 497ae797ca
3 files changed
+120 -1

No files matched your search

+55
View File
@@ -3,6 +3,11 @@ import { DateTime } from 'luxon'
import User from '#models/user'
import ComponentType from '#models/component_type'
import ScrapedPart from '#models/scraped_part'
import {
isCategoryStale,
ensureFreshCategory,
CACHE_TTL_HOURS,
} from '#services/part_scraper_service'
async function createUser(email: string) {
return User.create({
@@ -68,3 +73,53 @@ test.group('Scraped parts search', () => {
response.assertRedirectsTo('/auth/login')
})
})
test.group('Scraped parts 24h cache freshness', () => {
test('a category with no cached parts is considered stale', async ({ assert }) => {
const type = await createType('Processor-freshness-1')
assert.isTrue(await isCategoryStale(type))
})
test('a category scraped within the last 24h is not stale', async ({ assert }) => {
const type = await createType('Processor-freshness-2')
await ScrapedPart.create({
typeId: type.id,
name: 'ZZZ-Test-Fresh Part',
price: 100,
imageUrl: null,
source: 'pc-builder.io',
sourceUrl: 'https://fr.pc-builder.io/cpu/fresh-part',
sourceRef: 'fresh-part-ref',
scrapedAt: DateTime.now().minus({ hours: 1 }),
})
assert.isFalse(await isCategoryStale(type))
})
test('a category scraped more than 24h ago is stale', async ({ assert }) => {
const type = await createType('Processor-freshness-3')
await ScrapedPart.create({
typeId: type.id,
name: 'ZZZ-Test-Stale Part',
price: 100,
imageUrl: null,
source: 'pc-builder.io',
sourceUrl: 'https://fr.pc-builder.io/cpu/stale-part',
sourceRef: 'stale-part-ref',
scrapedAt: DateTime.now().minus({ hours: CACHE_TTL_HOURS + 1 }),
})
assert.isTrue(await isCategoryStale(type))
})
test('ensureFreshCategory does not scrape unmapped types (no network call)', async ({
assert,
}) => {
// "Other" is a real, unmapped ComponentType (see CATEGORY_SLUGS): this
// must resolve without making any outbound HTTP request.
const type = await createType('Other-freshness-unmapped', false)
await assert.doesNotReject(() => ensureFreshCategory(type))
})
})