feat: add 24h cache freshness with auto-refresh on search
- part_scraper_service.ts: new CACHE_TTL_HOURS (24), isCategoryStale() checks the most recent scrapedAt for a given ComponentType's mapped category, ensureFreshCategory() re-scrapes it synchronously if missing or stale (errors are logged, not thrown, so search still falls back to whatever is already cached) - ScrapedPartsController#index: when a typeId filter is provided, the corresponding category cache is refreshed eagerly if stale before querying results. Free-text searches without a typeId skip this to avoid re-scraping every mapped category on every keystroke - 'node ace scrape:parts' still refreshes everything at once on demand - 4 new tests covering staleness detection (missing cache, <24h, >24h) and that unmapped types never trigger a scrape/network call
This commit is contained in:
1 parent
5854ff8ed1
commit
497ae797ca
3 files changed
+120
-1
No files matched your search
@@ -1,6 +1,8 @@
|
|||||||
import type { HttpContext } from '@adonisjs/core/http'
|
import type { HttpContext } from '@adonisjs/core/http'
|
||||||
import vine from '@vinejs/vine'
|
import vine from '@vinejs/vine'
|
||||||
import ScrapedPart from '#models/scraped_part'
|
import ScrapedPart from '#models/scraped_part'
|
||||||
|
import ComponentType from '#models/component_type'
|
||||||
|
import { ensureFreshCategory } from '#services/part_scraper_service'
|
||||||
|
|
||||||
const searchValidator = vine.compile(
|
const searchValidator = vine.compile(
|
||||||
vine.object({
|
vine.object({
|
||||||
@@ -14,11 +16,26 @@ export default class ScrapedPartsController {
|
|||||||
* Search the new-parts cache (scraped from pc-builder.io) by name and/or
|
* Search the new-parts cache (scraped from pc-builder.io) by name and/or
|
||||||
* component type. Read-only, authenticated (the search UI lives inside
|
* component type. Read-only, authenticated (the search UI lives inside
|
||||||
* the authenticated "create component" flow).
|
* the authenticated "create component" flow).
|
||||||
|
*
|
||||||
|
* When a typeId is given, the corresponding category cache is refreshed
|
||||||
|
* synchronously first if it's missing or older than 24h (see
|
||||||
|
* part_scraper_service.CACHE_TTL_HOURS), so results stay reasonably
|
||||||
|
* fresh without needing an external cron job. Free-text searches across
|
||||||
|
* all types skip this eager refresh to avoid re-scraping every mapped
|
||||||
|
* category on every keystroke; run `node ace scrape:parts` to refresh
|
||||||
|
* everything at once.
|
||||||
*/
|
*/
|
||||||
async index({ request, auth, response }: HttpContext) {
|
async index({ request, auth, response }: HttpContext) {
|
||||||
await auth.authenticate()
|
await auth.authenticate()
|
||||||
const { q, typeId } = await request.validateUsing(searchValidator)
|
const { q, typeId } = await request.validateUsing(searchValidator)
|
||||||
|
|
||||||
|
if (typeId) {
|
||||||
|
const componentType = await ComponentType.find(typeId)
|
||||||
|
if (componentType) {
|
||||||
|
await ensureFreshCategory(componentType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const query = ScrapedPart.query().preload('type').orderBy('name')
|
const query = ScrapedPart.query().preload('type').orderBy('name')
|
||||||
|
|
||||||
if (q) {
|
if (q) {
|
||||||
|
|||||||
@@ -7,6 +7,13 @@ import ScrapedPart from '#models/scraped_part'
|
|||||||
const SOURCE = 'pc-builder.io'
|
const SOURCE = 'pc-builder.io'
|
||||||
const BASE_URL = 'https://fr.pc-builder.io'
|
const BASE_URL = 'https://fr.pc-builder.io'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Default freshness window for the scraped_parts cache. A category is
|
||||||
|
* re-scraped automatically the first time it is searched after this delay
|
||||||
|
* has elapsed since its last successful scrape.
|
||||||
|
*/
|
||||||
|
const CACHE_TTL_HOURS = 24
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Maps our own ComponentType names to the corresponding pc-builder.io
|
* Maps our own ComponentType names to the corresponding pc-builder.io
|
||||||
* category slug. Categories not listed here are not scraped.
|
* category slug. Categories not listed here are not scraped.
|
||||||
@@ -139,4 +146,44 @@ async function scrapeAllCategories(): Promise<Record<string, number>> {
|
|||||||
return summary
|
return summary
|
||||||
}
|
}
|
||||||
|
|
||||||
export { scrapeAllCategories, scrapeComponentType, CATEGORY_SLUGS }
|
/**
|
||||||
|
* Returns true if the given ComponentType's category has never been
|
||||||
|
* scraped, or was last scraped more than CACHE_TTL_HOURS ago.
|
||||||
|
*/
|
||||||
|
async function isCategoryStale(componentType: ComponentType): Promise<boolean> {
|
||||||
|
const mostRecent = await ScrapedPart.query()
|
||||||
|
.where('source', SOURCE)
|
||||||
|
.andWhere('typeId', componentType.id)
|
||||||
|
.orderBy('scrapedAt', 'desc')
|
||||||
|
.first()
|
||||||
|
|
||||||
|
if (!mostRecent) return true
|
||||||
|
return mostRecent.scrapedAt < DateTime.now().minus({ hours: CACHE_TTL_HOURS })
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensures the cache for a given ComponentType's mapped category is fresh
|
||||||
|
* (scraped within the last CACHE_TTL_HOURS), re-scraping it synchronously
|
||||||
|
* if it's missing or stale. No-op for unmapped types. Scraping errors are
|
||||||
|
* logged but not thrown, so a search still falls back to whatever is
|
||||||
|
* already cached.
|
||||||
|
*/
|
||||||
|
async function ensureFreshCategory(componentType: ComponentType): Promise<void> {
|
||||||
|
if (!CATEGORY_SLUGS[componentType.name]) return
|
||||||
|
if (!(await isCategoryStale(componentType))) return
|
||||||
|
|
||||||
|
try {
|
||||||
|
await scrapeComponentType(componentType)
|
||||||
|
} catch (error) {
|
||||||
|
logger.error({ type: componentType.name, err: error }, 'Failed to refresh stale category cache')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export {
|
||||||
|
scrapeAllCategories,
|
||||||
|
scrapeComponentType,
|
||||||
|
ensureFreshCategory,
|
||||||
|
isCategoryStale,
|
||||||
|
CATEGORY_SLUGS,
|
||||||
|
CACHE_TTL_HOURS,
|
||||||
|
}
|
||||||
@@ -3,6 +3,11 @@ import { DateTime } from 'luxon'
|
|||||||
import User from '#models/user'
|
import User from '#models/user'
|
||||||
import ComponentType from '#models/component_type'
|
import ComponentType from '#models/component_type'
|
||||||
import ScrapedPart from '#models/scraped_part'
|
import ScrapedPart from '#models/scraped_part'
|
||||||
|
import {
|
||||||
|
isCategoryStale,
|
||||||
|
ensureFreshCategory,
|
||||||
|
CACHE_TTL_HOURS,
|
||||||
|
} from '#services/part_scraper_service'
|
||||||
|
|
||||||
async function createUser(email: string) {
|
async function createUser(email: string) {
|
||||||
return User.create({
|
return User.create({
|
||||||
@@ -68,3 +73,53 @@ test.group('Scraped parts search', () => {
|
|||||||
response.assertRedirectsTo('/auth/login')
|
response.assertRedirectsTo('/auth/login')
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test.group('Scraped parts 24h cache freshness', () => {
|
||||||
|
test('a category with no cached parts is considered stale', async ({ assert }) => {
|
||||||
|
const type = await createType('Processor-freshness-1')
|
||||||
|
|
||||||
|
assert.isTrue(await isCategoryStale(type))
|
||||||
|
})
|
||||||
|
|
||||||
|
test('a category scraped within the last 24h is not stale', async ({ assert }) => {
|
||||||
|
const type = await createType('Processor-freshness-2')
|
||||||
|
await ScrapedPart.create({
|
||||||
|
typeId: type.id,
|
||||||
|
name: 'ZZZ-Test-Fresh Part',
|
||||||
|
price: 100,
|
||||||
|
imageUrl: null,
|
||||||
|
source: 'pc-builder.io',
|
||||||
|
sourceUrl: 'https://fr.pc-builder.io/cpu/fresh-part',
|
||||||
|
sourceRef: 'fresh-part-ref',
|
||||||
|
scrapedAt: DateTime.now().minus({ hours: 1 }),
|
||||||
|
})
|
||||||
|
|
||||||
|
assert.isFalse(await isCategoryStale(type))
|
||||||
|
})
|
||||||
|
|
||||||
|
test('a category scraped more than 24h ago is stale', async ({ assert }) => {
|
||||||
|
const type = await createType('Processor-freshness-3')
|
||||||
|
await ScrapedPart.create({
|
||||||
|
typeId: type.id,
|
||||||
|
name: 'ZZZ-Test-Stale Part',
|
||||||
|
price: 100,
|
||||||
|
imageUrl: null,
|
||||||
|
source: 'pc-builder.io',
|
||||||
|
sourceUrl: 'https://fr.pc-builder.io/cpu/stale-part',
|
||||||
|
sourceRef: 'stale-part-ref',
|
||||||
|
scrapedAt: DateTime.now().minus({ hours: CACHE_TTL_HOURS + 1 }),
|
||||||
|
})
|
||||||
|
|
||||||
|
assert.isTrue(await isCategoryStale(type))
|
||||||
|
})
|
||||||
|
|
||||||
|
test('ensureFreshCategory does not scrape unmapped types (no network call)', async ({
|
||||||
|
assert,
|
||||||
|
}) => {
|
||||||
|
// "Other" is a real, unmapped ComponentType (see CATEGORY_SLUGS): this
|
||||||
|
// must resolve without making any outbound HTTP request.
|
||||||
|
const type = await createType('Other-freshness-unmapped', false)
|
||||||
|
|
||||||
|
await assert.doesNotReject(() => ensureFreshCategory(type))
|
||||||
|
})
|
||||||
|
})
|
||||||
Reference in new issue
Block a user