From 497ae797ca318f25b84b281bd72fd0b226ea7221 Mon Sep 17 00:00:00 2001 From: Kevin L'Hereec Date: Thu, 23 Jul 2026 13:58:37 +0200 Subject: [PATCH] feat: add 24h cache freshness with auto-refresh on search - part_scraper_service.ts: new CACHE_TTL_HOURS (24), isCategoryStale() checks the most recent scrapedAt for a given ComponentType's mapped category, ensureFreshCategory() re-scrapes it synchronously if missing or stale (errors are logged, not thrown, so search still falls back to whatever is already cached) - ScrapedPartsController#index: when a typeId filter is provided, the corresponding category cache is refreshed eagerly if stale before querying results. Free-text searches without a typeId skip this to avoid re-scraping every mapped category on every keystroke - 'node ace scrape:parts' still refreshes everything at once on demand - 4 new tests covering staleness detection (missing cache, <24h, >24h) and that unmapped types never trigger a scrape/network call --- app/controllers/scraped_parts_controller.ts | 17 +++++++ app/services/part_scraper_service.ts | 49 +++++++++++++++++- tests/functional/scraped_parts.spec.ts | 55 +++++++++++++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) diff --git a/app/controllers/scraped_parts_controller.ts b/app/controllers/scraped_parts_controller.ts index fe6059d..e5f1b17 100644 --- a/app/controllers/scraped_parts_controller.ts +++ b/app/controllers/scraped_parts_controller.ts @@ -1,6 +1,8 @@ import type { HttpContext } from '@adonisjs/core/http' import vine from '@vinejs/vine' import ScrapedPart from '#models/scraped_part' +import ComponentType from '#models/component_type' +import { ensureFreshCategory } from '#services/part_scraper_service' const searchValidator = vine.compile( vine.object({ @@ -14,11 +16,26 @@ export default class ScrapedPartsController { * Search the new-parts cache (scraped from pc-builder.io) by name and/or * component type. Read-only, authenticated (the search UI lives inside * the authenticated "create component" flow). + * + * When a typeId is given, the corresponding category cache is refreshed + * synchronously first if it's missing or older than 24h (see + * part_scraper_service.CACHE_TTL_HOURS), so results stay reasonably + * fresh without needing an external cron job. Free-text searches across + * all types skip this eager refresh to avoid re-scraping every mapped + * category on every keystroke; run `node ace scrape:parts` to refresh + * everything at once. */ async index({ request, auth, response }: HttpContext) { await auth.authenticate() const { q, typeId } = await request.validateUsing(searchValidator) + if (typeId) { + const componentType = await ComponentType.find(typeId) + if (componentType) { + await ensureFreshCategory(componentType) + } + } + const query = ScrapedPart.query().preload('type').orderBy('name') if (q) { diff --git a/app/services/part_scraper_service.ts b/app/services/part_scraper_service.ts index aaa7c48..9f1ace0 100644 --- a/app/services/part_scraper_service.ts +++ b/app/services/part_scraper_service.ts @@ -7,6 +7,13 @@ import ScrapedPart from '#models/scraped_part' const SOURCE = 'pc-builder.io' const BASE_URL = 'https://fr.pc-builder.io' +/** + * Default freshness window for the scraped_parts cache. A category is + * re-scraped automatically the first time it is searched after this delay + * has elapsed since its last successful scrape. + */ +const CACHE_TTL_HOURS = 24 + /** * Maps our own ComponentType names to the corresponding pc-builder.io * category slug. Categories not listed here are not scraped. @@ -139,4 +146,44 @@ async function scrapeAllCategories(): Promise> { return summary } -export { scrapeAllCategories, scrapeComponentType, CATEGORY_SLUGS } +/** + * Returns true if the given ComponentType's category has never been + * scraped, or was last scraped more than CACHE_TTL_HOURS ago. + */ +async function isCategoryStale(componentType: ComponentType): Promise { + const mostRecent = await ScrapedPart.query() + .where('source', SOURCE) + .andWhere('typeId', componentType.id) + .orderBy('scrapedAt', 'desc') + .first() + + if (!mostRecent) return true + return mostRecent.scrapedAt < DateTime.now().minus({ hours: CACHE_TTL_HOURS }) +} + +/** + * Ensures the cache for a given ComponentType's mapped category is fresh + * (scraped within the last CACHE_TTL_HOURS), re-scraping it synchronously + * if it's missing or stale. No-op for unmapped types. Scraping errors are + * logged but not thrown, so a search still falls back to whatever is + * already cached. + */ +async function ensureFreshCategory(componentType: ComponentType): Promise { + if (!CATEGORY_SLUGS[componentType.name]) return + if (!(await isCategoryStale(componentType))) return + + try { + await scrapeComponentType(componentType) + } catch (error) { + logger.error({ type: componentType.name, err: error }, 'Failed to refresh stale category cache') + } +} + +export { + scrapeAllCategories, + scrapeComponentType, + ensureFreshCategory, + isCategoryStale, + CATEGORY_SLUGS, + CACHE_TTL_HOURS, +} diff --git a/tests/functional/scraped_parts.spec.ts b/tests/functional/scraped_parts.spec.ts index e86e94a..415e7b2 100644 --- a/tests/functional/scraped_parts.spec.ts +++ b/tests/functional/scraped_parts.spec.ts @@ -3,6 +3,11 @@ import { DateTime } from 'luxon' import User from '#models/user' import ComponentType from '#models/component_type' import ScrapedPart from '#models/scraped_part' +import { + isCategoryStale, + ensureFreshCategory, + CACHE_TTL_HOURS, +} from '#services/part_scraper_service' async function createUser(email: string) { return User.create({ @@ -68,3 +73,53 @@ test.group('Scraped parts search', () => { response.assertRedirectsTo('/auth/login') }) }) + +test.group('Scraped parts 24h cache freshness', () => { + test('a category with no cached parts is considered stale', async ({ assert }) => { + const type = await createType('Processor-freshness-1') + + assert.isTrue(await isCategoryStale(type)) + }) + + test('a category scraped within the last 24h is not stale', async ({ assert }) => { + const type = await createType('Processor-freshness-2') + await ScrapedPart.create({ + typeId: type.id, + name: 'ZZZ-Test-Fresh Part', + price: 100, + imageUrl: null, + source: 'pc-builder.io', + sourceUrl: 'https://fr.pc-builder.io/cpu/fresh-part', + sourceRef: 'fresh-part-ref', + scrapedAt: DateTime.now().minus({ hours: 1 }), + }) + + assert.isFalse(await isCategoryStale(type)) + }) + + test('a category scraped more than 24h ago is stale', async ({ assert }) => { + const type = await createType('Processor-freshness-3') + await ScrapedPart.create({ + typeId: type.id, + name: 'ZZZ-Test-Stale Part', + price: 100, + imageUrl: null, + source: 'pc-builder.io', + sourceUrl: 'https://fr.pc-builder.io/cpu/stale-part', + sourceRef: 'stale-part-ref', + scrapedAt: DateTime.now().minus({ hours: CACHE_TTL_HOURS + 1 }), + }) + + assert.isTrue(await isCategoryStale(type)) + }) + + test('ensureFreshCategory does not scrape unmapped types (no network call)', async ({ + assert, + }) => { + // "Other" is a real, unmapped ComponentType (see CATEGORY_SLUGS): this + // must resolve without making any outbound HTTP request. + const type = await createType('Other-freshness-unmapped', false) + + await assert.doesNotReject(() => ensureFreshCategory(type)) + }) +})