feat: add 24h cache freshness with auto-refresh on search
- part_scraper_service.ts: new CACHE_TTL_HOURS (24), isCategoryStale() checks the most recent scrapedAt for a given ComponentType's mapped category, ensureFreshCategory() re-scrapes it synchronously if missing or stale (errors are logged, not thrown, so search still falls back to whatever is already cached) - ScrapedPartsController#index: when a typeId filter is provided, the corresponding category cache is refreshed eagerly if stale before querying results. Free-text searches without a typeId skip this to avoid re-scraping every mapped category on every keystroke - 'node ace scrape:parts' still refreshes everything at once on demand - 4 new tests covering staleness detection (missing cache, <24h, >24h) and that unmapped types never trigger a scrape/network call
This commit is contained in:
1 parent
5854ff8ed1
commit
497ae797ca
3 files changed
+120
-1
No files matched your search
@@ -7,6 +7,13 @@ import ScrapedPart from '#models/scraped_part'
|
||||
const SOURCE = 'pc-builder.io'
|
||||
const BASE_URL = 'https://fr.pc-builder.io'
|
||||
|
||||
/**
|
||||
* Default freshness window for the scraped_parts cache. A category is
|
||||
* re-scraped automatically the first time it is searched after this delay
|
||||
* has elapsed since its last successful scrape.
|
||||
*/
|
||||
const CACHE_TTL_HOURS = 24
|
||||
|
||||
/**
|
||||
* Maps our own ComponentType names to the corresponding pc-builder.io
|
||||
* category slug. Categories not listed here are not scraped.
|
||||
@@ -139,4 +146,44 @@ async function scrapeAllCategories(): Promise<Record<string, number>> {
|
||||
return summary
|
||||
}
|
||||
|
||||
export { scrapeAllCategories, scrapeComponentType, CATEGORY_SLUGS }
|
||||
/**
|
||||
* Returns true if the given ComponentType's category has never been
|
||||
* scraped, or was last scraped more than CACHE_TTL_HOURS ago.
|
||||
*/
|
||||
async function isCategoryStale(componentType: ComponentType): Promise<boolean> {
|
||||
const mostRecent = await ScrapedPart.query()
|
||||
.where('source', SOURCE)
|
||||
.andWhere('typeId', componentType.id)
|
||||
.orderBy('scrapedAt', 'desc')
|
||||
.first()
|
||||
|
||||
if (!mostRecent) return true
|
||||
return mostRecent.scrapedAt < DateTime.now().minus({ hours: CACHE_TTL_HOURS })
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures the cache for a given ComponentType's mapped category is fresh
|
||||
* (scraped within the last CACHE_TTL_HOURS), re-scraping it synchronously
|
||||
* if it's missing or stale. No-op for unmapped types. Scraping errors are
|
||||
* logged but not thrown, so a search still falls back to whatever is
|
||||
* already cached.
|
||||
*/
|
||||
async function ensureFreshCategory(componentType: ComponentType): Promise<void> {
|
||||
if (!CATEGORY_SLUGS[componentType.name]) return
|
||||
if (!(await isCategoryStale(componentType))) return
|
||||
|
||||
try {
|
||||
await scrapeComponentType(componentType)
|
||||
} catch (error) {
|
||||
logger.error({ type: componentType.name, err: error }, 'Failed to refresh stale category cache')
|
||||
}
|
||||
}
|
||||
|
||||
export {
|
||||
scrapeAllCategories,
|
||||
scrapeComponentType,
|
||||
ensureFreshCategory,
|
||||
isCategoryStale,
|
||||
CATEGORY_SLUGS,
|
||||
CACHE_TTL_HOURS,
|
||||
}
|
||||
Reference in new issue
Block a user