The RTX 5070 case that prompted this: only 100 GPUs out of pc-builder.io's
actual 6222 were being scraped (its category pages only render the first
page server-side), so cheaper RTX 5070 variants existed on the site but
were invisible to our search. Investigating further also surfaced that
even with full pagination, most recent GPU listings (RTX 5070 included)
have no fixed Amazon price on pc-builder.io at all - just a generic
"search on Amazon" link - so pagination alone doesn't fully solve price
coverage either.
Pagination (part_scraper_service.ts):
- Reverse-engineered pc-builder.io's /products/data endpoint, which powers
the client-side DataTables widget behind every category page. Requires
a session cookie + CSRF token scraped from a fresh category page load,
and an exact field shape (columns[].data must be the literal strings
"Name"/"Add to Build", not numeric indices - a naive DataTables-shaped
request 500s otherwise).
- CATEGORY_CONFIG now tracks each category's real pc-builder.io
category_id and a per-category maxProducts depth, chosen by actual
catalog size and price-sensitivity: GPU near-full coverage (6000),
smaller/less price-volatile categories capped lower (300-800) to keep a
full `scrape:parts` run (all categories) in a reasonable time.
- ensureFreshCategory (triggered inline by a user's search when the cache
is stale) uses a much shallower QUICK_REFRESH_MAX_PRODUCTS=200 cap
instead of the category's full depth, since it runs inside the
request/response cycle of a search - full-depth coverage is only done
by the standalone `scrape:parts` command.
Always keep the best known price:
- scraped_parts.price is now nullable (migration): parts with no fixed
pc-builder.io offer are kept and searchable instead of being silently
skipped, so at least the model is discoverable even before a price is
resolved.
- New price_source / price_checked_at columns track where the currently
stored price came from ("pc-builder.io" or a merchant name from
merchant_price_service.ts).
- ScrapedPartsController#compare now persists the cheapest offer found
(via the new updatePriceIfCheaper()) when it beats the currently stored
price - so comparing prices benefits every future search/import for
that part, not just the user who triggered it. Never regresses the
price: a comparison finding a higher price than what's stored is
ignored.
- scrapeComponentType's catalog refresh applies the same "keep whichever
price is lower" rule per row, so a routine pc-builder.io re-scrape can
never overwrite a better price a comparison had already found and
persisted with a worse (or missing) one.
- Search results (GET /scraped-parts) are now ordered priced-cheapest-first,
then parts with no known price last (`ORDER BY price IS NULL, price ASC`),
so unpriced parts stay visible without pushing priced results down.
- NewPartSearch.vue displays the catalog price inline (or "No price yet"),
and the existing "Compare prices" flow now works for those unpriced
parts the same way it does for priced ones.
7 new/updated functional tests: nullable price is searchable and sorted
last, and 3 tests for updatePriceIfCheaper (cheaper offer replaces price,
pricier offer is ignored, first-ever price is accepted for a previously
unpriced part).
152 lines
4.4 KiB
TypeScript
152 lines
4.4 KiB
TypeScript
/**
|
|
* This file is automatically generated
|
|
* DO NOT EDIT manually
|
|
* Run "node ace migration:run" command to re-generate this file
|
|
*/
|
|
|
|
import { BaseModel, column } from '@adonisjs/lucid/orm'
|
|
import { DateTime } from 'luxon'
|
|
|
|
export class ComponentTypeSchema extends BaseModel {
|
|
static $columns = ['id', 'name', 'unique'] as const
|
|
$columns = ComponentTypeSchema.$columns
|
|
@column({ isPrimary: true })
|
|
declare id: number
|
|
@column()
|
|
declare name: string
|
|
@column()
|
|
declare unique: boolean
|
|
}
|
|
|
|
export class ComponentSchema extends BaseModel {
|
|
static $columns = ['computerId', 'createdAt', 'id', 'name', 'price', 'purchaseDate', 'purchaseUrl', 'typeId', 'updatedAt', 'userId'] as const
|
|
$columns = ComponentSchema.$columns
|
|
@column()
|
|
declare computerId: number | null
|
|
@column.dateTime({ autoCreate: true })
|
|
declare createdAt: DateTime | null
|
|
@column({ isPrimary: true })
|
|
declare id: number
|
|
@column()
|
|
declare name: string
|
|
@column()
|
|
declare price: string
|
|
@column.date()
|
|
declare purchaseDate: DateTime
|
|
@column()
|
|
declare purchaseUrl: string | null
|
|
@column()
|
|
declare typeId: number | null
|
|
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
|
declare updatedAt: DateTime | null
|
|
@column()
|
|
declare userId: number | null
|
|
}
|
|
|
|
export class ComputerPictureSchema extends BaseModel {
|
|
static $columns = ['computerId', 'createdAt', 'id', 'link', 'order', 'updatedAt'] as const
|
|
$columns = ComputerPictureSchema.$columns
|
|
@column()
|
|
declare computerId: number | null
|
|
@column.dateTime({ autoCreate: true })
|
|
declare createdAt: DateTime | null
|
|
@column({ isPrimary: true })
|
|
declare id: number
|
|
@column()
|
|
declare link: string
|
|
@column()
|
|
declare order: number
|
|
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
|
declare updatedAt: DateTime | null
|
|
}
|
|
|
|
export class ComputerStateSchema extends BaseModel {
|
|
static $columns = ['id', 'name'] as const
|
|
$columns = ComputerStateSchema.$columns
|
|
@column({ isPrimary: true })
|
|
declare id: number
|
|
@column()
|
|
declare name: string
|
|
}
|
|
|
|
export class ComputerSchema extends BaseModel {
|
|
static $columns = ['cpuScore', 'createdAt', 'globalScore', 'gpuScore', 'id', 'sellPrice', 'shareEnabled', 'shareShowCost', 'shareShowSellPrice', 'shareToken', 'stateId', 'title', 'updatedAt', 'userId'] as const
|
|
$columns = ComputerSchema.$columns
|
|
@column()
|
|
declare cpuScore: number
|
|
@column.dateTime({ autoCreate: true })
|
|
declare createdAt: DateTime | null
|
|
@column()
|
|
declare globalScore: number
|
|
@column()
|
|
declare gpuScore: number
|
|
@column({ isPrimary: true })
|
|
declare id: number
|
|
@column()
|
|
declare sellPrice: number | null
|
|
@column()
|
|
declare shareEnabled: boolean
|
|
@column()
|
|
declare shareShowCost: boolean
|
|
@column()
|
|
declare shareShowSellPrice: boolean
|
|
@column()
|
|
declare shareToken: string | null
|
|
@column()
|
|
declare stateId: number
|
|
@column()
|
|
declare title: string
|
|
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
|
declare updatedAt: DateTime | null
|
|
@column()
|
|
declare userId: number | null
|
|
}
|
|
|
|
export class ScrapedPartSchema extends BaseModel {
|
|
static $columns = ['createdAt', 'id', 'imageUrl', 'name', 'price', 'priceCheckedAt', 'priceSource', 'scrapedAt', 'source', 'sourceRef', 'sourceUrl', 'typeId', 'updatedAt'] as const
|
|
$columns = ScrapedPartSchema.$columns
|
|
@column.dateTime({ autoCreate: true })
|
|
declare createdAt: DateTime | null
|
|
@column({ isPrimary: true })
|
|
declare id: number
|
|
@column()
|
|
declare imageUrl: string | null
|
|
@column()
|
|
declare name: string
|
|
@column()
|
|
declare price: number | null
|
|
@column.dateTime()
|
|
declare priceCheckedAt: DateTime | null
|
|
@column()
|
|
declare priceSource: string | null
|
|
@column.dateTime()
|
|
declare scrapedAt: DateTime
|
|
@column()
|
|
declare source: string
|
|
@column()
|
|
declare sourceRef: string
|
|
@column()
|
|
declare sourceUrl: string
|
|
@column()
|
|
declare typeId: number
|
|
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
|
declare updatedAt: DateTime | null
|
|
}
|
|
|
|
export class UserSchema extends BaseModel {
|
|
static $columns = ['createdAt', 'email', 'id', 'name', 'password', 'updatedAt'] as const
|
|
$columns = UserSchema.$columns
|
|
@column.dateTime({ autoCreate: true })
|
|
declare createdAt: DateTime
|
|
@column()
|
|
declare email: string
|
|
@column({ isPrimary: true })
|
|
declare id: number
|
|
@column()
|
|
declare name: string | null
|
|
@column({ serializeAs: null })
|
|
declare password: string
|
|
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
|
declare updatedAt: DateTime | null
|
|
}
|