Kevin L'Hereec
5854ff8ed1
Idealo and PCPartPicker were evaluated and ruled out: - idealo.fr search pages return 503 (Cloudflare) - the internal Idealo GraphQL endpoint used by a public scraper (app.idealo.de/app-backend/api) is protected by Akamai Bot Manager and silently returns empty data even with mobile app headers - PCPartPicker's product list/search relies on a session-bound CSRF token refreshed per page load, against a scraping-hostile ToS pc-builder.io category pages are server-rendered, include name/price/image for ~100 products per category, and have no bot protection, so the feature is built on top of that source only. - New scraped_parts cache table (type_id, name, price, image_url, source, source_url, source_ref unique per source) populated by a batch job rather than scraped live per search, since only ~100 first results per category are available without reverse-engineering pc-builder.io's DataTables/CSRF pagination flow - part_scraper_service.ts: maps our ComponentType names to pc-builder.io category slugs (SSD -> storage, since pc-builder.io has no separate HDD category) and parses each category page with cheerio, skipping rows without a resolved price (generic 'search on Amazon' links) - node ace scrape:parts: refreshes the cache for every mapped category (upsert on source+source_ref), runnable manually or later via cron - GET /scraped-parts (authenticated): search the cache by name (LOWER() LIKE, avoids a MySQL utf8_bin/utf8mb4 collation mismatch from Lucid's whereLike) and optional typeId filter - NewPartSearch.vue: debounced search widget wired into components/create.vue; clicking 'Import' pre-fills the existing component creation form (name, price, type, purchase date = today) instead of creating the Component directly, so the user still reviews and saves it like any manually entered part - 3 functional tests covering search by name, filter by typeId, and unauthenticated access being redirected to login - Initial scrape run: 989 parts across 10 categories
102 lines
2.9 KiB
JSON
102 lines
2.9 KiB
JSON
{
|
|
"name": "pc-builder",
|
|
"version": "0.0.0",
|
|
"private": true,
|
|
"type": "module",
|
|
"license": "UNLICENSED",
|
|
"engines": {
|
|
"node": ">=24.0.0"
|
|
},
|
|
"scripts": {
|
|
"start": "node bin/server.js",
|
|
"build": "node ace build",
|
|
"dev": "node ace serve --hmr",
|
|
"test": "node ace test",
|
|
"lint": "eslint .",
|
|
"format": "prettier --write .",
|
|
"typecheck": "tsc --noEmit && vue-tsc --noEmit --project inertia/tsconfig.json"
|
|
},
|
|
"imports": {
|
|
"#controllers/*": "./app/controllers/*.js",
|
|
"#exceptions/*": "./app/exceptions/*.js",
|
|
"#models/*": "./app/models/*.js",
|
|
"#mails/*": "./app/mails/*.js",
|
|
"#services/*": "./app/services/*.js",
|
|
"#listeners/*": "./app/listeners/*.js",
|
|
"#events/*": "./app/events/*.js",
|
|
"#generated/*": "./.adonisjs/server/*.js",
|
|
"#middleware/*": "./app/middleware/*.js",
|
|
"#transformers/*": "./app/transformers/*.js",
|
|
"#types/*": "./types/*.js",
|
|
"#validators/*": "./app/validators/*.js",
|
|
"#providers/*": "./providers/*.js",
|
|
"#policies/*": "./app/policies/*.js",
|
|
"#abilities/*": "./app/abilities/*.js",
|
|
"#database/*": "./database/*.js",
|
|
"#tests/*": "./tests/*.js",
|
|
"#start/*": "./start/*.js",
|
|
"#config/*": "./config/*.js"
|
|
},
|
|
"devDependencies": {
|
|
"@adonisjs/assembler": "^8.4.0",
|
|
"@adonisjs/eslint-config": "^3.1.0",
|
|
"@adonisjs/prettier-config": "^1.5.0",
|
|
"@adonisjs/tsconfig": "^2.0.0",
|
|
"@japa/api-client": "^3.2.1",
|
|
"@japa/assert": "^4.2.0",
|
|
"@japa/browser-client": "^2.3.0",
|
|
"@japa/plugin-adonisjs": "^5.2.0",
|
|
"@japa/runner": "^5.3.0",
|
|
"@poppinss/ts-exec": "^1.4.4",
|
|
"@types/luxon": "^3.7.1",
|
|
"@types/node": "~25.9.1",
|
|
"@vitejs/plugin-vue": "^6.0.7",
|
|
"autoprefixer": "^10.5.2",
|
|
"eslint": "^10.4.0",
|
|
"eslint-plugin-vue": "^10.9.1",
|
|
"hot-hook": "^1.0.0",
|
|
"pino-pretty": "^13.1.3",
|
|
"postcss": "^8.5.15",
|
|
"prettier": "^3.8.3",
|
|
"tailwindcss": "^3.4.19",
|
|
"typescript": "~6.0.3",
|
|
"vite": "^7.3.3",
|
|
"vue-tsc": "^3.3.2",
|
|
"youch": "^4.1.1"
|
|
},
|
|
"dependencies": {
|
|
"@adonisjs/auth": "^10.1.0",
|
|
"@adonisjs/core": "^7.3.3",
|
|
"@adonisjs/cors": "^3.0.0",
|
|
"@adonisjs/inertia": "^4.2.0",
|
|
"@adonisjs/lucid": "^22.4.2",
|
|
"@adonisjs/session": "^8.1.0",
|
|
"@adonisjs/shield": "^9.0.0",
|
|
"@adonisjs/static": "^2.0.1",
|
|
"@adonisjs/vite": "^5.1.1",
|
|
"@inertiajs/vue3": "^2.3.24",
|
|
"@tuyau/core": "^1.2.2",
|
|
"@types/uuid": "^10.0.0",
|
|
"@vinejs/vine": "^4.4.0",
|
|
"@vue/server-renderer": "^3.5.35",
|
|
"@vuepic/vue-datepicker": "^13.0.0",
|
|
"better-sqlite3": "^12.10.0",
|
|
"cheerio": "^1.2.0",
|
|
"edge.js": "^6.5.1",
|
|
"luxon": "^3.7.2",
|
|
"mysql2": "^3.22.4",
|
|
"reflect-metadata": "^0.2.2",
|
|
"uuid": "^14.0.1",
|
|
"vue": "^3.5.35",
|
|
"vue-sonner": "^2.0.9",
|
|
"vue3-carousel": "^0.17.0"
|
|
},
|
|
"hotHook": {
|
|
"boundaries": [
|
|
"./app/controllers/**/*.ts",
|
|
"./app/middleware/*.ts"
|
|
]
|
|
},
|
|
"prettier": "@adonisjs/prettier-config"
|
|
}
|