Kevin L'Hereec
5854ff8ed1
Idealo and PCPartPicker were evaluated and ruled out: - idealo.fr search pages return 503 (Cloudflare) - the internal Idealo GraphQL endpoint used by a public scraper (app.idealo.de/app-backend/api) is protected by Akamai Bot Manager and silently returns empty data even with mobile app headers - PCPartPicker's product list/search relies on a session-bound CSRF token refreshed per page load, against a scraping-hostile ToS pc-builder.io category pages are server-rendered, include name/price/image for ~100 products per category, and have no bot protection, so the feature is built on top of that source only. - New scraped_parts cache table (type_id, name, price, image_url, source, source_url, source_ref unique per source) populated by a batch job rather than scraped live per search, since only ~100 first results per category are available without reverse-engineering pc-builder.io's DataTables/CSRF pagination flow - part_scraper_service.ts: maps our ComponentType names to pc-builder.io category slugs (SSD -> storage, since pc-builder.io has no separate HDD category) and parses each category page with cheerio, skipping rows without a resolved price (generic 'search on Amazon' links) - node ace scrape:parts: refreshes the cache for every mapped category (upsert on source+source_ref), runnable manually or later via cron - GET /scraped-parts (authenticated): search the cache by name (LOWER() LIKE, avoids a MySQL utf8_bin/utf8mb4 collation mismatch from Lucid's whereLike) and optional typeId filter - NewPartSearch.vue: debounced search widget wired into components/create.vue; clicking 'Import' pre-fills the existing component creation form (name, price, type, purchase date = today) instead of creating the Component directly, so the user still reviews and saves it like any manually entered part - 3 functional tests covering search by name, filter by typeId, and unauthenticated access being redirected to login - Initial scrape run: 989 parts across 10 categories
88 lines
2.5 KiB
Vue
88 lines
2.5 KiB
Vue
<script setup lang="ts">
|
|
import { ref, watch } from 'vue'
|
|
import type { ScrapedPart } from '~/pages/types/UITypes'
|
|
|
|
const emit = defineEmits<{
|
|
import: [part: ScrapedPart]
|
|
}>()
|
|
|
|
const query = ref('')
|
|
const results = ref<ScrapedPart[]>([])
|
|
const isLoading = ref(false)
|
|
const hasSearched = ref(false)
|
|
let debounceTimer: ReturnType<typeof setTimeout> | null = null
|
|
|
|
async function search() {
|
|
if (!query.value.trim()) {
|
|
results.value = []
|
|
hasSearched.value = false
|
|
return
|
|
}
|
|
|
|
isLoading.value = true
|
|
try {
|
|
const response = await fetch(`/scraped-parts?q=${encodeURIComponent(query.value.trim())}`, {
|
|
headers: { Accept: 'application/json' },
|
|
})
|
|
if (response.ok) {
|
|
results.value = await response.json()
|
|
} else {
|
|
results.value = []
|
|
}
|
|
} finally {
|
|
isLoading.value = false
|
|
hasSearched.value = true
|
|
}
|
|
}
|
|
|
|
watch(query, () => {
|
|
if (debounceTimer) clearTimeout(debounceTimer)
|
|
debounceTimer = setTimeout(search, 400)
|
|
})
|
|
</script>
|
|
|
|
<template>
|
|
<div class="border border-gray-200 rounded-lg p-4 bg-gray-50">
|
|
<label class="block text-gray-700 text-sm font-bold mb-2">
|
|
Search a new part (pc-builder.io)
|
|
</label>
|
|
<input
|
|
v-model="query"
|
|
type="text"
|
|
placeholder="e.g. Ryzen 5 7600X, RTX 4070..."
|
|
class="w-full border-gray-200 focus:border-indigo-600 focus:ring focus:ring-indigo-500/40 px-3 py-2 border rounded text-gray-700"
|
|
/>
|
|
|
|
<p v-if="isLoading" class="text-sm text-gray-500 mt-2">Searching…</p>
|
|
<p v-else-if="hasSearched && results.length === 0" class="text-sm text-gray-500 mt-2">
|
|
No matching part found.
|
|
</p>
|
|
|
|
<ul v-if="results.length > 0" class="mt-3 space-y-2 max-h-72 overflow-y-auto">
|
|
<li
|
|
v-for="part in results"
|
|
:key="part.id"
|
|
class="flex items-center gap-3 bg-white border border-gray-200 rounded p-2"
|
|
>
|
|
<img
|
|
v-if="part.imageUrl"
|
|
:src="part.imageUrl"
|
|
alt=""
|
|
class="w-10 h-10 object-contain shrink-0"
|
|
/>
|
|
<div class="flex-1 min-w-0">
|
|
<p class="text-sm text-gray-800 truncate" :title="part.name">{{ part.name }}</p>
|
|
<p class="text-xs text-gray-500">{{ part.type.name }} · €{{ part.price.toFixed(2) }}</p>
|
|
</div>
|
|
<button
|
|
type="button"
|
|
class="px-3 py-1 text-sm bg-indigo-600 text-white rounded hover:bg-indigo-700 transition shrink-0"
|
|
@click="emit('import', part)"
|
|
>
|
|
Import
|
|
</button>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</template>
|