Stephan Kerkman
ba0ca6b169
- Add scan-to-plant flow: upload card photos, OCR, VPC match on confirm form - OpenCV CLAHE + adaptive threshold preprocessing; dual psm 6/11 OCR pass - Regex fixes for OCR misreads: height (s→5), frost (=,„,i,l,G), density, bloom months (E→I) - Scientific name extraction: passport section + cultivar quote strategy - Last-wins merge so back card data overrides front card garbage - Crop thumbnail tool with Cropper.js; cropped image shown in list and detail - VPC image always supersedes user thumbnail on detail page - VPC sunlight mapping: zon→full_sun, halfschaduw→part_sun, schaduw→full_shade - Auto-set thumbnail on plant created from scan Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
250 lines
8.2 KiB
Python
250 lines
8.2 KiB
Python
"""
|
||
Scraper for vasteplantencatalogus.nl (VPC).
|
||
- search_species(query) -> list of lightweight result dicts
|
||
- scrape_plant(slug) -> full detail dict ready to store as Species
|
||
|
||
Notes on VPC data:
|
||
- Plant titles ARE the Latin names (no separate common name)
|
||
- Bloom months listed in Dutch ("september, oktober")
|
||
- Height in a dedicated "Hoogte in cm" field as plain integer
|
||
- Base URL is https://vasteplantencatalogus.nl (no www)
|
||
"""
|
||
import re
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
|
||
_BASE = 'https://vasteplantencatalogus.nl'
|
||
_TIMEOUT = 8
|
||
_HEADERS = {'User-Agent': 'PlantDB/1.0 (personal plant tracker)'}
|
||
|
||
_NL_MONTHS = {
|
||
'januari': 1, 'februari': 2, 'maart': 3, 'april': 4,
|
||
'mei': 5, 'juni': 6, 'juli': 7, 'augustus': 8,
|
||
'september': 9, 'oktober': 10, 'november': 11, 'december': 12,
|
||
}
|
||
|
||
# Also handle Roman numerals (some older pages still use them)
|
||
_ROMAN_MAP = {
|
||
'XII': 12, 'VIII': 8, 'VII': 7, 'VI': 6, 'XI': 11,
|
||
'IX': 9, 'IV': 4, 'III': 3, 'II': 2, 'X': 10, 'V': 5, 'I': 1,
|
||
}
|
||
|
||
# VPC uses curly/smart quotes around cultivar names in product titles.
|
||
# Straight apostrophe + left/right single quotation marks + modifier apostrophe.
|
||
_QUOTE_CHARS = "'" + chr(0x2018) + chr(0x2019) + chr(0x02bc)
|
||
_Q = '[' + re.escape(_QUOTE_CHARS) + ']'
|
||
_NQ = '[^' + re.escape(_QUOTE_CHARS) + ']'
|
||
_CULTIVAR_RE = re.compile(_Q + '(' + _NQ + '+)' + _Q)
|
||
_CULTIVAR_STRIP_RE = re.compile(r'\s*' + _Q + _NQ + '+' + _Q)
|
||
|
||
|
||
class VPCError(Exception):
|
||
pass
|
||
|
||
|
||
def _parse_months(text):
|
||
"""Parse Dutch or Roman numeral month strings into a sorted list of ints."""
|
||
result = set()
|
||
text_lower = text.lower().replace('–', '-').replace('—', '-')
|
||
|
||
# Try Dutch names first
|
||
for name, month in _NL_MONTHS.items():
|
||
if name in text_lower:
|
||
result.add(month)
|
||
|
||
if result:
|
||
return sorted(result)
|
||
|
||
# Fall back to Roman numerals
|
||
text_upper = text.upper().replace('–', '-').replace('—', '-')
|
||
for part in re.split(r'[,;\s]+', text_upper):
|
||
part = part.strip()
|
||
if not part:
|
||
continue
|
||
if '-' in part:
|
||
lo_str, _, hi_str = part.partition('-')
|
||
lo = _ROMAN_MAP.get(lo_str.strip())
|
||
hi = _ROMAN_MAP.get(hi_str.strip())
|
||
if lo and hi:
|
||
if lo <= hi:
|
||
result.update(range(lo, hi + 1))
|
||
else:
|
||
result.update(range(lo, 13))
|
||
result.update(range(1, hi + 1))
|
||
else:
|
||
m = _ROMAN_MAP.get(part)
|
||
if m:
|
||
result.add(m)
|
||
|
||
return sorted(result)
|
||
|
||
|
||
def _slug_from_url(url):
|
||
m = re.search(r'/soorten/([^/?#]+)/?', url)
|
||
return m.group(1) if m else ''
|
||
|
||
|
||
def slug_from_query(query):
|
||
"""Return a VPC slug if query is a VPC URL or bare slug, else empty string."""
|
||
query = query.strip()
|
||
# Full URL containing /soorten/
|
||
slug = _slug_from_url(query)
|
||
if slug:
|
||
return slug
|
||
# Bare slug: lowercase, hyphens, no spaces, must contain at least one hyphen
|
||
# (single words are normal search terms, not slugs)
|
||
if re.fullmatch(r'[a-z0-9][a-z0-9\-]+', query) and '-' in query:
|
||
return query
|
||
return ''
|
||
|
||
|
||
_STRIP_QUOTES_RE = re.compile('[' + re.escape(_QUOTE_CHARS) + ']')
|
||
|
||
|
||
def _strip_quotes(text):
|
||
return _STRIP_QUOTES_RE.sub('', text)
|
||
|
||
|
||
def search_species(query):
|
||
"""
|
||
Search VPC and return up to 8 result dicts:
|
||
{slug, common_name, scientific_name, thumbnail_url, source}
|
||
|
||
On VPC, the title IS the Latin name so both name fields are set to it.
|
||
Raises VPCError on network failure.
|
||
"""
|
||
# VPC's WordPress search breaks on multi-word and quoted queries.
|
||
# Strategy: search with only the first word (genus), then filter
|
||
# client-side so all query words appear in the result title.
|
||
# Quote chars are stripped from both query and titles before matching.
|
||
words = _strip_quotes(query).split()
|
||
vpc_query = words[0] if words else query
|
||
|
||
try:
|
||
resp = requests.get(
|
||
_BASE + '/',
|
||
params={'s': vpc_query, 'post_type': 'product'},
|
||
timeout=_TIMEOUT,
|
||
headers=_HEADERS,
|
||
)
|
||
resp.raise_for_status()
|
||
except requests.Timeout:
|
||
raise VPCError('VPC timed out.')
|
||
except Exception as e:
|
||
raise VPCError('VPC unreachable: ' + str(e))
|
||
|
||
soup = BeautifulSoup(resp.text, 'html.parser')
|
||
seen_slugs = set()
|
||
results = []
|
||
|
||
for item in soup.select('li.product, article.product'):
|
||
# Get the first product link (skip "Lees verder" duplicates)
|
||
for link in item.find_all('a', href=True):
|
||
slug = _slug_from_url(link['href'])
|
||
if slug and slug not in seen_slugs:
|
||
seen_slugs.add(slug)
|
||
title_el = item.find(['h2', 'h3'], class_=re.compile(r'title|product', re.I))
|
||
title = title_el.get_text(strip=True) if title_el else link.get_text(strip=True)
|
||
if title.lower() in ('lees verder', 'read more', ''):
|
||
continue
|
||
img = item.find('img')
|
||
thumbnail = img.get('src', '') if img else ''
|
||
results.append({
|
||
'slug': slug,
|
||
'common_name': title,
|
||
'scientific_name': title,
|
||
'thumbnail_url': thumbnail,
|
||
'source': 'vpc',
|
||
})
|
||
break
|
||
|
||
if len(results) >= 8:
|
||
break
|
||
|
||
if len(words) > 1:
|
||
words_lower = [w.lower() for w in words]
|
||
filtered = [
|
||
r for r in results
|
||
if all(w in _strip_quotes(r['common_name']).lower() for w in words_lower)
|
||
]
|
||
if filtered:
|
||
return filtered
|
||
|
||
return results
|
||
|
||
|
||
def scrape_plant(slug):
|
||
"""
|
||
Scrape a VPC plant detail page and return a dict with:
|
||
common_name, scientific_name, bloom_months, max_height_cm,
|
||
sunlight, growth_rate, description, api_image_url, vpc_slug
|
||
|
||
Raises VPCError on failure.
|
||
"""
|
||
url = _BASE + '/soorten/' + slug + '/'
|
||
try:
|
||
resp = requests.get(url, timeout=_TIMEOUT, headers=_HEADERS)
|
||
resp.raise_for_status()
|
||
except requests.Timeout:
|
||
raise VPCError('VPC timed out while fetching plant page.')
|
||
except Exception as e:
|
||
raise VPCError('Could not fetch plant page: ' + str(e))
|
||
|
||
soup = BeautifulSoup(resp.text, 'html.parser')
|
||
|
||
# Title IS the Latin name on VPC
|
||
h1 = soup.find('h1')
|
||
name = h1.get_text(strip=True) if h1 else slug.replace('-', ' ').title()
|
||
|
||
# Hero image -- VPC wraps the plant photo in a .wp-caption figure
|
||
hero_img = ''
|
||
for sel in ['.wp-caption img', 'figure img', '.elementor-widget-image img']:
|
||
img = soup.select_one(sel)
|
||
src = img.get('src', '') if img else ''
|
||
if src.startswith('http') and 'Grif_VPC' not in src and 'NoImage' not in src:
|
||
hero_img = src
|
||
break
|
||
|
||
result = {
|
||
'common_name': name,
|
||
'scientific_name': name,
|
||
'bloom_months': [],
|
||
'max_height_cm': None,
|
||
'sunlight': '',
|
||
'growth_rate': '',
|
||
'description': '',
|
||
'api_image_url': hero_img,
|
||
'vpc_slug': slug,
|
||
}
|
||
|
||
# Parse <b>Label:</b> + next sibling text
|
||
for b in soup.find_all('b'):
|
||
label = b.get_text(strip=True).rstrip(':').strip()
|
||
sibling = b.next_sibling
|
||
if sibling is None:
|
||
continue
|
||
value = str(sibling).strip().lstrip(':').strip()
|
||
if not value:
|
||
continue
|
||
|
||
if label in ('Bloeitijd', 'Bloeimaanden'):
|
||
result['bloom_months'] = _parse_months(value)
|
||
elif label == 'Hoogte in cm':
|
||
try:
|
||
result['max_height_cm'] = int(re.search(r'\d+', value).group())
|
||
except (AttributeError, ValueError):
|
||
pass
|
||
elif label == 'Standplaats':
|
||
v = value.lower()
|
||
if 'halfschaduw' in v or 'half' in v or 'part' in v:
|
||
result['sunlight'] = 'part_sun'
|
||
elif 'schaduw' in v or 'shade' in v:
|
||
result['sunlight'] = 'full_shade'
|
||
elif 'zon' in v or 'sun' in v:
|
||
result['sunlight'] = 'full_sun'
|
||
else:
|
||
result['sunlight'] = value
|
||
elif label in ('Groeisnelheid', 'Groeivorm'):
|
||
result['growth_rate'] = value
|
||
|
||
return result
|