bloombase/plants/services/vpc.py

"""
Scraper for vasteplantencatalogus.nl (VPC).
- search_species(query) -> list of lightweight result dicts
- scrape_plant(slug)    -> full detail dict ready to store as Species

Notes on VPC data:
- Plant titles ARE the Latin names (no separate common name)
- Bloom months listed in Dutch ("september, oktober")
- Height in a dedicated "Hoogte in cm" field as plain integer
- Base URL is https://vasteplantencatalogus.nl (no www)
"""
import re
import requests
from bs4 import BeautifulSoup

_BASE = 'https://vasteplantencatalogus.nl'
_TIMEOUT = 8
_HEADERS = {'User-Agent': 'PlantDB/1.0 (personal plant tracker)'}

_NL_MONTHS = {
    'januari': 1, 'februari': 2, 'maart': 3, 'april': 4,
    'mei': 5, 'juni': 6, 'juli': 7, 'augustus': 8,
    'september': 9, 'oktober': 10, 'november': 11, 'december': 12,
}

# Also handle Roman numerals (some older pages still use them)
_ROMAN_MAP = {
    'XII': 12, 'VIII': 8, 'VII': 7, 'VI': 6, 'XI': 11,
    'IX': 9, 'IV': 4, 'III': 3, 'II': 2, 'X': 10, 'V': 5, 'I': 1,
}

# VPC uses curly/smart quotes around cultivar names in product titles.
# Straight apostrophe + left/right single quotation marks + modifier apostrophe.
_QUOTE_CHARS = "'" + chr(0x2018) + chr(0x2019) + chr(0x02bc)
_Q = '[' + re.escape(_QUOTE_CHARS) + ']'
_NQ = '[^' + re.escape(_QUOTE_CHARS) + ']'
_CULTIVAR_RE = re.compile(_Q + '(' + _NQ + '+)' + _Q)
_CULTIVAR_STRIP_RE = re.compile(r'\s*' + _Q + _NQ + '+' + _Q)


class VPCError(Exception):
    pass


def _parse_months(text):
    """Parse Dutch or Roman numeral month strings into a sorted list of ints."""
    result = set()
    text_lower = text.lower().replace('–', '-').replace('—', '-')

    # Try Dutch names first
    for name, month in _NL_MONTHS.items():
        if name in text_lower:
            result.add(month)

    if result:
        return sorted(result)

    # Fall back to Roman numerals
    text_upper = text.upper().replace('–', '-').replace('—', '-')
    for part in re.split(r'[,;\s]+', text_upper):
        part = part.strip()
        if not part:
            continue
        if '-' in part:
            lo_str, _, hi_str = part.partition('-')
            lo = _ROMAN_MAP.get(lo_str.strip())
            hi = _ROMAN_MAP.get(hi_str.strip())
            if lo and hi:
                if lo <= hi:
                    result.update(range(lo, hi + 1))
                else:
                    result.update(range(lo, 13))
                    result.update(range(1, hi + 1))
        else:
            m = _ROMAN_MAP.get(part)
            if m:
                result.add(m)

    return sorted(result)


def _slug_from_url(url):
    m = re.search(r'/soorten/([^/?#]+)/?', url)
    return m.group(1) if m else ''


def slug_from_query(query):
    """Return a VPC slug if query is a VPC URL or bare slug, else empty string."""
    query = query.strip()
    # Full URL containing /soorten/
    slug = _slug_from_url(query)
    if slug:
        return slug
    # Bare slug: lowercase, hyphens, no spaces, must contain at least one hyphen
    # (single words are normal search terms, not slugs)
    if re.fullmatch(r'[a-z0-9][a-z0-9\-]+', query) and '-' in query:
        return query
    return ''


_STRIP_QUOTES_RE = re.compile('[' + re.escape(_QUOTE_CHARS) + ']')


def _strip_quotes(text):
    return _STRIP_QUOTES_RE.sub('', text)


def search_species(query):
    """
    Search VPC and return up to 8 result dicts:
      {slug, common_name, scientific_name, thumbnail_url, source}

    On VPC, the title IS the Latin name so both name fields are set to it.
    Raises VPCError on network failure.
    """
    # VPC's WordPress search breaks on multi-word and quoted queries.
    # Strategy: search with only the first word (genus), then filter
    # client-side so all query words appear in the result title.
    # Quote chars are stripped from both query and titles before matching.
    words = _strip_quotes(query).split()
    vpc_query = words[0] if words else query

    try:
        resp = requests.get(
            _BASE + '/',
            params={'s': vpc_query, 'post_type': 'product'},
            timeout=_TIMEOUT,
            headers=_HEADERS,
        )
        resp.raise_for_status()
    except requests.Timeout:
        raise VPCError('VPC timed out.')
    except Exception as e:
        raise VPCError('VPC unreachable: ' + str(e))

    soup = BeautifulSoup(resp.text, 'html.parser')
    seen_slugs = set()
    results = []

    for item in soup.select('li.product, article.product'):
        # Get the first product link (skip "Lees verder" duplicates)
        for link in item.find_all('a', href=True):
            slug = _slug_from_url(link['href'])
            if slug and slug not in seen_slugs:
                seen_slugs.add(slug)
                title_el = item.find(['h2', 'h3'], class_=re.compile(r'title|product', re.I))
                title = title_el.get_text(strip=True) if title_el else link.get_text(strip=True)
                if title.lower() in ('lees verder', 'read more', ''):
                    continue
                img = item.find('img')
                thumbnail = img.get('src', '') if img else ''
                results.append({
                    'slug': slug,
                    'common_name': title,
                    'scientific_name': title,
                    'thumbnail_url': thumbnail,
                    'source': 'vpc',
                })
                break

        if len(results) >= 8:
            break

    if len(words) > 1:
        words_lower = [w.lower() for w in words]
        filtered = [
            r for r in results
            if all(w in _strip_quotes(r['common_name']).lower() for w in words_lower)
        ]
        if filtered:
            return filtered

    return results


def scrape_plant(slug):
    """
    Scrape a VPC plant detail page and return a dict with:
      common_name, scientific_name, bloom_months, max_height_cm,
      sunlight, growth_rate, description, api_image_url, vpc_slug

    Raises VPCError on failure.
    """
    url = _BASE + '/soorten/' + slug + '/'
    try:
        resp = requests.get(url, timeout=_TIMEOUT, headers=_HEADERS)
        resp.raise_for_status()
    except requests.Timeout:
        raise VPCError('VPC timed out while fetching plant page.')
    except Exception as e:
        raise VPCError('Could not fetch plant page: ' + str(e))

    soup = BeautifulSoup(resp.text, 'html.parser')

    # Title IS the Latin name on VPC
    h1 = soup.find('h1')
    name = h1.get_text(strip=True) if h1 else slug.replace('-', ' ').title()

    # Hero image -- VPC wraps the plant photo in a .wp-caption figure
    hero_img = ''
    for sel in ['.wp-caption img', 'figure img', '.elementor-widget-image img']:
        img = soup.select_one(sel)
        src = img.get('src', '') if img else ''
        if src.startswith('http') and 'Grif_VPC' not in src and 'NoImage' not in src:
            hero_img = src
            break

    result = {
        'common_name': name,
        'scientific_name': name,
        'bloom_months': [],
        'max_height_cm': None,
        'sunlight': '',
        'growth_rate': '',
        'description': '',
        'api_image_url': hero_img,
        'vpc_slug': slug,
    }

    # Parse <b>Label:</b> + next sibling text
    for b in soup.find_all('b'):
        label = b.get_text(strip=True).rstrip(':').strip()
        sibling = b.next_sibling
        if sibling is None:
            continue
        value = str(sibling).strip().lstrip(':').strip()
        if not value:
            continue

        if label in ('Bloeitijd', 'Bloeimaanden'):
            result['bloom_months'] = _parse_months(value)
        elif label == 'Hoogte in cm':
            try:
                result['max_height_cm'] = int(re.search(r'\d+', value).group())
            except (AttributeError, ValueError):
                pass
        elif label == 'Standplaats':
            v = value.lower()
            if 'halfschaduw' in v or 'half' in v or 'part' in v:
                result['sunlight'] = 'part_sun'
            elif 'schaduw' in v or 'shade' in v:
                result['sunlight'] = 'full_shade'
            elif 'zon' in v or 'sun' in v:
                result['sunlight'] = 'full_sun'
            else:
                result['sunlight'] = value
        elif label in ('Groeisnelheid', 'Groeivorm'):
            result['growth_rate'] = value

    return result