Add species_detail trait pipeline with static dataset imports

- Add trait_importer.py to populate species_detail from PanTHERIA
  (mammals), EltonTraits (birds/mammals), and AmphiBIO (amphibians)
- Add EOL_requester.py with Wikidata SPARQL trait fetcher as fallback
- Add source trait CSVs to wildlife/trait_data/
- Update wilddex.db with initial trait data

dev_scripts/EOL_requester.py

@@ -0,0 +1,106 @@
+import requests
+
+SPARQL_URL = "https://query.wikidata.org/sparql"
+HEADERS = {"User-Agent": "WildDex/1.0 (wildlife app; contact via github)", "Accept": "application/sparql-results+json"}
+
+
+def fetch_wikidata_traits(scientific_name: str) -> dict:
+    """
+    Query Wikidata by scientific name and return available trait data.
+    Returns a dict with keys matching species_detail columns (values may be None).
+    """
+    query = f"""
+    SELECT
+      ?mass ?massUnit ?massUnitLabel
+      ?lifespan ?lifespanUnit ?lifespanUnitLabel
+      ?dietLabel
+      ?habitatLabel
+      ?activityLabel
+    WHERE {{
+      ?item wdt:P225 "{scientific_name}" .
+
+      OPTIONAL {{
+        ?item p:P2067 ?massStmt .
+        ?massStmt psv:P2067 ?massVal .
+        ?massVal wikibase:quantityAmount ?mass .
+        ?massVal wikibase:quantityUnit ?massUnit .
+      }}
+      OPTIONAL {{
+        ?item p:P2250 ?lifespanStmt .
+        ?lifespanStmt psv:P2250 ?lifespanVal .
+        ?lifespanVal wikibase:quantityAmount ?lifespan .
+        ?lifespanVal wikibase:quantityUnit ?lifespanUnit .
+      }}
+      OPTIONAL {{ ?item wdt:P1034 ?diet . }}
+      OPTIONAL {{ ?item wdt:P2974 ?habitat . }}
+      OPTIONAL {{ ?item wdt:P5204 ?activity . }}
+
+      SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en" . }}
+    }}
+    LIMIT 5
+    """
+
+    resp = requests.get(SPARQL_URL, params={"query": query, "format": "json"}, headers=HEADERS)
+    resp.raise_for_status()
+    bindings = resp.json()["results"]["bindings"]
+
+    if not bindings:
+        print(f"  No Wikidata results for '{scientific_name}'")
+        return {}
+
+    return _extract_traits(bindings)
+
+
+def _extract_traits(bindings: list[dict]) -> dict:
+    result = {
+        "size_range": None,
+        "weight_range": None,
+        "lifespan": None,
+        "diet": None,
+        "activity_period": None,
+        "habitat_type": None,
+    }
+
+    diets, habitats, activities = set(), set(), set()
+    masses, lifespans = [], []
+
+    for row in bindings:
+        if "mass" in row and "massUnitLabel" in row:
+            mass = float(row["mass"]["value"])
+            unit = row["massUnitLabel"]["value"]
+            masses.append(f"{mass:g} {unit}")
+
+        if "lifespan" in row and "lifespanUnitLabel" in row:
+            ls = float(row["lifespan"]["value"])
+            unit = row["lifespanUnitLabel"]["value"]
+            lifespans.append(f"{ls:g} {unit}")
+
+        if "dietLabel" in row:
+            diets.add(row["dietLabel"]["value"])
+
+        if "habitatLabel" in row:
+            habitats.add(row["habitatLabel"]["value"])
+
+        if "activityLabel" in row:
+            activities.add(row["activityLabel"]["value"])
+
+    if masses:
+        result["weight_range"] = " / ".join(sorted(set(masses)))
+    if lifespans:
+        result["lifespan"] = " / ".join(sorted(set(lifespans)))
+    if diets:
+        result["diet"] = ", ".join(sorted(diets))
+    if habitats:
+        result["habitat_type"] = ", ".join(sorted(habitats))
+    if activities:
+        result["activity_period"] = ", ".join(sorted(activities))
+
+    return result
+
+
+if __name__ == "__main__":
+    test_species = "Thamnophis sirtalis"
+    print(f"Fetching Wikidata traits for '{test_species}'...")
+    traits = fetch_wikidata_traits(test_species)
+    for key, value in traits.items():
+        print(f"  {key}: {value}")

dev_scripts/trait_importer.py

@@ -0,0 +1,230 @@
+"""
+Populates the species_detail table from static trait CSV downloads.
+
+Required files in wildlife/trait_data/:
+  - PanTHERIA_1-0_WR05_Aug2008.txt   (mammals)  https://ecologicaldata.org/wiki/pantheria
+  - EltonTraits_BirdFuncDat.txt       (birds)    https://figshare.com/articles/dataset/EltonTraits_1_0/949799
+  - EltonTraits_MamFuncDat.txt        (mammals)  same Figshare link above
+  - AmphiBIO_v1.csv                   (amphibians) https://figshare.com/articles/dataset/AmphiBIO_v1/4644424
+
+Place all files in: wildlife/trait_data/
+"""
+
+import csv
+import sqlite3
+from pathlib import Path
+
+DB_PATH = Path(__file__).parent.parent / "wilddex.db"
+TRAIT_DATA_DIR = Path(__file__).parent.parent / "wildlife" / "trait_data"
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def get_species_map(conn: sqlite3.Connection) -> dict[str, int]:
+    """Return scientific_name -> species_id for all rows in the species table."""
+    rows = conn.execute("SELECT species_id, scientific_name FROM species").fetchall()
+    return {sci: sid for sid, sci in rows}
+
+
+def upsert_detail(conn: sqlite3.Connection, species_id: int, traits: dict) -> None:
+    """Insert or update a species_detail row, skipping None values."""
+    traits = {k: v for k, v in traits.items() if v is not None}
+    if not traits:
+        return
+
+    existing = conn.execute(
+        "SELECT detail_id FROM species_detail WHERE species_id = ?", (species_id,)
+    ).fetchone()
+
+    if existing:
+        sets = ", ".join(f"{col} = ?" for col in traits)
+        conn.execute(
+            f"UPDATE species_detail SET {sets} WHERE species_id = ?",
+            (*traits.values(), species_id),
+        )
+    else:
+        cols = "species_id, " + ", ".join(traits.keys())
+        placeholders = ", ".join("?" * (len(traits) + 1))
+        conn.execute(
+            f"INSERT INTO species_detail ({cols}) VALUES ({placeholders})",
+            (species_id, *traits.values()),
+        )
+
+
+# ---------------------------------------------------------------------------
+# PanTHERIA — mammals
+# ---------------------------------------------------------------------------
+
+def import_pantheria(conn: sqlite3.Connection, species_map: dict[str, int]) -> None:
+    path = TRAIT_DATA_DIR / "PanTHERIA_1-0_WR05_Aug2008.txt"
+    if not path.exists():
+        print(f"  Skipping PanTHERIA (file not found: {path})")
+        return
+
+    matched = 0
+    with open(path, newline="", encoding="utf-8") as f:
+        reader = csv.DictReader(f, delimiter="\t")
+        for row in reader:
+            sci_name = row.get("MSW05_Binomial", "").strip()
+            species_id = species_map.get(sci_name)
+            if not species_id:
+                continue
+
+            mass_g = _float(row.get("5-1_AdultBodyMass_g"))
+            longevity_m = _float(row.get("17-1_MaxLongevity_m"))
+
+            traits = {
+                "weight_range": f"{mass_g:g} g" if mass_g else None,
+                "lifespan": f"{longevity_m / 12:.1f} years" if longevity_m else None,
+            }
+            upsert_detail(conn, species_id, traits)
+            matched += 1
+
+    conn.commit()
+    print(f"  PanTHERIA: {matched} species matched")
+
+
+# ---------------------------------------------------------------------------
+# EltonTraits — birds and mammals
+# ---------------------------------------------------------------------------
+
+ELTON_DIET_COLS = [
+    ("Diet.Inv",   "invertebrates"),
+    ("Diet.Vend",  "vertebrates (endotherm)"),
+    ("Diet.Vect",  "vertebrates (ectotherm)"),
+    ("Diet.Vfish", "fish"),
+    ("Diet.Vunk",  "vertebrates (unknown)"),
+    ("Diet.Scav",  "scavenging"),
+    ("Diet.Fruit", "fruit"),
+    ("Diet.Nect",  "nectar"),
+    ("Diet.Seed",  "seeds"),
+    ("Diet.PlantO","plant matter"),
+]
+
+def _elton_diet(row: dict) -> str | None:
+    parts = []
+    for col, label in ELTON_DIET_COLS:
+        val = _float(row.get(col))
+        if val and val > 10:  # only include if >10% of diet
+            parts.append(f"{label} ({val:.0f}%)")
+    return ", ".join(parts) if parts else None
+
+def _elton_activity(row: dict) -> str | None:
+    val = _float(row.get("Nocturnal"))
+    if val is None:
+        return None
+    return "nocturnal" if val == 1 else "diurnal"
+
+def _import_elton_file(conn: sqlite3.Connection, species_map: dict[str, int], path: Path, sci_col: str) -> int:
+    matched = 0
+    with open(path, newline="", encoding="latin-1") as f:
+        reader = csv.DictReader(f, delimiter="\t")
+        for row in reader:
+            sci_name = row.get(sci_col, "").strip().replace("_", " ")
+            species_id = species_map.get(sci_name)
+            if not species_id:
+                continue
+
+            mass_g = _float(row.get("BodyMass.Value"))
+            traits = {
+                "weight_range": f"{mass_g:g} g" if mass_g else None,
+                "diet": _elton_diet(row),
+                "activity_period": _elton_activity(row),
+            }
+            upsert_detail(conn, species_id, traits)
+            matched += 1
+    return matched
+
+def import_elton(conn: sqlite3.Connection, species_map: dict[str, int]) -> None:
+    bird_path = TRAIT_DATA_DIR / "EltonTraits_BirdFuncDat.txt"
+    mam_path  = TRAIT_DATA_DIR / "EltonTraits_MamFuncDat.txt"
+
+    total = 0
+    if bird_path.exists():
+        n = _import_elton_file(conn, species_map, bird_path, sci_col="Scientific")
+        total += n
+        print(f"  EltonTraits birds: {n} species matched")
+    else:
+        print(f"  Skipping EltonTraits birds (file not found)")
+
+    if mam_path.exists():
+        n = _import_elton_file(conn, species_map, mam_path, sci_col="Scientific")
+        total += n
+        print(f"  EltonTraits mammals: {n} species matched")
+    else:
+        print(f"  Skipping EltonTraits mammals (file not found)")
+
+    conn.commit()
+
+
+# ---------------------------------------------------------------------------
+# AmphiBIO — amphibians
+# ---------------------------------------------------------------------------
+
+def import_amphibio(conn: sqlite3.Connection, species_map: dict[str, int]) -> None:
+    path = TRAIT_DATA_DIR / "AmphiBIO_v1.csv"
+    if not path.exists():
+        print(f"  Skipping AmphiBIO (file not found: {path})")
+        return
+
+    matched = 0
+    with open(path, newline="", encoding="latin-1") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            sci_name = row.get("Species", "").strip()
+            species_id = species_map.get(sci_name)
+            if not species_id:
+                continue
+
+            size_min = _float(row.get("Body_size_mm"))
+            longevity = _float(row.get("Longevity_max_y"))
+            diet = row.get("Prey") or None
+            habitat = row.get("Habitat") or None
+            activity = row.get("Diel_activity") or None
+
+            traits = {
+                "size_range": f"{size_min:g} mm" if size_min else None,
+                "lifespan": f"{longevity:g} years" if longevity else None,
+                "diet": diet,
+                "habitat_type": habitat,
+                "activity_period": activity,
+            }
+            upsert_detail(conn, species_id, traits)
+            matched += 1
+
+    conn.commit()
+    print(f"  AmphiBIO: {matched} species matched")
+
+
+# ---------------------------------------------------------------------------
+# Utilities
+# ---------------------------------------------------------------------------
+
+def _float(val) -> float | None:
+    try:
+        f = float(val)
+        return None if f < 0 else f  # datasets use -999 for missing
+    except (TypeError, ValueError):
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    if not TRAIT_DATA_DIR.exists():
+        TRAIT_DATA_DIR.mkdir(parents=True)
+        print(f"Created {TRAIT_DATA_DIR} — place trait CSVs here and re-run.")
+    else:
+        with sqlite3.connect(DB_PATH) as conn:
+            species_map = get_species_map(conn)
+            print(f"Loaded {len(species_map)} species from DB\n")
+
+            import_pantheria(conn, species_map)
+            import_elton(conn, species_map)
+            import_amphibio(conn, species_map)
+
+        print("\nDone.")