Add species_detail trait pipeline with static dataset imports
- Add trait_importer.py to populate species_detail from PanTHERIA (mammals), EltonTraits (birds/mammals), and AmphiBIO (amphibians) - Add EOL_requester.py with Wikidata SPARQL trait fetcher as fallback - Add source trait CSVs to wildlife/trait_data/ - Update wilddex.db with initial trait data
This commit is contained in:
Suppai_Neko committed
1 parent
de8a7e7c66
commit
afa683b7e3
7 files changed
+28140
No files matched your search
@@ -0,0 +1,106 @@
|
||||
import requests
|
||||
|
||||
SPARQL_URL = "https://query.wikidata.org/sparql"
|
||||
HEADERS = {"User-Agent": "WildDex/1.0 (wildlife app; contact via github)", "Accept": "application/sparql-results+json"}
|
||||
|
||||
|
||||
def fetch_wikidata_traits(scientific_name: str) -> dict:
|
||||
"""
|
||||
Query Wikidata by scientific name and return available trait data.
|
||||
Returns a dict with keys matching species_detail columns (values may be None).
|
||||
"""
|
||||
query = f"""
|
||||
SELECT
|
||||
?mass ?massUnit ?massUnitLabel
|
||||
?lifespan ?lifespanUnit ?lifespanUnitLabel
|
||||
?dietLabel
|
||||
?habitatLabel
|
||||
?activityLabel
|
||||
WHERE {{
|
||||
?item wdt:P225 "{scientific_name}" .
|
||||
|
||||
OPTIONAL {{
|
||||
?item p:P2067 ?massStmt .
|
||||
?massStmt psv:P2067 ?massVal .
|
||||
?massVal wikibase:quantityAmount ?mass .
|
||||
?massVal wikibase:quantityUnit ?massUnit .
|
||||
}}
|
||||
OPTIONAL {{
|
||||
?item p:P2250 ?lifespanStmt .
|
||||
?lifespanStmt psv:P2250 ?lifespanVal .
|
||||
?lifespanVal wikibase:quantityAmount ?lifespan .
|
||||
?lifespanVal wikibase:quantityUnit ?lifespanUnit .
|
||||
}}
|
||||
OPTIONAL {{ ?item wdt:P1034 ?diet . }}
|
||||
OPTIONAL {{ ?item wdt:P2974 ?habitat . }}
|
||||
OPTIONAL {{ ?item wdt:P5204 ?activity . }}
|
||||
|
||||
SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en" . }}
|
||||
}}
|
||||
LIMIT 5
|
||||
"""
|
||||
|
||||
resp = requests.get(SPARQL_URL, params={"query": query, "format": "json"}, headers=HEADERS)
|
||||
resp.raise_for_status()
|
||||
bindings = resp.json()["results"]["bindings"]
|
||||
|
||||
if not bindings:
|
||||
print(f" No Wikidata results for '{scientific_name}'")
|
||||
return {}
|
||||
|
||||
return _extract_traits(bindings)
|
||||
|
||||
|
||||
def _extract_traits(bindings: list[dict]) -> dict:
|
||||
result = {
|
||||
"size_range": None,
|
||||
"weight_range": None,
|
||||
"lifespan": None,
|
||||
"diet": None,
|
||||
"activity_period": None,
|
||||
"habitat_type": None,
|
||||
}
|
||||
|
||||
diets, habitats, activities = set(), set(), set()
|
||||
masses, lifespans = [], []
|
||||
|
||||
for row in bindings:
|
||||
if "mass" in row and "massUnitLabel" in row:
|
||||
mass = float(row["mass"]["value"])
|
||||
unit = row["massUnitLabel"]["value"]
|
||||
masses.append(f"{mass:g} {unit}")
|
||||
|
||||
if "lifespan" in row and "lifespanUnitLabel" in row:
|
||||
ls = float(row["lifespan"]["value"])
|
||||
unit = row["lifespanUnitLabel"]["value"]
|
||||
lifespans.append(f"{ls:g} {unit}")
|
||||
|
||||
if "dietLabel" in row:
|
||||
diets.add(row["dietLabel"]["value"])
|
||||
|
||||
if "habitatLabel" in row:
|
||||
habitats.add(row["habitatLabel"]["value"])
|
||||
|
||||
if "activityLabel" in row:
|
||||
activities.add(row["activityLabel"]["value"])
|
||||
|
||||
if masses:
|
||||
result["weight_range"] = " / ".join(sorted(set(masses)))
|
||||
if lifespans:
|
||||
result["lifespan"] = " / ".join(sorted(set(lifespans)))
|
||||
if diets:
|
||||
result["diet"] = ", ".join(sorted(diets))
|
||||
if habitats:
|
||||
result["habitat_type"] = ", ".join(sorted(habitats))
|
||||
if activities:
|
||||
result["activity_period"] = ", ".join(sorted(activities))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_species = "Thamnophis sirtalis"
|
||||
print(f"Fetching Wikidata traits for '{test_species}'...")
|
||||
traits = fetch_wikidata_traits(test_species)
|
||||
for key, value in traits.items():
|
||||
print(f" {key}: {value}")
|
||||
@@ -0,0 +1,230 @@
|
||||
"""
|
||||
Populates the species_detail table from static trait CSV downloads.
|
||||
|
||||
Required files in wildlife/trait_data/:
|
||||
- PanTHERIA_1-0_WR05_Aug2008.txt (mammals) https://ecologicaldata.org/wiki/pantheria
|
||||
- EltonTraits_BirdFuncDat.txt (birds) https://figshare.com/articles/dataset/EltonTraits_1_0/949799
|
||||
- EltonTraits_MamFuncDat.txt (mammals) same Figshare link above
|
||||
- AmphiBIO_v1.csv (amphibians) https://figshare.com/articles/dataset/AmphiBIO_v1/4644424
|
||||
|
||||
Place all files in: wildlife/trait_data/
|
||||
"""
|
||||
|
||||
import csv
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
DB_PATH = Path(__file__).parent.parent / "wilddex.db"
|
||||
TRAIT_DATA_DIR = Path(__file__).parent.parent / "wildlife" / "trait_data"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_species_map(conn: sqlite3.Connection) -> dict[str, int]:
|
||||
"""Return scientific_name -> species_id for all rows in the species table."""
|
||||
rows = conn.execute("SELECT species_id, scientific_name FROM species").fetchall()
|
||||
return {sci: sid for sid, sci in rows}
|
||||
|
||||
|
||||
def upsert_detail(conn: sqlite3.Connection, species_id: int, traits: dict) -> None:
|
||||
"""Insert or update a species_detail row, skipping None values."""
|
||||
traits = {k: v for k, v in traits.items() if v is not None}
|
||||
if not traits:
|
||||
return
|
||||
|
||||
existing = conn.execute(
|
||||
"SELECT detail_id FROM species_detail WHERE species_id = ?", (species_id,)
|
||||
).fetchone()
|
||||
|
||||
if existing:
|
||||
sets = ", ".join(f"{col} = ?" for col in traits)
|
||||
conn.execute(
|
||||
f"UPDATE species_detail SET {sets} WHERE species_id = ?",
|
||||
(*traits.values(), species_id),
|
||||
)
|
||||
else:
|
||||
cols = "species_id, " + ", ".join(traits.keys())
|
||||
placeholders = ", ".join("?" * (len(traits) + 1))
|
||||
conn.execute(
|
||||
f"INSERT INTO species_detail ({cols}) VALUES ({placeholders})",
|
||||
(species_id, *traits.values()),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PanTHERIA — mammals
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def import_pantheria(conn: sqlite3.Connection, species_map: dict[str, int]) -> None:
|
||||
path = TRAIT_DATA_DIR / "PanTHERIA_1-0_WR05_Aug2008.txt"
|
||||
if not path.exists():
|
||||
print(f" Skipping PanTHERIA (file not found: {path})")
|
||||
return
|
||||
|
||||
matched = 0
|
||||
with open(path, newline="", encoding="utf-8") as f:
|
||||
reader = csv.DictReader(f, delimiter="\t")
|
||||
for row in reader:
|
||||
sci_name = row.get("MSW05_Binomial", "").strip()
|
||||
species_id = species_map.get(sci_name)
|
||||
if not species_id:
|
||||
continue
|
||||
|
||||
mass_g = _float(row.get("5-1_AdultBodyMass_g"))
|
||||
longevity_m = _float(row.get("17-1_MaxLongevity_m"))
|
||||
|
||||
traits = {
|
||||
"weight_range": f"{mass_g:g} g" if mass_g else None,
|
||||
"lifespan": f"{longevity_m / 12:.1f} years" if longevity_m else None,
|
||||
}
|
||||
upsert_detail(conn, species_id, traits)
|
||||
matched += 1
|
||||
|
||||
conn.commit()
|
||||
print(f" PanTHERIA: {matched} species matched")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# EltonTraits — birds and mammals
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ELTON_DIET_COLS = [
|
||||
("Diet.Inv", "invertebrates"),
|
||||
("Diet.Vend", "vertebrates (endotherm)"),
|
||||
("Diet.Vect", "vertebrates (ectotherm)"),
|
||||
("Diet.Vfish", "fish"),
|
||||
("Diet.Vunk", "vertebrates (unknown)"),
|
||||
("Diet.Scav", "scavenging"),
|
||||
("Diet.Fruit", "fruit"),
|
||||
("Diet.Nect", "nectar"),
|
||||
("Diet.Seed", "seeds"),
|
||||
("Diet.PlantO","plant matter"),
|
||||
]
|
||||
|
||||
def _elton_diet(row: dict) -> str | None:
|
||||
parts = []
|
||||
for col, label in ELTON_DIET_COLS:
|
||||
val = _float(row.get(col))
|
||||
if val and val > 10: # only include if >10% of diet
|
||||
parts.append(f"{label} ({val:.0f}%)")
|
||||
return ", ".join(parts) if parts else None
|
||||
|
||||
def _elton_activity(row: dict) -> str | None:
|
||||
val = _float(row.get("Nocturnal"))
|
||||
if val is None:
|
||||
return None
|
||||
return "nocturnal" if val == 1 else "diurnal"
|
||||
|
||||
def _import_elton_file(conn: sqlite3.Connection, species_map: dict[str, int], path: Path, sci_col: str) -> int:
|
||||
matched = 0
|
||||
with open(path, newline="", encoding="latin-1") as f:
|
||||
reader = csv.DictReader(f, delimiter="\t")
|
||||
for row in reader:
|
||||
sci_name = row.get(sci_col, "").strip().replace("_", " ")
|
||||
species_id = species_map.get(sci_name)
|
||||
if not species_id:
|
||||
continue
|
||||
|
||||
mass_g = _float(row.get("BodyMass.Value"))
|
||||
traits = {
|
||||
"weight_range": f"{mass_g:g} g" if mass_g else None,
|
||||
"diet": _elton_diet(row),
|
||||
"activity_period": _elton_activity(row),
|
||||
}
|
||||
upsert_detail(conn, species_id, traits)
|
||||
matched += 1
|
||||
return matched
|
||||
|
||||
def import_elton(conn: sqlite3.Connection, species_map: dict[str, int]) -> None:
|
||||
bird_path = TRAIT_DATA_DIR / "EltonTraits_BirdFuncDat.txt"
|
||||
mam_path = TRAIT_DATA_DIR / "EltonTraits_MamFuncDat.txt"
|
||||
|
||||
total = 0
|
||||
if bird_path.exists():
|
||||
n = _import_elton_file(conn, species_map, bird_path, sci_col="Scientific")
|
||||
total += n
|
||||
print(f" EltonTraits birds: {n} species matched")
|
||||
else:
|
||||
print(f" Skipping EltonTraits birds (file not found)")
|
||||
|
||||
if mam_path.exists():
|
||||
n = _import_elton_file(conn, species_map, mam_path, sci_col="Scientific")
|
||||
total += n
|
||||
print(f" EltonTraits mammals: {n} species matched")
|
||||
else:
|
||||
print(f" Skipping EltonTraits mammals (file not found)")
|
||||
|
||||
conn.commit()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AmphiBIO — amphibians
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def import_amphibio(conn: sqlite3.Connection, species_map: dict[str, int]) -> None:
|
||||
path = TRAIT_DATA_DIR / "AmphiBIO_v1.csv"
|
||||
if not path.exists():
|
||||
print(f" Skipping AmphiBIO (file not found: {path})")
|
||||
return
|
||||
|
||||
matched = 0
|
||||
with open(path, newline="", encoding="latin-1") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
sci_name = row.get("Species", "").strip()
|
||||
species_id = species_map.get(sci_name)
|
||||
if not species_id:
|
||||
continue
|
||||
|
||||
size_min = _float(row.get("Body_size_mm"))
|
||||
longevity = _float(row.get("Longevity_max_y"))
|
||||
diet = row.get("Prey") or None
|
||||
habitat = row.get("Habitat") or None
|
||||
activity = row.get("Diel_activity") or None
|
||||
|
||||
traits = {
|
||||
"size_range": f"{size_min:g} mm" if size_min else None,
|
||||
"lifespan": f"{longevity:g} years" if longevity else None,
|
||||
"diet": diet,
|
||||
"habitat_type": habitat,
|
||||
"activity_period": activity,
|
||||
}
|
||||
upsert_detail(conn, species_id, traits)
|
||||
matched += 1
|
||||
|
||||
conn.commit()
|
||||
print(f" AmphiBIO: {matched} species matched")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Utilities
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _float(val) -> float | None:
|
||||
try:
|
||||
f = float(val)
|
||||
return None if f < 0 else f # datasets use -999 for missing
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not TRAIT_DATA_DIR.exists():
|
||||
TRAIT_DATA_DIR.mkdir(parents=True)
|
||||
print(f"Created {TRAIT_DATA_DIR} — place trait CSVs here and re-run.")
|
||||
else:
|
||||
with sqlite3.connect(DB_PATH) as conn:
|
||||
species_map = get_species_map(conn)
|
||||
print(f"Loaded {len(species_map)} species from DB\n")
|
||||
|
||||
import_pantheria(conn, species_map)
|
||||
import_elton(conn, species_map)
|
||||
import_amphibio(conn, species_map)
|
||||
|
||||
print("\nDone.")
|
||||
Reference in new issue
Block a user