Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 36 additions & 13 deletions src/applypilot/discovery/jobspy.py
Original file line number Diff line number Diff line change
Expand Up @@ -441,25 +441,48 @@ def _full_crawl(
# -- Public entry point ------------------------------------------------------

def run_discovery(cfg: dict | None = None) -> dict:
"""Main entry point for JobSpy-based job discovery.

Loads search queries and locations from the user's search config YAML,
then runs a full crawl across all configured job boards.

Args:
cfg: Override the search configuration dict. If None, loads from
the user's searches.yaml file.

Returns:
Dict with stats: new, existing, errors, db_total, queries.
"""
if cfg is None:
cfg = config.load_search_config()

if not cfg:
log.warning("No search configuration found. Run `applypilot init` to create one.")
return {"new": 0, "existing": 0, "errors": 0, "db_total": 0, "queries": 0}

# Support new flat `searches:` format
if "searches" in cfg:
searches_list = cfg["searches"]
proxy = cfg.get("proxy")
proxy_config = parse_proxy(proxy) if proxy else None
init_db()
total_new = total_existing = total_errors = 0

for s in searches_list:
site_names = s.get("site_name", ["linkedin", "indeed"])
# Map searches format to _run_one_search format
search = {
"query": s["search_term"],
"location": s.get("location", ""),
"remote": s.get("is_remote", False),
}
defaults = {"country_indeed": "switzerland"}
result = _run_one_search(
search, site_names,
s.get("results_wanted", 20),
cfg.get("defaults", {}).get("hours_old", 72),
proxy_config, defaults, 2, ["switzerland", "zurich", "zug", "basel", "bern", "st. gallen", "lausanne", "winterthur"], [], {},
)
total_new += result["new"]
total_existing += result["existing"]
total_errors += result["errors"]

conn = get_connection()
db_total = conn.execute("SELECT COUNT(*) FROM jobs").fetchone()[0]
log.info("Full crawl complete: %d new | %d dupes | %d errors | %d total in DB",
total_new, total_existing, total_errors, db_total)
return {"new": total_new, "existing": total_existing,
"errors": total_errors, "db_total": db_total}

# Legacy format fallback
proxy = cfg.get("proxy")
sites = cfg.get("sites")
results_per_site = cfg.get("defaults", {}).get("results_per_site", 100)
Expand All @@ -475,4 +498,4 @@ def run_discovery(cfg: dict | None = None) -> dict:
results_per_site=results_per_site,
hours_old=hours_old,
proxy=proxy,
)
)