#!/usr/bin/env python3
"""Search every chain term independently on the Korean public web.

This is deliberately separate from the five known supplier catalog crawl. It
uses Naver's general web index for every canonical name and alias, keeps only
results whose visible anchor text contains the searched term, and records the
direct page/domain for later human terminology review.
"""

from __future__ import annotations

import json
import re
import time
from collections import defaultdict
from pathlib import Path
from urllib.parse import quote, urlsplit

import requests
from bs4 import BeautifulSoup


HERE = Path(__file__).resolve().parent
INPUT = HERE / "candidates-v0.json"
OUTPUT = HERE / "open-web-term-evidence-2026-08-21.json"
CACHE = HERE / "open-web-search-cache-2026-08-21.json"
HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
        "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
    )
}
EXCLUDED_HOSTS = {
    "search.naver.com",
    "search.shopping.naver.com",
    "cr3.shopping.naver.com",
    "ader.naver.com",
    "m.search.naver.com",
    "nid.naver.com",
    "help.naver.com",
}


def normalize(value: str) -> str:
    value = value.lower().replace("커팅", "컷팅").replace("비형", "b형")
    return re.sub(r"[^0-9a-z가-힣]+", "", value)


def clean(value: str) -> str:
    return " ".join(value.split())


def host_for(url: str) -> str:
    host = urlsplit(url).netloc.lower().split(":")[0]
    return host.removeprefix("www.").removeprefix("m.")


def direct_result_anchors(html_text: str, searched_term: str, accepted_terms: list[str]) -> list[dict]:
    soup = BeautifulSoup(html_text, "html.parser")
    needles = [normalize(term) for term in accepted_terms]
    results = {}
    for anchor in soup.select("a[href]"):
        title = clean(anchor.get_text(" ", strip=True))
        url = anchor.get("href", "")
        if not title or not url.startswith(("http://", "https://")):
            continue
        host = host_for(url)
        if not host or host in EXCLUDED_HOSTS or host.endswith("search.naver.com"):
            continue
        if not any(needle in normalize(title) for needle in needles):
            continue
        key = (host, url)
        matched = next(term for term in accepted_terms if normalize(term) in normalize(title))
        results[key] = {
            "title": title,
            "url": url,
            "domain": host,
            "matched_term": matched,
            "searched_term": searched_term,
        }
    return list(results.values())


def load_cache() -> dict[str, str]:
    if CACHE.exists():
        return json.loads(CACHE.read_text(encoding="utf-8"))
    return {}


def save_cache(cache: dict[str, str]) -> None:
    CACHE.write_text(json.dumps(cache, ensure_ascii=False), encoding="utf-8")


def search(
    session: requests.Session,
    cache: dict[str, str],
    term: str,
    accepted_terms: list[str],
) -> list[dict]:
    if term not in cache:
        response = None
        for where in ("web", "nexearch"):
            url = f"https://search.naver.com/search.naver?where={where}&query={quote(term)}"
            response = session.get(url, timeout=30)
            if response.ok:
                break
            time.sleep(1.5)
        assert response is not None
        response.raise_for_status()
        cache[term] = response.text
        save_cache(cache)
        time.sleep(0.45)
    return direct_result_anchors(cache[term], term, accepted_terms)


def main() -> None:
    source = json.loads(INPUT.read_text(encoding="utf-8"))
    candidates = [item for group in source["candidate_groups"] for item in group["items"]]
    cache = load_cache()
    rows = []
    with requests.Session() as session:
        session.headers.update(HEADERS)
        for index, item in enumerate(candidates, start=1):
            terms = list(dict.fromkeys([item["name"], *item.get("aliases", [])]))
            evidence = []
            for term in terms:
                evidence.extend(search(session, cache, term, terms))
            deduped = {}
            for result in evidence:
                key = (result["domain"], result["url"])
                if key not in deduped:
                    deduped[key] = result
                elif result["matched_term"] not in deduped[key]["matched_term"]:
                    deduped[key]["matched_term"] += f" / {result['matched_term']}"
            grouped = defaultdict(list)
            for result in deduped.values():
                grouped[result["domain"]].append(result)
            representatives = []
            for domain in sorted(grouped):
                representatives.append(
                    sorted(grouped[domain], key=lambda result: (len(result["title"]), result["title"]))[0]
                )
            rows.append(
                {
                    "index": index,
                    "name": item["name"],
                    "aliases": item.get("aliases", []),
                    "searched_terms": terms,
                    "independent_domains": len(representatives),
                    "representative_results": representatives,
                    "all_result_count": len(deduped),
                }
            )
            print(f"{index:02d}/77 {item['name']}: {len(representatives)} domains")
    OUTPUT.write_text(
        json.dumps(
            {
                "checked_on": "2026-08-21",
                "method": "Every canonical chain name and alias independently searched in Naver's general web index; exact visible-term matches retained",
                "rows": rows,
            },
            ensure_ascii=False,
            indent=2,
        ),
        encoding="utf-8",
    )
    print(f"wrote {OUTPUT}")


if __name__ == "__main__":
    main()