"""
help_processor.py
=================
Обработва help-файлове (.doc, .docx, .html, .htm, .txt, .pdf),
декомпозира ги на смислови секции, извлича ключови думи чрез Anthropic API
и записва резултатите в SQL Server + изходна директория.

Поддържа инкрементална обработка: файлове, чийто hash не се е променил,
се прескачат при повторно пускане.

Изисквания (pip install):
    pip install anthropic pyodbc python-docx beautifulsoup4 lxml
    pip install pdfplumber striprtf chardet
    pip install pywin32           # за MS Word fallback на Windows

За .doc (стар формат) е необходим един от:
    - LibreOffice (soffice в PATH) — кросплатформено
    - MS Word — Windows, чрез pywin32 COM (автоматичен fallback)
    - antiword — Linux (apt install antiword)
"""

import os
import re
import sys
import json
import hashlib
import logging
import argparse
import subprocess
import tempfile
from pathlib import Path
from datetime import datetime
from dataclasses import dataclass, field
from typing import Optional

import psycopg2
import anthropic
from docx import Document
from bs4 import BeautifulSoup

from help_codes import (
    WIPED_HASH,
    file_result as _file_result,
    make_code,
    parse_code,
    remove_section_outputs,
    source_basename,
    source_identity,
)

try:
    import pdfplumber
    HAS_PDF = True
except ImportError:
    HAS_PDF = False

try:
    from PIL import Image
    HAS_PIL = True
except ImportError:
    HAS_PIL = False

# ──────────────────────────────────────────────
# Конфигурация
# ──────────────────────────────────────────────

# На Windows конзолата често е cp1251 → пренастройваме stdout на utf-8
try:
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
except AttributeError:
    pass

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s  %(levelname)-8s  %(message)s",
    handlers=[
        logging.StreamHandler(sys.stdout),
        logging.FileHandler("help_processor.log", encoding="utf-8"),
    ],
)
log = logging.getLogger(__name__)

MIN_SECTION_TOKENS = 60        # секции под тази граница се сливат с предишната
MAX_AI_CHARS      = 4000       # максимален текст, изпращан към Claude за класификация
AI_MODEL          = "claude-haiku-4-5"
MIN_IMAGE_PX      = 50         # картинки под NxN px се пропускат (иконки/булети)


# ──────────────────────────────────────────────
# Изображения — помощни
# ──────────────────────────────────────────────

@dataclass
class ImageRef:
    placeholder: str        # вътрешен ID в текста, напр. "img_01"
    data: bytes
    ext: str                # "png", "jpg", "gif"...


def _img_dimensions(data: bytes) -> Optional[tuple[int, int]]:
    if not HAS_PIL:
        return None
    try:
        from io import BytesIO
        with Image.open(BytesIO(data)) as im:
            return im.size
    except Exception:
        return None


def _should_keep_image(data: bytes) -> bool:
    """Връща False за дребни иконки/булети под MIN_IMAGE_PX × MIN_IMAGE_PX."""
    if not data:
        return False
    dims = _img_dimensions(data)
    if dims is None:
        # Не можем да преценим — пазим по подразбиране
        return True
    w, h = dims
    return w >= MIN_IMAGE_PX and h >= MIN_IMAGE_PX


def _ext_from_content_type(ct: str) -> str:
    ct = (ct or "").lower()
    if "png"  in ct: return "png"
    if "jpeg" in ct or "jpg" in ct: return "jpg"
    if "gif"  in ct: return "gif"
    if "bmp"  in ct: return "bmp"
    if "svg"  in ct: return "svg"
    if "webp" in ct: return "webp"
    return "png"


_IMG_PLACEHOLDER_RE = re.compile(r"\[IMG:\s*([A-Za-z0-9_./\\-]+)\s*\]")


# ──────────────────────────────────────────────
# Структури
# ──────────────────────────────────────────────

@dataclass
class Section:
    title: str
    text: str
    level: int = 1          # 1=H1, 2=H2, 3=H3, 0=без заглавие
    images: list = field(default_factory=list)   # list[ImageRef]
    html_text: Optional[str] = None              # rich HTML с [IMG: ...] placeholders


@dataclass
class ProcessedSection:
    code: str               # DOC_003_SEC_012
    source_file: str
    title: str
    keywords: str           # "кл1, кл2, кл3"
    text: str
    images_json: str = "[]"        # JSON масив с относителни пътища
    html_text: str   = ""          # rich HTML (само за HTML-source файлове)
    char_count: int = 0

    def __post_init__(self):
        self.char_count = len(self.text)


# ──────────────────────────────────────────────
# База данни
# ──────────────────────────────────────────────

class Database:
    """PostgreSQL backend (psycopg2). Connection string е libpq формат:
       'host=... port=... dbname=... user=... password=...'
    """
    def __init__(self, conn_str: str):
        self.conn_str = conn_str
        self.conn = psycopg2.connect(conn_str)
        self._ensure_schema()

    def _ensure_schema(self):
        """Създава таблиците ако не съществуват (Postgres syntax)."""
        cur = self.conn.cursor()
        cur.execute("""
        CREATE TABLE IF NOT EXISTS rip_help_files (
            id            SERIAL PRIMARY KEY,
            prefix        VARCHAR(50)    NOT NULL DEFAULT 'HLP',
            file_path     VARCHAR(1000)  NOT NULL,
            file_hash     CHAR(64)       NOT NULL,
            processed_at  TIMESTAMP      NOT NULL DEFAULT NOW(),
            section_count INTEGER        NOT NULL DEFAULT 0,
            UNIQUE (prefix, file_path)
        )
        """)
        cur.execute("""
        CREATE TABLE IF NOT EXISTS rip_help_sections (
            id           SERIAL PRIMARY KEY,
            prefix       VARCHAR(50)    NOT NULL DEFAULT 'HLP',
            code         VARCHAR(80)    NOT NULL UNIQUE,
            source_file  VARCHAR(1000)  NOT NULL,
            title        VARCHAR(500),
            keywords     VARCHAR(300),
            char_count   INTEGER,
            output_path  VARCHAR(1000),
            images       TEXT,
            html_text    TEXT,
            created_at   TIMESTAMP      NOT NULL DEFAULT NOW(),
            updated_at   TIMESTAMP      NOT NULL DEFAULT NOW()
        )
        """)
        cur.execute("""
        CREATE INDEX IF NOT EXISTS ix_rip_help_sections_keywords
            ON rip_help_sections(keywords)
        """)
        cur.execute("""
        CREATE INDEX IF NOT EXISTS ix_rip_help_sections_prefix
            ON rip_help_sections(prefix)
        """)
        cur.execute("""
        CREATE INDEX IF NOT EXISTS ix_rip_help_sections_source
            ON rip_help_sections(prefix, source_file)
        """)
        cur.execute(
            "ALTER TABLE rip_help_files ADD COLUMN IF NOT EXISTS file_index INTEGER"
        )
        self.conn.commit()
        log.info("Схемата е проверена / създадена.")

    def matching_source_paths(self, prefix: str, identity: str) -> list[str]:
        """Всички file_path/source_file за същия help файл (basename, вкл. стари temp пътища)."""
        name = source_basename(identity).lower()
        if not name:
            return []
        found: list[str] = []
        seen: set[str] = set()
        for p in self.all_source_files(prefix):
            if source_basename(p).lower() == name and p not in seen:
                seen.add(p)
                found.append(p)
        return found

    def get_file_hash(self, prefix: str, file_path: str) -> Optional[str]:
        paths = self.matching_source_paths(prefix, file_path) or [file_path]
        cur = self.conn.cursor()
        cur.execute(
            "SELECT file_hash FROM rip_help_files "
            "WHERE prefix=%s AND file_path = ANY(%s)",
            (prefix, list(paths)),
        )
        for (h,) in cur.fetchall():
            if not h:
                continue
            val = str(h).strip()
            if val and val != WIPED_HASH:
                return val
        return None

    def upsert_file(
        self,
        prefix: str,
        file_path: str,
        file_hash: str,
        section_count: int,
        file_index: Optional[int] = None,
    ):
        canonical = source_basename(file_path) or file_path
        stale = [p for p in self.matching_source_paths(prefix, canonical) if p != canonical]
        cur = self.conn.cursor()
        if stale:
            cur.execute(
                "DELETE FROM rip_help_files WHERE prefix=%s AND file_path = ANY(%s)",
                (prefix, stale),
            )
        cur.execute("""
        INSERT INTO rip_help_files (prefix, file_path, file_hash, section_count, file_index)
        VALUES (%s, %s, %s, %s, %s)
        ON CONFLICT (prefix, file_path) DO UPDATE SET
            file_hash    = EXCLUDED.file_hash,
            section_count= EXCLUDED.section_count,
            processed_at = NOW(),
            file_index   = COALESCE(EXCLUDED.file_index, rip_help_files.file_index)
        """, (prefix, canonical, file_hash, section_count, file_index))
        self.conn.commit()

    def delete_sections_for_file(self, prefix: str, file_path: str):
        paths = self.matching_source_paths(prefix, file_path) or [file_path]
        cur = self.conn.cursor()
        cur.execute(
            "DELETE FROM rip_help_sections WHERE prefix=%s AND source_file = ANY(%s)",
            (prefix, list(paths)),
        )
        self.conn.commit()

    def sections_for_file(self, prefix: str, identity: str) -> list[tuple[str, str, Optional[str]]]:
        """(code, source_file, output_path) за всички извлечения на файла."""
        paths = self.matching_source_paths(prefix, identity)
        if not paths:
            return []
        cur = self.conn.cursor()
        cur.execute(
            "SELECT code, source_file, output_path FROM rip_help_sections "
            "WHERE prefix=%s AND source_file = ANY(%s) ORDER BY code",
            (prefix, list(paths)),
        )
        return [(r[0], r[1], r[2]) for r in cur.fetchall()]

    def file_index_for(self, prefix: str, identity: str) -> Optional[int]:
        paths = self.matching_source_paths(prefix, identity)
        if not paths:
            return None
        cur = self.conn.cursor()
        cur.execute(
            "SELECT file_index FROM rip_help_files "
            "WHERE prefix=%s AND file_path = ANY(%s) AND file_index IS NOT NULL",
            (prefix, list(paths)),
        )
        from_col = [r[0] for r in cur.fetchall() if r[0]]
        if from_col:
            return min(from_col)
        cur.execute(
            "SELECT code FROM rip_help_sections WHERE prefix=%s AND source_file = ANY(%s)",
            (prefix, list(paths)),
        )
        found: list[int] = []
        for (code,) in cur.fetchall():
            parsed = parse_code(code)
            if parsed:
                found.append(parsed[1])
        if not found:
            return None
        tally: dict[int, int] = {}
        for idx in found:
            tally[idx] = tally.get(idx, 0) + 1
        return max(tally, key=lambda k: (tally[k], -k))

    def max_file_index(self, prefix: str) -> int:
        cur = self.conn.cursor()
        cur.execute(
            "SELECT COALESCE(MAX(file_index), 0) FROM rip_help_files WHERE prefix=%s",
            (prefix,),
        )
        m1 = int(cur.fetchone()[0] or 0)
        cur.execute("SELECT code FROM rip_help_sections WHERE prefix=%s", (prefix,))
        m2 = 0
        for (code,) in cur.fetchall():
            parsed = parse_code(code)
            if parsed:
                m2 = max(m2, parsed[1])
        return max(m1, m2)

    def file_index_used_by_others(self, prefix: str, identity: str, idx: int) -> bool:
        name = source_basename(identity).lower()
        cur = self.conn.cursor()
        cur.execute(
            "SELECT code, source_file FROM rip_help_sections WHERE prefix=%s",
            (prefix,),
        )
        for code, src in cur.fetchall():
            parsed = parse_code(code)
            if parsed and parsed[1] == idx and source_basename(src).lower() != name:
                return True
        return False

    def allocate_file_index(self, prefix: str, identity: str) -> int:
        existing = self.file_index_for(prefix, identity)
        if existing and not self.file_index_used_by_others(prefix, identity, existing):
            return existing
        return self.max_file_index(prefix) + 1

    def wipe_extractions_for_file(
        self,
        prefix: str,
        identity: str,
        output_dir: Optional[Path] = None,
    ) -> dict:
        """Изтрива всички извлечения за файла. Запазва file_index; следващият scan почва от SEC_0001."""
        rows = self.sections_for_file(prefix, identity)
        codes = [r[0] for r in rows]
        file_index = self.file_index_for(prefix, identity)
        if file_index and self.file_index_used_by_others(prefix, identity, file_index):
            file_index = None
        paths = self.matching_source_paths(prefix, identity)
        if output_dir:
            remove_section_outputs(output_dir, codes, [r[2] for r in rows if r[2]])
        self.delete_sections_for_file(prefix, identity)
        canonical = source_basename(identity) or identity
        stale = list(paths) if paths else []
        cur = self.conn.cursor()
        if stale:
            cur.execute(
                "DELETE FROM rip_help_files WHERE prefix=%s AND file_path = ANY(%s)",
                (prefix, stale),
            )
        if file_index:
            cur.execute("""
            INSERT INTO rip_help_files (prefix, file_path, file_hash, section_count, file_index)
            VALUES (%s, %s, %s, 0, %s)
            ON CONFLICT (prefix, file_path) DO UPDATE SET
                file_hash     = EXCLUDED.file_hash,
                section_count = 0,
                processed_at  = NOW(),
                file_index    = EXCLUDED.file_index
            """, (prefix, canonical, WIPED_HASH, file_index))
        self.conn.commit()
        return {
            "file": canonical,
            "prefix": prefix,
            "deleted": len(codes),
            "codes": codes,
            "file_index": file_index,
        }

    def all_source_files(self, prefix: str) -> list[str]:
        """Връща всички source_file пътища за даден префикс."""
        cur = self.conn.cursor()
        cur.execute("""
            SELECT file_path FROM rip_help_files    WHERE prefix=%s
            UNION
            SELECT source_file FROM rip_help_sections WHERE prefix=%s
        """, (prefix, prefix))
        return [r[0] for r in cur.fetchall()]

    def section_output_paths_for(self, prefix: str, source_files: list[str]) -> list[str]:
        if not source_files:
            return []
        cur = self.conn.cursor()
        cur.execute(
            "SELECT output_path FROM rip_help_sections "
            "WHERE prefix=%s AND source_file = ANY(%s)",
            (prefix, list(source_files))
        )
        return [r[0] for r in cur.fetchall() if r[0]]

    def purge_sources(self, prefix: str, source_files: list[str]) -> int:
        if not source_files:
            return 0
        cur = self.conn.cursor()
        cur.execute(
            "DELETE FROM rip_help_sections "
            "WHERE prefix=%s AND source_file = ANY(%s)",
            (prefix, list(source_files))
        )
        sec_deleted = cur.rowcount
        cur.execute(
            "DELETE FROM rip_help_files "
            "WHERE prefix=%s AND file_path = ANY(%s)",
            (prefix, list(source_files))
        )
        self.conn.commit()
        return sec_deleted

    def insert_section(self, prefix: str, ps: ProcessedSection, output_path: str):
        cur = self.conn.cursor()
        cur.execute("""
        INSERT INTO rip_help_sections
            (prefix, code, source_file, title, keywords,
             char_count, output_path, images, html_text)
        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
        ON CONFLICT (code) DO UPDATE SET
            prefix      = EXCLUDED.prefix,
            source_file = EXCLUDED.source_file,
            title       = EXCLUDED.title,
            keywords    = EXCLUDED.keywords,
            char_count  = EXCLUDED.char_count,
            output_path = EXCLUDED.output_path,
            images      = EXCLUDED.images,
            html_text   = EXCLUDED.html_text,
            updated_at  = NOW()
        """, (prefix, ps.code, ps.source_file, ps.title, ps.keywords,
              ps.char_count, output_path, ps.images_json, ps.html_text))
        self.conn.commit()

    def close(self):
        self.conn.close()


# ──────────────────────────────────────────────
# Парсъри
# ──────────────────────────────────────────────

def file_hash(path: Path) -> str:
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(65536), b""):
            h.update(chunk)
    return h.hexdigest()


def _load_html_image(src: str, base_dir: Path) -> Optional[tuple[bytes, str]]:
    """Връща (data, ext) или None. Пропуска HTTP/HTTPS."""
    if not src:
        return None
    s = src.strip()
    if s.startswith("data:"):
        # data:image/png;base64,XXXX
        m = re.match(r"data:([^;]+);base64,(.+)$", s, re.DOTALL)
        if not m:
            return None
        import base64
        try:
            data = base64.b64decode(m.group(2))
        except Exception:
            return None
        return data, _ext_from_content_type(m.group(1))
    if s.startswith(("http://", "https://")):
        return None      # по правило пропускаме мрежови картинки
    # локален път, относителен или абсолютен
    p = (base_dir / s).resolve() if not Path(s).is_absolute() else Path(s)
    try:
        if p.is_file():
            data = p.read_bytes()
            ext  = p.suffix.lstrip(".").lower() or "png"
            return data, ext
    except Exception:
        return None
    return None


def _detect_html_encoding(raw: bytes) -> str:
    """Връща име на encoding: BOM → chardet → fallback (utf-8 ако ASCII, иначе windows-1251)."""
    # BOM-и
    if raw.startswith(b"\xef\xbb\xbf"):
        return "utf-8"
    if raw.startswith((b"\xff\xfe", b"\xfe\xff")):
        return "utf-16"
    # chardet
    try:
        import chardet
        det = chardet.detect(raw[:65536]) or {}
        enc = (det.get("encoding") or "").lower()
        conf = det.get("confidence", 0) or 0
        if enc and conf >= 0.6:
            # нормализиране на често срещани имена
            if enc in ("cp1251", "ms-cyrl", "windows-1251"):
                return "windows-1251"
            if enc.startswith("utf"):
                return enc
            return enc
    except Exception:
        pass
    # fallback: ако байтовете изглеждат "над 127" (т.е. има не-ASCII), приемаме CP1251
    if any(b > 127 for b in raw[:8192]):
        return "windows-1251"
    return "utf-8"


_HTML_BLOCK_TAGS  = ["h1", "h2", "h3", "h4", "h5", "h6",
                     "p", "ul", "ol", "table", "dl", "pre",
                     "blockquote", "figure", "hr"]
_HTML_PLAIN_NL_TAGS = frozenset({"ul", "ol", "table", "dl", "pre", "blockquote"})
_HTML_DROP_ATTRS  = ("class", "style", "id", "lang", "dir", "align",
                     "valign", "width", "height", "bgcolor", "border")


def _html_block_plain_text(el) -> str:
    """Plain ingest text: newlines inside lists/tables, spaces for inline runs."""
    sep = "\n" if (el.name or "").lower() in _HTML_PLAIN_NL_TAGS else " "
    return el.get_text(sep, strip=True)


def _strip_attrs(el):
    """Премахва decorative атрибути (class, style, on*, data-*)."""
    for t in el.find_all(True):
        for a in list(t.attrs):
            if a in _HTML_DROP_ATTRS or a.startswith("on") or a.startswith("data-"):
                del t[a]


def _swap_imgs_in_block(el, base_dir: Path, sec_images: list, img_counter: list) -> None:
    """Намира всички <img> в подадения елемент, извлича данните и подменя с
    NavigableString placeholder ([IMG: img_NN])."""
    from bs4 import NavigableString
    for img in el.find_all("img"):
        src = img.get("src") or img.get("data-src") or ""
        loaded = _load_html_image(src, base_dir)
        if not loaded:
            img.decompose()
            continue
        data, ext = loaded
        if not _should_keep_image(data):
            img.decompose()
            continue
        img_counter[0] += 1
        ref = ImageRef(placeholder=f"img_{img_counter[0]:02d}", data=data, ext=ext)
        sec_images.append(ref)
        img.replace_with(NavigableString(f"[IMG: {ref.placeholder}]"))


def parse_html(path: Path) -> list[Section]:
    raw = path.read_bytes()
    enc = _detect_html_encoding(raw)
    log.debug(f"    {path.name} encoding: {enc}")
    try:
        soup = BeautifulSoup(raw, "lxml", from_encoding=enc)
    except Exception:
        soup = BeautifulSoup(raw, "lxml")

    # Премахваме скриптове и стилове
    for tag in soup(["script", "style", "nav", "footer", "header", "noscript"]):
        tag.decompose()

    base_dir = path.parent
    body     = soup.body or soup

    heading_map = {"h1": 1, "h2": 2, "h3": 3, "h4": 3, "h5": 3, "h6": 3}

    # Събираме top-level блокови елементи (без да включваме вложените в тях)
    consumed = set()
    blocks   = []
    for el in body.find_all(_HTML_BLOCK_TAGS + ["img"]):
        if any(id(par) in consumed for par in el.parents):
            continue
        consumed.add(id(el))
        blocks.append(el)

    sections: list[Section]   = []
    current_title             = ""
    current_level             = 1
    sec_text:  list[str]      = []
    sec_html:  list[str]      = []
    sec_images: list[ImageRef] = []
    img_counter = [0]

    def flush():
        if sec_text or sec_html or sec_images:
            sec = Section(current_title, "\n".join(sec_text), current_level)
            sec.images    = list(sec_images)
            sec.html_text = "\n".join(sec_html) if sec_html else None
            sections.append(sec)

    for el in blocks:
        if el.name in heading_map:
            txt = el.get_text(" ", strip=True)
            if not txt:
                continue
            flush()
            current_title = txt
            current_level = heading_map[el.name]
            sec_text, sec_html, sec_images = [], [], []
            continue

        if el.name == "img":
            # самостоятелен <img> (не вътре в блок)
            _swap_imgs_in_block(el.parent if el.parent and el.parent.name else el,
                                base_dir, sec_images, img_counter)
            # ако е заменен с placeholder, добавяме като текст
            txt = el.get_text(" ", strip=True) if el.name else ""
            if txt:
                sec_text.append(txt)
                sec_html.append(f"<p>{txt}</p>")
            continue

        _swap_imgs_in_block(el, base_dir, sec_images, img_counter)
        _strip_attrs(el)
        txt = _html_block_plain_text(el)
        if txt:
            sec_text.append(txt)
        try:
            sec_html.append(str(el))
        except Exception:
            pass

    flush()

    if not sections:
        plain = body.get_text(" ", strip=True)
        return [Section("", plain, 0)]
    return sections


def _extract_docx_paragraph_images(para, doc) -> list[ImageRef]:
    """Намира drawing-и в параграф; връща ImageRef-и за филтрираните по размер."""
    from docx.oxml.ns import qn
    imgs: list[ImageRef] = []
    try:
        blips = para._element.findall(".//" + qn("a:blip"))
    except Exception:
        return imgs

    embed_attr = qn("r:embed")
    for blip in blips:
        rId = blip.get(embed_attr)
        if not rId:
            continue
        try:
            part = doc.part.related_parts[rId]
            data = part.blob
            ct   = getattr(part, "content_type", "") or ""
        except Exception:
            continue
        if not _should_keep_image(data):
            continue
        ext = _ext_from_content_type(ct)
        imgs.append(ImageRef(placeholder=f"__IMG_{len(imgs)+1}__", data=data, ext=ext))
    return imgs


def parse_docx(path: Path) -> list[Section]:
    doc = Document(path)
    sections: list[Section] = []
    current_title, current_level = "", 1
    buf: list[str] = []
    sec_images: list[ImageRef] = []
    img_counter = [0]   # списък за nonlocal-стил мутация

    HEADING_STYLES = {"heading 1": 1, "heading 2": 2, "heading 3": 3,
                      "title": 1, "subtitle": 2}

    def flush():
        if buf or sec_images:
            sec = Section(current_title, "\n".join(buf), current_level)
            sec.images = list(sec_images)
            sections.append(sec)

    for para in doc.paragraphs:
        style_name = para.style.name.lower() if para.style else ""
        text       = para.text.strip()
        para_imgs  = _extract_docx_paragraph_images(para, doc)

        if not text and not para_imgs:
            continue

        level = HEADING_STYLES.get(style_name)
        is_bold_heading = bool(text and len(text) < 120 and not style_name.startswith("list")
                               and para.runs
                               and all(run.bold for run in para.runs if run.text.strip()))

        if level or (is_bold_heading and not para_imgs):
            flush()
            buf, sec_images = [], []
            current_title = text
            current_level = level or 2
            continue

        if text:
            buf.append(text)
        for im in para_imgs:
            img_counter[0] += 1
            im.placeholder = f"img_{img_counter[0]:02d}"
            sec_images.append(im)
            buf.append(f"[IMG: {im.placeholder}]")

    flush()

    if not sections:
        fallback_text = "\n".join(p.text for p in doc.paragraphs if p.text.strip())
        return [Section("", fallback_text, 0)]
    return sections


def _convert_doc_with_libreoffice(path: Path, out_dir: Path) -> Optional[Path]:
    try:
        subprocess.run(
            ["soffice", "--headless", "--convert-to", "docx",
             "--outdir", str(out_dir), str(path)],
            check=True, capture_output=True, timeout=60
        )
    except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired) as e:
        log.debug(f"LibreOffice конверсия неуспешна: {e}")
        return None
    out = list(out_dir.glob("*.docx"))
    return out[0] if out else None


def _convert_doc_with_word(path: Path, out_dir: Path) -> Optional[Path]:
    """Fallback: ползва MS Word през COM на Windows."""
    try:
        import win32com.client       # noqa: F401
        import pythoncom
    except ImportError:
        log.debug("pywin32 не е инсталиран — MS Word fallback недостъпен.")
        return None

    import win32com.client as wcc
    pythoncom.CoInitialize()
    word = None
    doc  = None
    try:
        word = wcc.DispatchEx("Word.Application")
        word.Visible       = False
        word.DisplayAlerts = False
        doc = word.Documents.Open(str(path.resolve()), ReadOnly=True)
        out_path = out_dir / (path.stem + ".docx")
        # FileFormat=16 → wdFormatXMLDocument (.docx)
        doc.SaveAs2(str(out_path.resolve()), FileFormat=16)
        return out_path if out_path.exists() else None
    except Exception as e:
        log.debug(f"MS Word конверсия неуспешна: {e}")
        return None
    finally:
        try:
            if doc is not None:
                doc.Close(SaveChanges=False)
        except Exception:
            pass
        try:
            if word is not None:
                word.Quit()
        except Exception:
            pass
        pythoncom.CoUninitialize()


def parse_doc_old(path: Path) -> list[Section]:
    """Конвертира стар .doc до .docx чрез LibreOffice или MS Word, после парси."""
    with tempfile.TemporaryDirectory() as tmp:
        tmp_dir = Path(tmp)

        converted = _convert_doc_with_libreoffice(path, tmp_dir)
        engine    = "LibreOffice"

        if not converted:
            converted = _convert_doc_with_word(path, tmp_dir)
            engine    = "MS Word"

        if not converted:
            log.warning(
                f"Нито LibreOffice, нито MS Word успяха да конвертират {path.name}. "
                f"Пробваме като текст."
            )
            return parse_txt(path)

        log.info(f"    {path.name} конвертиран чрез {engine}")
        return parse_docx(converted)


def _render_pdf_image(page, img_info, resolution: int = 150) -> Optional[bytes]:
    """Кропва картинката от PDF страницата и я записва като PNG bytes."""
    try:
        x0 = float(img_info.get("x0", 0))
        x1 = float(img_info.get("x1", 0))
        top = float(img_info.get("top", img_info.get("y0", 0)))
        bot = float(img_info.get("bottom", img_info.get("y1", 0)))
        if x1 <= x0 or bot <= top:
            return None
        # ограничаваме до страницата (pdfplumber иначе хвърля)
        x0 = max(0, x0); top = max(0, top)
        x1 = min(page.width, x1); bot = min(page.height, bot)
        if x1 - x0 < 1 or bot - top < 1:
            return None
        cropped = page.crop((x0, top, x1, bot))
        pil = cropped.to_image(resolution=resolution).original
        from io import BytesIO
        buf = BytesIO()
        pil.save(buf, format="PNG")
        return buf.getvalue()
    except Exception as e:
        log.debug(f"PDF image render failed: {e}")
        return None


_PDF_LINE_Y_TOL = 3.0
# Typical word space in these PDFs is ~0.2–0.3em; only merge tighter (or drop-caps).
_PDF_LETTER_GAP_FRAC = 0.12
# Bare "1" / "2." / "3)" followed by a capital — TOC/list, not "виж 1 и 2".
_PDF_LIST_MARK_RE = re.compile(
    r"(?<!\d)(?<![A-Za-zА-Яа-яЁёІіЇїЄєҐґ])(\d{1,2})([.)])?(?=\s+[A-ZА-ЯЁІЇЄҐ])"
)


def _pdf_word_band(w) -> tuple[float, float]:
    top = float(w.get("top", 0))
    bot = float(w.get("bottom", top + float(w.get("size") or 10)))
    if bot <= top:
        bot = top + max(float(w.get("size") or 10), 1.0)
    return top, bot


def _pdf_cluster_words_by_y(words: list) -> list[dict]:
    """Visual lines by Y (and vertical overlap), not PDF stream order."""
    items = [w for w in words if (w.get("text") or "").strip()]
    items.sort(key=lambda w: (float(w.get("top", 0)), float(w.get("x0", 0))))
    lines: list[dict] = []
    for w in items:
        top, bot = _pdf_word_band(w)
        placed = False
        for line in reversed(lines):
            overlap = min(bot, line["bot"]) - max(top, line["top"])
            band = min(bot - top, line["bot"] - line["top"])
            close = abs(top - line["top"]) <= _PDF_LINE_Y_TOL
            if close or (band > 0 and overlap >= 0.35 * band):
                line["words"].append(w)
                line["top"] = min(line["top"], top)
                line["bot"] = max(line["bot"], bot)
                placed = True
                break
            # Sorted by top: once this word sits clearly below a line, earlier
            # lines are even higher.
            if top > line["bot"] + _PDF_LINE_Y_TOL:
                break
        if not placed:
            lines.append({"top": top, "bot": bot, "words": [w]})
    lines.sort(key=lambda ln: ln["top"])
    return lines


def _pdf_merge_split_letters(ws: list) -> list[dict]:
    """Join drop-cap / styled first letters: 'C' + 'orrelation' → 'Correlation'."""
    ordered = sorted(ws, key=lambda w: float(w.get("x0", 0)))
    clusters: list[dict] = []
    for w in ordered:
        text = (w.get("text") or "").strip()
        if not text:
            continue
        x0 = float(w.get("x0", 0))
        x1 = float(w.get("x1", x0))
        size = float(w.get("size") or 10)
        font = str(w.get("fontname") or "")
        if not clusters:
            clusters.append({"text": text, "x1": x1, "size": size, "font": font})
            continue
        prev = clusters[-1]
        gap = x0 - prev["x1"]
        ref = max(size, prev["size"], 1.0)
        fonts_differ = bool(prev["font"] and font and prev["font"] != font)
        sizes_differ = abs(size - prev["size"]) > 0.6
        drop = (
            len(prev["text"]) == 1
            and prev["text"].isalpha()
            and text[0].islower()
        )
        tiny = gap < _PDF_LETTER_GAP_FRAC * ref or gap < 0.8
        if tiny or (drop and gap < 0.55 * ref and (fonts_differ or sizes_differ or gap < 0.2 * ref)):
            prev["text"] += text
            prev["x1"] = max(prev["x1"], x1)
            prev["size"] = max(prev["size"], size)
            if font:
                prev["font"] = font
        else:
            clusters.append({"text": text, "x1": x1, "size": size, "font": font})
    return clusters


def _pdf_split_list_text(text: str) -> list[str]:
    """Break a flattened TOC/list: '... 1 Foo 2 Bar' → one item per line."""
    text = text.strip()
    if not text:
        return []
    matches = list(_PDF_LIST_MARK_RE.finditer(text))
    if len(matches) < 2:
        return [text]
    nums = [int(m.group(1)) for m in matches]
    if nums[0] not in (1, 2):
        return [text]
    if any(nums[i] != nums[0] + i for i in range(len(nums))):
        return [text]
    parts: list[str] = []
    last = 0
    for m in matches:
        if m.start() > last:
            head = text[last:m.start()].strip()
            if head:
                parts.append(head)
        last = m.start()
    tail = text[last:].strip()
    if tail:
        parts.append(tail)
    return parts or [text]


def _pdf_words_to_lines(words: list) -> list[dict]:
    """Group pdfplumber words into visual lines by Y, then left-to-right."""
    result = []
    for line in _pdf_cluster_words_by_y(words):
        ws = line["words"]
        tokens = _pdf_merge_split_letters(ws)
        if not tokens:
            continue
        joined = " ".join(t["text"] for t in tokens).strip()
        if not joined:
            continue
        size = round(float(ws[0].get("size", 10)), 1)
        for piece in _pdf_split_list_text(joined):
            result.append({"top": line["top"], "size": size, "text": piece})
    return result


def parse_pdf(path: Path) -> list[Section]:
    if not HAS_PDF:
        log.warning("pdfplumber не е инсталиран. PDF се прескача.")
        return []

    sections: list[Section] = []
    current_title = ""
    buf: list[str]            = []
    sec_images: list[ImageRef] = []
    img_counter = [0]
    prev_size = None

    def flush():
        if buf or sec_images:
            sec = Section(current_title, "\n".join(buf), 2)
            sec.images = list(sec_images)
            sections.append(sec)

    with pdfplumber.open(path) as pdf:
        for page in pdf.pages:
            # Картинките за страницата (сортирани по y отгоре надолу)
            page_images = sorted(
                page.images or [],
                key=lambda im: float(im.get("top", im.get("y0", 0)))
            )
            img_queue = []
            for im in page_images:
                data = _render_pdf_image(page, im)
                if not data or not _should_keep_image(data):
                    continue
                img_queue.append((float(im.get("top", 0)), data))

            words = page.extract_words(extra_attrs=["size", "fontname"])
            visual_lines = _pdf_words_to_lines(words)
            line_buf, line_size = [], None

            def emit_images_before(y: float):
                while img_queue and img_queue[0][0] <= y:
                    _, data = img_queue.pop(0)
                    img_counter[0] += 1
                    ref = ImageRef(placeholder=f"img_{img_counter[0]:02d}",
                                   data=data, ext="png")
                    sec_images.append(ref)
                    buf.append(f"[IMG: {ref.placeholder}]")

            for vl in visual_lines:
                sz = vl["size"]
                y  = vl["top"]
                if line_size is None:
                    line_size = sz
                if abs(sz - line_size) > 1:
                    line_text = "\n".join(line_buf).strip()
                    if line_text:
                        if line_size > (prev_size or 10) + 1 and len(line_text) < 150:
                            flush()
                            buf, sec_images = [], []
                            current_title = " ".join(line_text.split())
                        else:
                            emit_images_before(y)
                            buf.append(line_text)
                    prev_size = line_size
                    line_buf, line_size = [vl["text"]], sz
                else:
                    line_buf.append(vl["text"])

            if line_buf:
                emit_images_before(page.height)
                buf.append("\n".join(line_buf))

            # картинките след всичкия текст на страницата
            emit_images_before(page.height + 1)

    flush()

    return sections or [Section("", "", 0)]


def parse_txt(path: Path) -> list[Section]:
    import chardet
    raw = path.read_bytes()
    enc = chardet.detect(raw)["encoding"] or "utf-8"
    text = raw.decode(enc, errors="replace")
    return [Section("", text, 0)]


PARSERS = {
    ".html": parse_html,
    ".htm":  parse_html,
    ".docx": parse_docx,
    ".doc":  parse_doc_old,
    ".txt":  parse_txt,
    ".pdf":  parse_pdf,
}


# ──────────────────────────────────────────────
# Сегментиране и почистване
# ──────────────────────────────────────────────

def merge_short_sections(sections: list[Section]) -> list[Section]:
    """Слива секции, по-кратки от MIN_SECTION_TOKENS думи, с предишната."""
    result: list[Section] = []
    for sec in sections:
        words = len(sec.text.split())
        if result and words < MIN_SECTION_TOKENS:
            prev = result[-1]
            merged = Section(
                prev.title,
                prev.text + "\n" + sec.text,
                prev.level,
            )
            merged.images = (prev.images or []) + (sec.images or [])
            html_parts = [h for h in (prev.html_text, sec.html_text) if h]
            merged.html_text = "\n".join(html_parts) if html_parts else None
            result[-1] = merged
        else:
            result.append(sec)
    return result


def clean_text(text: str) -> str:
    """Collapse spaces/tabs but keep newlines (lists, paragraphs)."""
    text = text.replace("\r\n", "\n").replace("\r", "\n")
    text = re.sub(r"[^\S\n]+", " ", text)
    text = re.sub(r"\n{3,}", "\n\n", text)
    text = "\n".join(line.strip() for line in text.split("\n"))
    return text.strip()


# ──────────────────────────────────────────────
# AI класификация
# ──────────────────────────────────────────────

def classify_section(client: anthropic.Anthropic, title: str, text: str) -> tuple[str, str]:
    """Връща (наименование, 'кл1, кл2, кл3') чрез Claude."""
    snippet = text[:MAX_AI_CHARS]
    prompt = f"""Анализирай следната секция от help-документация и върни JSON обект с два ключа:
- "title": кратко наименование на секцията (до 8 думи, на езика на текста)
- "keywords": списък от до 5 ключови думи/фрази, разделени със запетая (на езика на текста)

Съществуващо заглавие (може да е празно): {title!r}

Текст:
{snippet}

Върни САМО валиден JSON без markdown, без коментари."""

    msg = client.messages.create(
        model=AI_MODEL,
        max_tokens=200,
        messages=[{"role": "user", "content": prompt}]
    )
    raw = msg.content[0].text.strip()
    raw = re.sub(r"^```[a-z]*\n?", "", raw)
    raw = re.sub(r"\n?```$", "", raw)

    try:
        data = json.loads(raw)
        t = str(data.get("title", title or "Секция"))[:200]
        k = str(data.get("keywords", ""))[:300]
        return t, k
    except json.JSONDecodeError:
        log.warning(f"AI върна невалиден JSON: {raw[:120]}")
        return title or "Секция", ""


# ──────────────────────────────────────────────
# Генериране на кодове — help_codes.py
# ──────────────────────────────────────────────


# ──────────────────────────────────────────────
# Основна обработка
# ──────────────────────────────────────────────

def _db_output_path(local_path: Path, code: str, remote_root: Optional[str]) -> str:
    """Локален staging файл; в БД — сървърен път ако е зададен remote_root."""
    if remote_root:
        root = remote_root.replace("\\", "/").rstrip("/")
        return f"{root}/{code}.txt"
    return str(local_path)


def process_file(
    path: Path,
    file_index: int,
    db: Database,
    client: anthropic.Anthropic,
    output_dir: Path,
    prefix: str = "HLP",
    force: bool = False,
    remote_root: Optional[str] = None,
    source_key: Optional[str] = None,
) -> dict:
    """Обработва един файл. Връща статистика (saved, codes, ok)."""
    rel = source_key or path.name
    fh = file_hash(path)

    existing_rows = db.sections_for_file(prefix, rel)
    existing_codes = [r[0] for r in existing_rows]

    if not force:
        stored = db.get_file_hash(prefix, rel)
        if stored == fh:
            log.info(f"  [SKIP] {path.name}  (непроменен)")
            return _file_result(rel, file_index, existing_codes, saved=0, skipped=True)

    log.info(f"  [PROC] {path.name}")
    ext = path.suffix.lower()
    parser = PARSERS.get(ext)
    if not parser:
        log.warning(f"    Неподдържан формат: {ext}")
        return _file_result(rel, file_index, existing_codes, saved=0)

    try:
        sections = parser(path)
    except Exception as e:
        log.error(f"    Грешка при парсване: {e}")
        return _file_result(rel, file_index, existing_codes, saved=0)

    sections = merge_short_sections(sections)

    remove_section_outputs(
        output_dir,
        existing_codes,
        [r[2] for r in existing_rows if r[2]],
    )
    db.delete_sections_for_file(prefix, rel)

    images_dir = output_dir / "images"
    images_dir.mkdir(parents=True, exist_ok=True)

    saved = 0
    codes: list[str] = []
    for sec in sections:
        text      = clean_text(sec.text)
        html_text = sec.html_text or ""
        if not text and not sec.images and not html_text:
            continue

        sec_index = saved + 1
        code = make_code(prefix, file_index, sec_index)

        # Записваме картинките на диск и заменяме placeholder-ите в текста + HTML
        image_rel_paths: list[str] = []
        for ref in sec.images or []:
            fname     = f"{code}_{ref.placeholder}.{ref.ext}"
            disk_path = images_dir / fname
            try:
                disk_path.write_bytes(ref.data)
            except Exception as e:
                log.warning(f"    Грешка при запис на картинка {fname}: {e}")
                continue
            rel_path = f"images/{fname}"
            image_rel_paths.append(rel_path)
            old_ph = f"[IMG: {ref.placeholder}]"
            new_ph = f"[IMG: {rel_path}]"
            text      = text.replace(old_ph, new_ph)
            html_text = html_text.replace(old_ph, new_ph)

        # Премахваме placeholder-и, останали без файл
        text = _IMG_PLACEHOLDER_RE.sub(
            lambda m: m.group(0) if "/" in m.group(1) or "\\" in m.group(1) else "",
            text
        ).strip()
        html_text = _IMG_PLACEHOLDER_RE.sub(
            lambda m: m.group(0) if "/" in m.group(1) or "\\" in m.group(1) else "",
            html_text
        ).strip()
        if not text and not image_rel_paths and not html_text:
            continue

        try:
            title, keywords = classify_section(client, sec.title, text)
        except Exception as e:
            log.warning(f"    AI грешка за {code}: {e}")
            title, keywords = sec.title or f"Секция {sec_index}", ""

        images_json = json.dumps(image_rel_paths, ensure_ascii=False)
        ps = ProcessedSection(
            code=code,
            source_file=rel,
            title=title,
            keywords=keywords,
            text=text,
            images_json=images_json,
            html_text=html_text,
        )

        # Локален staging; в БД — remote път (ако е зададен) за Coolify/API
        out_path = output_dir / f"{code}.txt"
        out_path.write_text(
            f"КОД: {code}\nФАЙЛ: {rel}\nЗАГЛАВИЕ: {title}\nКЛЮЧОВИ ДУМИ: {keywords}\n"
            f"КАРТИНКИ: {len(image_rel_paths)}\n"
            f"{'─'*60}\n{text}",
            encoding="utf-8"
        )

        db.insert_section(prefix, ps, _db_output_path(out_path, code, remote_root))
        saved += 1
        codes.append(code)
        log.debug(f"    {code}: {title[:60]}  ({len(image_rel_paths)} img)")

    db.upsert_file(prefix, rel, fh, saved, file_index=file_index)
    log.info(f"    → {saved} секции записани")
    return _file_result(rel, file_index, codes, saved=saved)


_PREFIX_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,49}$")


def process_directory(
    input_dir: Path,
    output_dir: Path,
    conn_str: str,
    api_key: str,
    prefix: str = "HLP",
    force: bool = False,
    purge_missing: bool = False,
    remote_root: Optional[str] = None,
):
    if not _PREFIX_RE.match(prefix):
        raise ValueError(
            f"Невалиден prefix {prefix!r}. Допустими: буква + букви/цифри/подчертавки, до 50 символа."
        )

    output_dir.mkdir(parents=True, exist_ok=True)
    db = Database(conn_str)
    client = anthropic.Anthropic(api_key=api_key)

    extensions = set(PARSERS.keys())
    output_resolved = output_dir.resolve()

    def _under_output(p: Path) -> bool:
        try:
            p.resolve().relative_to(output_resolved)
            return True
        except ValueError:
            return False

    files = [
        p for p in input_dir.rglob("*")
        if p.is_file() and p.suffix.lower() in extensions and not _under_output(p)
    ]
    log.info(f"Prefix={prefix}  Намерени {len(files)} файла в {input_dir}")
    if remote_root:
        log.info(f"DB output_path root: {remote_root.replace(chr(92), '/').rstrip('/')}")

    current_ids = {source_identity(p, input_dir) for p in files}
    current_names = {source_basename(i).lower() for i in current_ids}
    file_results: list[dict] = []
    total_sections = 0
    try:
        for path in sorted(files):
            identity = source_identity(path, input_dir)
            idx = db.allocate_file_index(prefix, identity)
            info = process_file(
                path, idx, db, client, output_dir,
                prefix=prefix, force=force, remote_root=remote_root,
                source_key=identity,
            )
            file_results.append(info)
            total_sections += int(info.get("saved") or 0)

        if purge_missing:
            existing = set(db.all_source_files(prefix))
            orphans = sorted(
                e for e in existing
                if source_basename(e).lower() not in current_names
            )
            if not orphans:
                log.info(f"Purge: няма orphan записи в БД за prefix={prefix}.")
            else:
                log.info(f"Purge ({prefix}): намерени {len(orphans)} orphan източника:")
                for o in orphans:
                    log.info(f"   - {o}")
                disk_paths = db.section_output_paths_for(prefix, orphans)
                removed_files = 0
                for op in disk_paths:
                    try:
                        code = Path(str(op).replace("\\", "/")).stem
                        local_txt = output_dir / f"{code}.txt"
                        if local_txt.exists():
                            local_txt.unlink()
                            removed_files += 1
                        opath = Path(op)
                        if opath.exists():
                            try:
                                if opath.resolve() != local_txt.resolve():
                                    opath.unlink()
                                    removed_files += 1
                            except Exception:
                                pass
                        for img in (output_dir / "images").glob(f"{code}_*"):
                            try:
                                img.unlink()
                                removed_files += 1
                            except Exception:
                                pass
                    except Exception as e:
                        log.debug(f"   не успях да изтрия {op}: {e}")
                deleted = db.purge_sources(prefix, orphans)
                log.info(f"Purge: изтрити {deleted} секции от БД, {removed_files} файла от диска.")
    finally:
        db.close()

    log.info(f"Готово. Prefix={prefix}. Общо нови/обновени секции: {total_sections}")
    return {"sections": total_sections, "files": file_results}


# ──────────────────────────────────────────────
# CLI
# ──────────────────────────────────────────────

def main():
    parser = argparse.ArgumentParser(
        description="Help-файл декомпозитор с PostgreSQL + Anthropic"
    )
    parser.add_argument("input_dir",  help="Входна директория с help-файлове")
    parser.add_argument("output_dir", help="Изходна директория за текстови секции")
    parser.add_argument(
        "--conn",
        default=os.getenv("HELP_DB_CONN"),
        help="Postgres libpq connection string (или HELP_DB_CONN env var)"
    )
    parser.add_argument(
        "--api-key",
        default=os.getenv("ANTHROPIC_API_KEY"),
        help="Anthropic API ключ (или ANTHROPIC_API_KEY env var)"
    )
    parser.add_argument(
        "--prefix",
        default=os.getenv("HELP_PREFIX", "HLP"),
        help="Префикс за кодовете/scope в БД (буква + букви/цифри/_, до 50 знака). "
             "Default: 'HLP' (или env HELP_PREFIX)."
    )
    parser.add_argument(
        "--force",
        action="store_true",
        help="Преобработва всички файлове, независимо от hash"
    )
    parser.add_argument(
        "--purge-missing",
        action="store_true",
        help="След обработката изтрива от БД и диска секциите за източници, "
             "които вече не съществуват във входната директория (само в дадения prefix)"
    )
    parser.add_argument(
        "--remote-root",
        default=os.getenv("HELP_REMOTE_ROOT"),
        help="Сървърен корен за output_path в БД "
             "(напр. /mnt/mssql/share/RIP/RIP_Help_Source/Output). "
             "Файловете се пишат локално; в БД се записва remote път. "
             "Или env HELP_REMOTE_ROOT."
    )
    args = parser.parse_args()

    if not args.api_key:
        sys.exit("Грешка: липсва Anthropic API ключ (--api-key или ANTHROPIC_API_KEY).")
    if not args.conn:
        sys.exit("Грешка: липсва Postgres connection string (--conn или HELP_DB_CONN).")

    process_directory(
        input_dir=Path(args.input_dir),
        output_dir=Path(args.output_dir),
        conn_str=args.conn,
        api_key=args.api_key,
        prefix=args.prefix,
        force=args.force,
        purge_missing=args.purge_missing,
        remote_root=args.remote_root,
    )


if __name__ == "__main__":
    main()