From 46a2f2272bdab4e22fdd1588d3ef2eecc1389c3d Mon Sep 17 00:00:00 2001 From: sabo Date: Wed, 26 Aug 2026 09:20:45 +0300 Subject: [PATCH] Add server-side /api/rescan upload pipeline and Home UI. Accept ZIP/single help files, process with help_processor into OUTPUT_DIR+Postgres, pollable jobs, optional RESCAN_TOKEN. Co-authored-by: Cursor --- .env.example | 8 +- COOLIFY.md | 38 +++++-- Dockerfile | 17 ++- docker-compose.yml | 3 +- requirements-webapp.txt | 10 ++ webapp/main.py | 5 +- webapp/rescan.py | 214 +++++++++++++++++++++++++++++++++++ webapp/templates/viewer.html | 84 +++++++++++++- 8 files changed, 352 insertions(+), 27 deletions(-) create mode 100644 webapp/rescan.py diff --git a/.env.example b/.env.example index cf7bc90..e6191bf 100644 --- a/.env.example +++ b/.env.example @@ -5,8 +5,12 @@ REM HELP_DB_CONN е libpq формат за PostgreSQL. ANTHROPIC_API_KEY=sk-ant-api03-XXXXXXXXXXXXXXXXXXXXXXXXX HELP_DB_CONN=host=192.168.88.18 port=5432 dbname=rip_help_system user=postgres password=YOUR_PASSWORD HELP_REMOTE_ROOT=/mnt/mssql/share/RIP/RIP_Help_Source/Output +RESCAN_TOKEN=change-me -REM Coolify Environment Variables (същите ключове): +REM Coolify Environment Variables: REM HELP_DB_CONN=... REM OUTPUT_DIR=/mnt/mssql/share/RIP/RIP_Help_Source/Output -REM HOME_IMAGE=/mnt/mssql/share/RIP/RIP_Help_Source/Output/Bairaci.png +REM HELP_REMOTE_ROOT=/mnt/mssql/share/RIP/RIP_Help_Source/Output +REM HOME_IMAGE=/app/Bairaci.png +REM ANTHROPIC_API_KEY=... +REM RESCAN_TOKEN=... diff --git a/COOLIFY.md b/COOLIFY.md index f50bd6e..0deb74e 100644 --- a/COOLIFY.md +++ b/COOLIFY.md @@ -8,30 +8,44 @@ | Base Directory | `/` | | Dockerfile Location | `/Dockerfile` | -Dockerfile се редактира **само в git** (няма Edit Dockerfile след Add Resource). +Dockerfile се редактира **само в git**. ## Environment Variables ``` HELP_DB_CONN=host=... port=5432 dbname=rip_help_system user=... password=... OUTPUT_DIR=/mnt/mssql/share/RIP/RIP_Help_Source/Output -HOME_IMAGE=/mnt/mssql/share/RIP/RIP_Help_Source/Output/Bairaci.png +HELP_REMOTE_ROOT=/mnt/mssql/share/RIP/RIP_Help_Source/Output +HOME_IMAGE=/app/Bairaci.png +ANTHROPIC_API_KEY=sk-ant-... +RESCAN_TOKEN=избери-тайна # опционално, но препоръчително ``` -## Persistent Storage +## Persistent Storage (read-write!) -| Host path | Container path | Mode | -|-----------|----------------|------| -| `/mnt/mssql/share/RIP/RIP_Help_Source/Output` | `/mnt/mssql/share/RIP/RIP_Help_Source/Output` | read-only OK | +| Host path | Container path | +|-----------|----------------| +| `/mnt/mssql/share/RIP/RIP_Help_Source/Output` | `/mnt/mssql/share/RIP/RIP_Help_Source/Output` | -(Файловете се качват с WinSCP в този host path.) +Volume-ът трябва да е **writable** — rescan пише `.txt` + `images/`. -## Проверки след Redeploy +## API rescan -1. `/healthz` → `status/db ok`, `output_dir_ok: true`, `txt_count` > 0 -2. `/api/sections?prefix=RIP` → `text` не е празен -3. `/api/search?q=поръчки&prefix=RIP` -4. `/?prefix=RIP` — dashboard/viewer +- `POST /api/rescan` — multipart `file` (+ `prefix`, `force`) +- `GET /api/rescan/{job_id}` — статус +- UI: Home таб → Rescan + +Ако `RESCAN_TOKEN` е зададен: +``` +localStorage.setItem('RESCAN_TOKEN', 'тайната') +``` +в конзолата на браузъра (веднъж), или header `X-Rescan-Token`. + +## Проверки + +1. `/healthz` → `output_dir_ok: true` +2. `/?prefix=RIP` — Home + Rescan +3. `/api/search?q=...&prefix=RIP` ## Порт diff --git a/Dockerfile b/Dockerfile index 72ccb4c..f1510b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,27 @@ -# Slim Python image; psycopg2-binary включва свои libs, не е нужен build deps +# Python 3.12 — webapp + server-side rescan (help_processor) FROM python:3.12-slim WORKDIR /app -# Сваляме само файловете нужни за webapp-а (не и Windows-only deps) -# Затова инсталираме отделен webapp-requirements без pywin32 / pdfplumber / docx +RUN apt-get update && apt-get install -y --no-install-recommends \ + libxml2 \ + && rm -rf /var/lib/apt/lists/* + COPY requirements-webapp.txt /app/ RUN pip install --no-cache-dir -r requirements-webapp.txt -# Кода + home image в образа (fallback ако липсва в share) COPY webapp /app/webapp +COPY help_processor.py /app/help_processor.py COPY Bairaci.png /app/Bairaci.png -# Секции/картинки — Coolify Persistent Storage mount ENV OUTPUT_DIR=/mnt/mssql/share/RIP/RIP_Help_Source/Output ENV HOME_IMAGE=/app/Bairaci.png +ENV HELP_REMOTE_ROOT=/mnt/mssql/share/RIP/RIP_Help_Source/Output +# Set in Coolify (required for /api/rescan): +# ANTHROPIC_API_KEY=... +# HELP_DB_CONN=... +# Optional: +# RESCAN_TOKEN=secret → require header X-Rescan-Token EXPOSE 8000 diff --git a/docker-compose.yml b/docker-compose.yml index 6371f14..9e7c20d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,8 +9,7 @@ services: OUTPUT_DIR: "/mnt/mssql/share/RIP/RIP_Help_Source/Output" HOME_IMAGE: "/mnt/mssql/share/RIP/RIP_Help_Source/Output/Bairaci.png" volumes: - # Host share (качен с WinSCP) → същият път в контейнера - - /mnt/mssql/share/RIP/RIP_Help_Source/Output:/mnt/mssql/share/RIP/RIP_Help_Source/Output:ro + - /mnt/mssql/share/RIP/RIP_Help_Source/Output:/mnt/mssql/share/RIP/RIP_Help_Source/Output ports: - "8000:8000" restart: unless-stopped diff --git a/requirements-webapp.txt b/requirements-webapp.txt index 7cdc7df..3f2dfae 100644 --- a/requirements-webapp.txt +++ b/requirements-webapp.txt @@ -3,3 +3,13 @@ uvicorn[standard]>=0.27.0 jinja2>=3.1.0 psycopg2-binary>=2.9.9 pydantic>=2.0.0 +python-multipart>=0.0.9 + +# help_processor (server-side rescan) +anthropic>=0.25.0 +python-docx>=1.1.0 +beautifulsoup4>=4.12.0 +lxml>=5.0.0 +pdfplumber>=0.11.0 +chardet>=5.0.0 +Pillow>=10.0.0 diff --git a/webapp/main.py b/webapp/main.py index a7ae1e1..70b1d6a 100644 --- a/webapp/main.py +++ b/webapp/main.py @@ -48,8 +48,11 @@ HOME_IMAGE = os.getenv("HOME_IMAGE") # absolute path or None _IMG_PLACEHOLDER_RE = re.compile(r"\[IMG:\s*([^\]]+?)\s*\]") BASE_DIR = Path(__file__).parent -app = FastAPI(title="RIP Help System", version="0.2.0") +from webapp import rescan as rescan_mod + +app = FastAPI(title="RIP Help System", version="0.3.0") templates = Jinja2Templates(directory=str(BASE_DIR / "templates")) +app.include_router(rescan_mod.router) # ────────────────────────────────────────────── diff --git a/webapp/rescan.py b/webapp/rescan.py new file mode 100644 index 0000000..79de0c0 --- /dev/null +++ b/webapp/rescan.py @@ -0,0 +1,214 @@ +""" +Server-side rescan: upload help files (zip or single) → process → OUTPUT_DIR + Postgres. +""" +from __future__ import annotations + +import io +import shutil +import tempfile +import threading +import uuid +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +from fastapi import APIRouter, BackgroundTasks, File, Form, Header, HTTPException, UploadFile +from fastapi.responses import JSONResponse + +router = APIRouter(prefix="/api", tags=["rescan"]) + +_jobs: dict[str, dict] = {} +_lock = threading.Lock() + +ALLOWED_EXT = {".html", ".htm", ".docx", ".doc", ".txt", ".pdf", ".zip"} + + +def _cfg(): + import os + + output = Path( + os.getenv( + "OUTPUT_DIR", + "/mnt/mssql/share/RIP/RIP_Help_Source/Output", + ) + ) + return { + "conn": os.getenv("HELP_DB_CONN", ""), + "api_key": os.getenv("ANTHROPIC_API_KEY", ""), + "token": os.getenv("RESCAN_TOKEN", ""), + "output_dir": output, + "remote_root": os.getenv( + "HELP_REMOTE_ROOT", + str(output).replace("\\", "/"), + ), + } + + +def _check_token(provided: Optional[str]): + expected = _cfg()["token"] + if not expected: + return + if not provided or provided != expected: + raise HTTPException(401, "Invalid or missing RESCAN_TOKEN") + + +def _set_job(job_id: str, **fields): + with _lock: + job = _jobs.setdefault(job_id, {"id": job_id}) + job.update(fields) + job["updated_at"] = datetime.now(timezone.utc).isoformat() + + +def _safe_extract_zip(data: bytes, dest: Path) -> int: + dest = dest.resolve() + count = 0 + with zipfile.ZipFile(io.BytesIO(data)) as zf: + for info in zf.infolist(): + if info.is_dir(): + continue + name = Path(info.filename) + # skip macOS junk / absolute paths + if any(p.startswith("__MACOSX") or p.startswith(".") for p in name.parts): + continue + target = (dest / name.name).resolve() + if not str(target).startswith(str(dest)): + raise ValueError(f"unsafe zip path: {info.filename}") + target.parent.mkdir(parents=True, exist_ok=True) + with zf.open(info) as src, open(target, "wb") as out: + shutil.copyfileobj(src, out) + count += 1 + return count + + +def _run_rescan(job_id: str, staging: Path, prefix: str, force: bool): + cfg = _cfg() + try: + if not cfg["conn"]: + raise RuntimeError("HELP_DB_CONN is not set on the server") + if not cfg["api_key"]: + raise RuntimeError("ANTHROPIC_API_KEY is not set on the server") + + out = cfg["output_dir"] + out.mkdir(parents=True, exist_ok=True) + if not os_access_writable(out): + raise RuntimeError( + f"OUTPUT_DIR is not writable: {out} " + "(Coolify Persistent Storage must be read-write)" + ) + + _set_job(job_id, status="processing", message="running help_processor") + + # Import here so webapp boots even if processor deps fail later + from help_processor import process_directory + + process_directory( + input_dir=staging, + output_dir=out, + conn_str=cfg["conn"], + api_key=cfg["api_key"], + prefix=prefix, + force=force, + purge_missing=False, + remote_root=cfg["remote_root"], + ) + _set_job( + job_id, + status="done", + message="ok", + output_dir=str(out), + prefix=prefix, + ) + except Exception as e: + _set_job(job_id, status="error", message=str(e)) + finally: + shutil.rmtree(staging, ignore_errors=True) + + +def os_access_writable(path: Path) -> bool: + try: + path.mkdir(parents=True, exist_ok=True) + probe = path / f".write_probe_{uuid.uuid4().hex}" + probe.write_text("ok", encoding="utf-8") + probe.unlink() + return True + except Exception: + return False + + +@router.post("/rescan") +async def rescan( + background_tasks: BackgroundTasks, + file: UploadFile = File(..., description="ZIP or single help file"), + prefix: str = Form("RIP"), + force: str = Form("false"), + x_rescan_token: Optional[str] = Header(None, alias="X-Rescan-Token"), + token: Optional[str] = Form(None), +): + """ + Upload a .zip (or one .docx/.html/.pdf/.txt/.doc), process on server, + write sections to OUTPUT_DIR and Postgres. + """ + _check_token(x_rescan_token or token) + + name = (file.filename or "upload.bin").strip() + ext = Path(name).suffix.lower() + if ext not in ALLOWED_EXT: + raise HTTPException( + 400, + f"Unsupported type {ext}. Allowed: {sorted(ALLOWED_EXT)}", + ) + + data = await file.read() + if not data: + raise HTTPException(400, "Empty upload") + if len(data) > 120 * 1024 * 1024: + raise HTTPException(400, "Upload too large (max 120 MB)") + + staging = Path(tempfile.mkdtemp(prefix="rip_rescan_")) + input_dir = staging / "input" + input_dir.mkdir(parents=True, exist_ok=True) + + try: + if ext == ".zip": + n = _safe_extract_zip(data, input_dir) + if n == 0: + raise HTTPException(400, "ZIP has no usable files") + else: + (input_dir / Path(name).name).write_bytes(data) + except HTTPException: + shutil.rmtree(staging, ignore_errors=True) + raise + except Exception as e: + shutil.rmtree(staging, ignore_errors=True) + raise HTTPException(400, f"Failed to unpack upload: {e}") from e + + job_id = uuid.uuid4().hex[:12] + force_flag = str(force).lower() in ("1", "true", "yes", "on") + _set_job( + job_id, + status="queued", + message="accepted", + filename=name, + prefix=prefix, + force=force_flag, + created_at=datetime.now(timezone.utc).isoformat(), + ) + background_tasks.add_task(_run_rescan, job_id, staging, prefix, force_flag) + return JSONResponse( + { + "ok": True, + "job_id": job_id, + "status": "queued", + "poll": f"/api/rescan/{job_id}", + } + ) + + +@router.get("/rescan/{job_id}") +def rescan_status(job_id: str): + with _lock: + job = _jobs.get(job_id) + if not job: + raise HTTPException(404, "job not found") + return JSONResponse(job) diff --git a/webapp/templates/viewer.html b/webapp/templates/viewer.html index fa90069..0e665c8 100644 --- a/webapp/templates/viewer.html +++ b/webapp/templates/viewer.html @@ -195,12 +195,23 @@ .sep { color: var(--border); margin: 0 4px; } - .home-wrap { display: flex; justify-content: center; align-items: flex-start; padding: 24px; } + .home-wrap { display: flex; flex-direction: column; align-items: center; gap: 20px; padding: 24px; } .home-wrap img { - max-width: 100%; max-height: calc(100vh - 200px); - border: 1px solid var(--border); border-radius: var(--radius-lg); - box-shadow: 0 4px 16px rgba(0,0,0,.06); + max-width: min(920px, 100%); height: auto; border-radius: var(--radius-lg); + border: 1px solid var(--border); box-shadow: 0 8px 24px rgba(0,0,0,.08); } + .rescan-box { + width: min(640px, 100%); + background: var(--bg2); border: 1px solid var(--border); border-radius: var(--radius-lg); + padding: 16px 18px; + } + .rescan-box h3 { + font-family: var(--mono); font-size: 12px; color: var(--muted); + letter-spacing: .06em; text-transform: uppercase; margin-bottom: 12px; + } + .rescan-row { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; margin-bottom: 10px; } + .rescan-box input[type=file] { font-size: 12px; } + .rescan-status { font-family: var(--mono); font-size: 12px; color: var(--muted); min-height: 1.2em; } @@ -222,7 +233,18 @@ {% if home_url %}
-
Home
+
+ Home +
+

Rescan — качи help файлове

+
+ + + +
+
ZIP или един файл → обработка на сървъра → БД + Output
+
+
{% endif %} @@ -307,6 +329,58 @@ function switchTab(name) { if (name === 'generator') renderGenerator(); } +async function startRescan() { + const inp = document.getElementById('rescan-file'); + const st = document.getElementById('rescan-status'); + const btn = document.getElementById('rescan-btn'); + if (!inp.files || !inp.files[0]) { + st.textContent = 'Избери файл първо.'; + return; + } + const fd = new FormData(); + fd.append('file', inp.files[0]); + fd.append('prefix', {{ (prefix or 'RIP') | tojson }}); + fd.append('force', document.getElementById('rescan-force').checked ? 'true' : 'false'); + const token = localStorage.getItem('RESCAN_TOKEN') || ''; + if (token) fd.append('token', token); + + btn.disabled = true; + st.textContent = 'Качване...'; + try { + const headers = {}; + if (token) headers['X-Rescan-Token'] = token; + const res = await fetch('/api/rescan', { method: 'POST', body: fd, headers }); + const data = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(data.detail || ('HTTP ' + res.status)); + st.textContent = 'Job ' + data.job_id + ' — обработка...'; + await pollRescan(data.job_id); + } catch (e) { + st.textContent = 'Грешка: ' + e.message; + } finally { + btn.disabled = false; + } +} + +async function pollRescan(jobId) { + const st = document.getElementById('rescan-status'); + for (let i = 0; i < 180; i++) { + await new Promise(r => setTimeout(r, 2000)); + const res = await fetch('/api/rescan/' + encodeURIComponent(jobId)); + const data = await res.json(); + st.textContent = data.status + (data.message ? ' — ' + data.message : ''); + if (data.status === 'done') { + st.textContent = 'Готово. Презареждане...'; + location.reload(); + return; + } + if (data.status === 'error') { + st.textContent = 'Грешка: ' + (data.message || 'unknown'); + return; + } + } + st.textContent = 'Timeout — провери /api/rescan/' + jobId; +} + // ── EDITOR ──────────────────────────── function renderEditor(rows) { const tbody = document.getElementById('editor-body');