Add server-side /api/rescan upload pipeline and Home UI.
Accept ZIP/single help files, process with help_processor into OUTPUT_DIR+Postgres, pollable jobs, optional RESCAN_TOKEN. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -48,8 +48,11 @@ HOME_IMAGE = os.getenv("HOME_IMAGE") # absolute path or None
|
||||
_IMG_PLACEHOLDER_RE = re.compile(r"\[IMG:\s*([^\]]+?)\s*\]")
|
||||
|
||||
BASE_DIR = Path(__file__).parent
|
||||
app = FastAPI(title="RIP Help System", version="0.2.0")
|
||||
from webapp import rescan as rescan_mod
|
||||
|
||||
app = FastAPI(title="RIP Help System", version="0.3.0")
|
||||
templates = Jinja2Templates(directory=str(BASE_DIR / "templates"))
|
||||
app.include_router(rescan_mod.router)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
214
webapp/rescan.py
Normal file
214
webapp/rescan.py
Normal file
@@ -0,0 +1,214 @@
|
||||
"""
|
||||
Server-side rescan: upload help files (zip or single) → process → OUTPUT_DIR + Postgres.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
import uuid
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, File, Form, Header, HTTPException, UploadFile
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["rescan"])
|
||||
|
||||
_jobs: dict[str, dict] = {}
|
||||
_lock = threading.Lock()
|
||||
|
||||
ALLOWED_EXT = {".html", ".htm", ".docx", ".doc", ".txt", ".pdf", ".zip"}
|
||||
|
||||
|
||||
def _cfg():
|
||||
import os
|
||||
|
||||
output = Path(
|
||||
os.getenv(
|
||||
"OUTPUT_DIR",
|
||||
"/mnt/mssql/share/RIP/RIP_Help_Source/Output",
|
||||
)
|
||||
)
|
||||
return {
|
||||
"conn": os.getenv("HELP_DB_CONN", ""),
|
||||
"api_key": os.getenv("ANTHROPIC_API_KEY", ""),
|
||||
"token": os.getenv("RESCAN_TOKEN", ""),
|
||||
"output_dir": output,
|
||||
"remote_root": os.getenv(
|
||||
"HELP_REMOTE_ROOT",
|
||||
str(output).replace("\\", "/"),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _check_token(provided: Optional[str]):
|
||||
expected = _cfg()["token"]
|
||||
if not expected:
|
||||
return
|
||||
if not provided or provided != expected:
|
||||
raise HTTPException(401, "Invalid or missing RESCAN_TOKEN")
|
||||
|
||||
|
||||
def _set_job(job_id: str, **fields):
|
||||
with _lock:
|
||||
job = _jobs.setdefault(job_id, {"id": job_id})
|
||||
job.update(fields)
|
||||
job["updated_at"] = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _safe_extract_zip(data: bytes, dest: Path) -> int:
|
||||
dest = dest.resolve()
|
||||
count = 0
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as zf:
|
||||
for info in zf.infolist():
|
||||
if info.is_dir():
|
||||
continue
|
||||
name = Path(info.filename)
|
||||
# skip macOS junk / absolute paths
|
||||
if any(p.startswith("__MACOSX") or p.startswith(".") for p in name.parts):
|
||||
continue
|
||||
target = (dest / name.name).resolve()
|
||||
if not str(target).startswith(str(dest)):
|
||||
raise ValueError(f"unsafe zip path: {info.filename}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(info) as src, open(target, "wb") as out:
|
||||
shutil.copyfileobj(src, out)
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def _run_rescan(job_id: str, staging: Path, prefix: str, force: bool):
|
||||
cfg = _cfg()
|
||||
try:
|
||||
if not cfg["conn"]:
|
||||
raise RuntimeError("HELP_DB_CONN is not set on the server")
|
||||
if not cfg["api_key"]:
|
||||
raise RuntimeError("ANTHROPIC_API_KEY is not set on the server")
|
||||
|
||||
out = cfg["output_dir"]
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
if not os_access_writable(out):
|
||||
raise RuntimeError(
|
||||
f"OUTPUT_DIR is not writable: {out} "
|
||||
"(Coolify Persistent Storage must be read-write)"
|
||||
)
|
||||
|
||||
_set_job(job_id, status="processing", message="running help_processor")
|
||||
|
||||
# Import here so webapp boots even if processor deps fail later
|
||||
from help_processor import process_directory
|
||||
|
||||
process_directory(
|
||||
input_dir=staging,
|
||||
output_dir=out,
|
||||
conn_str=cfg["conn"],
|
||||
api_key=cfg["api_key"],
|
||||
prefix=prefix,
|
||||
force=force,
|
||||
purge_missing=False,
|
||||
remote_root=cfg["remote_root"],
|
||||
)
|
||||
_set_job(
|
||||
job_id,
|
||||
status="done",
|
||||
message="ok",
|
||||
output_dir=str(out),
|
||||
prefix=prefix,
|
||||
)
|
||||
except Exception as e:
|
||||
_set_job(job_id, status="error", message=str(e))
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def os_access_writable(path: Path) -> bool:
|
||||
try:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
probe = path / f".write_probe_{uuid.uuid4().hex}"
|
||||
probe.write_text("ok", encoding="utf-8")
|
||||
probe.unlink()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
@router.post("/rescan")
|
||||
async def rescan(
|
||||
background_tasks: BackgroundTasks,
|
||||
file: UploadFile = File(..., description="ZIP or single help file"),
|
||||
prefix: str = Form("RIP"),
|
||||
force: str = Form("false"),
|
||||
x_rescan_token: Optional[str] = Header(None, alias="X-Rescan-Token"),
|
||||
token: Optional[str] = Form(None),
|
||||
):
|
||||
"""
|
||||
Upload a .zip (or one .docx/.html/.pdf/.txt/.doc), process on server,
|
||||
write sections to OUTPUT_DIR and Postgres.
|
||||
"""
|
||||
_check_token(x_rescan_token or token)
|
||||
|
||||
name = (file.filename or "upload.bin").strip()
|
||||
ext = Path(name).suffix.lower()
|
||||
if ext not in ALLOWED_EXT:
|
||||
raise HTTPException(
|
||||
400,
|
||||
f"Unsupported type {ext}. Allowed: {sorted(ALLOWED_EXT)}",
|
||||
)
|
||||
|
||||
data = await file.read()
|
||||
if not data:
|
||||
raise HTTPException(400, "Empty upload")
|
||||
if len(data) > 120 * 1024 * 1024:
|
||||
raise HTTPException(400, "Upload too large (max 120 MB)")
|
||||
|
||||
staging = Path(tempfile.mkdtemp(prefix="rip_rescan_"))
|
||||
input_dir = staging / "input"
|
||||
input_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
if ext == ".zip":
|
||||
n = _safe_extract_zip(data, input_dir)
|
||||
if n == 0:
|
||||
raise HTTPException(400, "ZIP has no usable files")
|
||||
else:
|
||||
(input_dir / Path(name).name).write_bytes(data)
|
||||
except HTTPException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
except Exception as e:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise HTTPException(400, f"Failed to unpack upload: {e}") from e
|
||||
|
||||
job_id = uuid.uuid4().hex[:12]
|
||||
force_flag = str(force).lower() in ("1", "true", "yes", "on")
|
||||
_set_job(
|
||||
job_id,
|
||||
status="queued",
|
||||
message="accepted",
|
||||
filename=name,
|
||||
prefix=prefix,
|
||||
force=force_flag,
|
||||
created_at=datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
background_tasks.add_task(_run_rescan, job_id, staging, prefix, force_flag)
|
||||
return JSONResponse(
|
||||
{
|
||||
"ok": True,
|
||||
"job_id": job_id,
|
||||
"status": "queued",
|
||||
"poll": f"/api/rescan/{job_id}",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/rescan/{job_id}")
|
||||
def rescan_status(job_id: str):
|
||||
with _lock:
|
||||
job = _jobs.get(job_id)
|
||||
if not job:
|
||||
raise HTTPException(404, "job not found")
|
||||
return JSONResponse(job)
|
||||
@@ -195,12 +195,23 @@
|
||||
|
||||
.sep { color: var(--border); margin: 0 4px; }
|
||||
|
||||
.home-wrap { display: flex; justify-content: center; align-items: flex-start; padding: 24px; }
|
||||
.home-wrap { display: flex; flex-direction: column; align-items: center; gap: 20px; padding: 24px; }
|
||||
.home-wrap img {
|
||||
max-width: 100%; max-height: calc(100vh - 200px);
|
||||
border: 1px solid var(--border); border-radius: var(--radius-lg);
|
||||
box-shadow: 0 4px 16px rgba(0,0,0,.06);
|
||||
max-width: min(920px, 100%); height: auto; border-radius: var(--radius-lg);
|
||||
border: 1px solid var(--border); box-shadow: 0 8px 24px rgba(0,0,0,.08);
|
||||
}
|
||||
.rescan-box {
|
||||
width: min(640px, 100%);
|
||||
background: var(--bg2); border: 1px solid var(--border); border-radius: var(--radius-lg);
|
||||
padding: 16px 18px;
|
||||
}
|
||||
.rescan-box h3 {
|
||||
font-family: var(--mono); font-size: 12px; color: var(--muted);
|
||||
letter-spacing: .06em; text-transform: uppercase; margin-bottom: 12px;
|
||||
}
|
||||
.rescan-row { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; margin-bottom: 10px; }
|
||||
.rescan-box input[type=file] { font-size: 12px; }
|
||||
.rescan-status { font-family: var(--mono); font-size: 12px; color: var(--muted); min-height: 1.2em; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
@@ -222,7 +233,18 @@
|
||||
|
||||
{% if home_url %}
|
||||
<div id="tab-home" class="panel active">
|
||||
<div class="home-wrap"><img src="{{ home_url }}" alt="Home"></div>
|
||||
<div class="home-wrap">
|
||||
<img src="{{ home_url }}" alt="Home">
|
||||
<div class="rescan-box">
|
||||
<h3>Rescan — качи help файлове</h3>
|
||||
<div class="rescan-row">
|
||||
<input type="file" id="rescan-file" accept=".zip,.docx,.doc,.html,.htm,.pdf,.txt">
|
||||
<label class="stats"><input type="checkbox" id="rescan-force"> force</label>
|
||||
<button class="btn-primary" id="rescan-btn" onclick="startRescan()">Старт</button>
|
||||
</div>
|
||||
<div class="rescan-status" id="rescan-status">ZIP или един файл → обработка на сървъра → БД + Output</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
@@ -307,6 +329,58 @@ function switchTab(name) {
|
||||
if (name === 'generator') renderGenerator();
|
||||
}
|
||||
|
||||
async function startRescan() {
|
||||
const inp = document.getElementById('rescan-file');
|
||||
const st = document.getElementById('rescan-status');
|
||||
const btn = document.getElementById('rescan-btn');
|
||||
if (!inp.files || !inp.files[0]) {
|
||||
st.textContent = 'Избери файл първо.';
|
||||
return;
|
||||
}
|
||||
const fd = new FormData();
|
||||
fd.append('file', inp.files[0]);
|
||||
fd.append('prefix', {{ (prefix or 'RIP') | tojson }});
|
||||
fd.append('force', document.getElementById('rescan-force').checked ? 'true' : 'false');
|
||||
const token = localStorage.getItem('RESCAN_TOKEN') || '';
|
||||
if (token) fd.append('token', token);
|
||||
|
||||
btn.disabled = true;
|
||||
st.textContent = 'Качване...';
|
||||
try {
|
||||
const headers = {};
|
||||
if (token) headers['X-Rescan-Token'] = token;
|
||||
const res = await fetch('/api/rescan', { method: 'POST', body: fd, headers });
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) throw new Error(data.detail || ('HTTP ' + res.status));
|
||||
st.textContent = 'Job ' + data.job_id + ' — обработка...';
|
||||
await pollRescan(data.job_id);
|
||||
} catch (e) {
|
||||
st.textContent = 'Грешка: ' + e.message;
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function pollRescan(jobId) {
|
||||
const st = document.getElementById('rescan-status');
|
||||
for (let i = 0; i < 180; i++) {
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
const res = await fetch('/api/rescan/' + encodeURIComponent(jobId));
|
||||
const data = await res.json();
|
||||
st.textContent = data.status + (data.message ? ' — ' + data.message : '');
|
||||
if (data.status === 'done') {
|
||||
st.textContent = 'Готово. Презареждане...';
|
||||
location.reload();
|
||||
return;
|
||||
}
|
||||
if (data.status === 'error') {
|
||||
st.textContent = 'Грешка: ' + (data.message || 'unknown');
|
||||
return;
|
||||
}
|
||||
}
|
||||
st.textContent = 'Timeout — провери /api/rescan/' + jobId;
|
||||
}
|
||||
|
||||
// ── EDITOR ────────────────────────────
|
||||
function renderEditor(rows) {
|
||||
const tbody = document.getElementById('editor-body');
|
||||
|
||||
Reference in New Issue
Block a user