sabo/rip-help-system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Give each help file unique sequential extraction codes, show the count, and allow wiping all extractions for a file while testing.

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
sabo
2026-09-10 13:26:21 +03:00
parent 170c551d6e
commit 3a30959ae5
5 changed files with 761 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
webapp/main.py
View File

@@ -27,11 +27,19 @@ from pathlib import Path
from typing import Optional
import psycopg2
from fastapi import FastAPI, HTTPException, Query, Request
from fastapi import FastAPI, Header, HTTPException, Query, Request
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
from fastapi.templating import Jinja2Templates
from pydantic import BaseModel
from help_codes import (
WIPED_HASH,
numbering_report,
parse_code,
remove_section_outputs,
source_basename,
)
# ──────────────────────────────────────────────
# Configuration
# ──────────────────────────────────────────────
@@ -317,6 +325,183 @@ def update_keywords(code: str, body: KeywordsUpdate):
return {"ok": True, "code": code}
def _safe_help_filename(file: str) -> str:
name = Path(str(file or "").replace("\\", "/")).name
if not name or name in (".", ".."):
raise HTTPException(400, "file is required")
return name
def _ensure_file_index_col(cur):
cur.execute(
"ALTER TABLE rip_help_files ADD COLUMN IF NOT EXISTS file_index INTEGER"
)
def _matching_source_paths(cur, prefix: str, identity: str) -> list[str]:
name = source_basename(identity).lower()
if not name:
return []
cur.execute(
"""
SELECT file_path FROM rip_help_files WHERE prefix=%s
UNION
SELECT source_file FROM rip_help_sections WHERE prefix=%s
""",
(prefix, prefix),
)
found: list[str] = []
seen: set[str] = set()
for (p,) in cur.fetchall():
if p and source_basename(p).lower() == name and p not in seen:
seen.add(p)
found.append(p)
return found
def _file_index_for(cur, prefix: str, paths: list[str]) -> Optional[int]:
if not paths:
return None
cur.execute(
"SELECT file_index FROM rip_help_files "
"WHERE prefix=%s AND file_path = ANY(%s) AND file_index IS NOT NULL",
(prefix, list(paths)),
)
from_col = [r[0] for r in cur.fetchall() if r[0]]
if from_col:
return min(from_col)
cur.execute(
"SELECT code FROM rip_help_sections WHERE prefix=%s AND source_file = ANY(%s)",
(prefix, list(paths)),
)
found: list[int] = []
for (code,) in cur.fetchall():
parsed = parse_code(code)
if parsed:
found.append(parsed[1])
if not found:
return None
tally: dict[int, int] = {}
for idx in found:
tally[idx] = tally.get(idx, 0) + 1
return max(tally, key=lambda k: (tally[k], -k))
@app.get("/api/file-extractions")
def api_file_extractions(
file: str = Query(..., min_length=1),
prefix: str = Query("RIP"),
):
"""Брой и номерация на извлеченията за даден help файл."""
name = _safe_help_filename(file)
conn = db_conn()
try:
cur = conn.cursor()
_ensure_file_index_col(cur)
conn.commit()
paths = _matching_source_paths(cur, prefix, name)
rows = []
if paths:
cur.execute(
"SELECT code, source_file FROM rip_help_sections "
"WHERE prefix=%s AND source_file = ANY(%s) ORDER BY code",
(prefix, list(paths)),
)
rows = cur.fetchall()
codes = [r[0] for r in rows]
report = numbering_report(codes)
fi = _file_index_for(cur, prefix, paths)
if fi is not None:
report["file_index"] = fi
return {
"file": name,
"prefix": prefix,
"sources": sorted({source_basename(r[1]) for r in rows if r[1]}),
**report,
}
finally:
conn.close()
@app.delete("/api/file-extractions")
def api_delete_file_extractions(
file: str = Query(..., min_length=1),
prefix: str = Query("RIP"),
x_rescan_token: Optional[str] = Header(None, alias="X-Rescan-Token"),
token: Optional[str] = Query(None),
):
"""Изтрива всички извлечения за файла. Самият help файл не се пипа."""
rescan_mod._check_token(x_rescan_token or token)
name = _safe_help_filename(file)
conn = db_conn()
try:
cur = conn.cursor()
_ensure_file_index_col(cur)
paths = _matching_source_paths(cur, prefix, name)
rows = []
if paths:
cur.execute(
"SELECT code, output_path FROM rip_help_sections "
"WHERE prefix=%s AND source_file = ANY(%s)",
(prefix, list(paths)),
)
rows = cur.fetchall()
codes = [r[0] for r in rows]
file_index = _file_index_for(cur, prefix, paths)
remove_section_outputs(OUTPUT_DIR, codes, [r[1] for r in rows if r[1]])
if file_index:
cur.execute(
"SELECT code, source_file FROM rip_help_sections WHERE prefix=%s",
(prefix,),
)
name_l = name.lower()
shared = False
for code, src in cur.fetchall():
parsed = parse_code(code)
if (
parsed
and parsed[1] == file_index
and source_basename(src).lower() != name_l
):
shared = True
break
if shared:
file_index = None
if paths:
cur.execute(
"DELETE FROM rip_help_sections WHERE prefix=%s AND source_file = ANY(%s)",
(prefix, list(paths)),
)
cur.execute(
"DELETE FROM rip_help_files WHERE prefix=%s AND file_path = ANY(%s)",
(prefix, list(paths)),
)
if file_index:
cur.execute(
"""
INSERT INTO rip_help_files (prefix, file_path, file_hash, section_count, file_index)
VALUES (%s, %s, %s, 0, %s)
ON CONFLICT (prefix, file_path) DO UPDATE SET
file_hash = EXCLUDED.file_hash,
section_count = 0,
processed_at = NOW(),
file_index = EXCLUDED.file_index
""",
(prefix, name, WIPED_HASH, file_index),
)
conn.commit()
return {
"ok": True,
"file": name,
"prefix": prefix,
"deleted": len(codes),
"codes": codes,
"file_index": file_index,
}
finally:
conn.close()
@app.get("/healthz", summary="Статус на услугата и БД; output_dir_ok, txt_count.")
def healthz():
db_status = "ok"

15
webapp/rescan.py
View File

@@ -102,8 +102,11 @@ def _run_rescan(job_id: str, staging: Path, prefix: str, force: bool):
# Import here so webapp boots even if processor deps fail later
from help_processor import process_directory
process_directory(
input_dir=staging,
input_dir = staging / "input"
if not input_dir.is_dir():
input_dir = staging
summary = process_directory(
input_dir=input_dir,
output_dir=out,
conn_str=cfg["conn"],
api_key=cfg["api_key"],
@@ -112,12 +115,20 @@ def _run_rescan(job_id: str, staging: Path, prefix: str, force: bool):
purge_missing=False,
remote_root=cfg["remote_root"],
)
files = []
if isinstance(summary, dict):
files = summary.get("files") or []
sections = summary.get("sections")
else:
sections = summary
_set_job(
job_id,
status="done",
message="ok",
output_dir=str(out),
prefix=prefix,
files=files,
sections=sections,
)
except Exception as e:
_set_job(job_id, status="error", message=str(e))

195
webapp/templates/viewer.html
View File

@@ -268,6 +268,12 @@
.rescan-status.busy { color: var(--accent); }
.rescan-status.ok { color: var(--accent2); }
.rescan-status.error { color: var(--danger); }
.rescan-filemeta {
font-family: var(--mono); font-size: 12px; color: var(--muted);
margin-bottom: 12px; line-height: 1.45;
}
.rescan-filemeta:empty { display: none; }
.rescan-filemeta.warn { color: var(--danger); }
.api-layout { display: grid; grid-template-columns: minmax(280px, 1fr) minmax(320px, 1.1fr); gap: 20px; }
@media (max-width: 900px) { .api-layout { grid-template-columns: 1fr; } }
@@ -350,9 +356,12 @@
webkitdirectory directory multiple
onchange="onScanFolderPicked()">
<div class="rescan-selection" id="rescan-selection"></div>
<div class="rescan-filemeta" id="rescan-filemeta"></div>
<div class="rescan-row">
<label class="stats"><input type="checkbox" id="rescan-force"> force</label>
<button type="button" class="btn-primary" id="rescan-btn" onclick="startRescan()">Старт</button>
<button type="button" class="btn-danger" id="rescan-wipe-btn" hidden
onclick="confirmWipeFile()">Изтрий извлеченията</button>
</div>
<div class="rescan-status" id="rescan-status">Избери файл или папка.</div>
</div>
@@ -362,6 +371,7 @@
<div class="toolbar">
<input type="text" class="search-box" id="editor-search" placeholder="Филтрирай по код, заглавие, ключова дума..." oninput="filterEditor()">
<span class="stats" id="editor-stats"></span>
<span class="stats" id="editor-file-stats"></span>
</div>
<div class="tbl-wrap">
<table id="editor-table">
@@ -460,6 +470,10 @@
<div class="desc">multipart: <code>file</code> (един или повече: zip/docx/…), <code>prefix</code>, <code>force</code>. Header опционално: <code>X-Rescan-Token</code>.</div></div>
<div class="api-ep"><span class="method">GET</span><code>/api/rescan/{job_id}</code>
<div class="desc">Статус на rescan задача (<code>queued</code> / <code>processing</code> / <code>done</code> / <code>error</code>).</div></div>
<div class="api-ep"><span class="method">GET</span><code>/api/file-extractions?file=…&amp;prefix=RIP</code>
<div class="desc">Брой и кодове на извлеченията за даден файл; <code>ok</code> ако SEC номерацията е уникална и последователна.</div></div>
<div class="api-ep"><span class="method post">DELETE</span><code>/api/file-extractions?file=…&amp;prefix=RIP</code>
<div class="desc">Изтрива всички извлечения за файла (не самия help файл). Header опционално: <code>X-Rescan-Token</code>.</div></div>
<p class="api-note">OpenAPI / Swagger: <a href="/docs" target="_blank" rel="noopener">/docs</a> · ReDoc: <a href="/redoc" target="_blank" rel="noopener">/redoc</a></p>
<p class="api-note">Типичен ERP поток: <code>GET /api/search</code> → избор на <code>code</code> → <code>GET /api/section/{code}</code> за съдържание.</p>
@@ -490,6 +504,20 @@
</div>
</div>
<div class="modal-backdrop" id="wipe-modal" onclick="if(event.target===this)closeWipeModal()">
<div class="modal" role="dialog" aria-modal="true" style="width:min(460px,100%)">
<div class="modal-head">
<h2>Изтриване на извлечения</h2>
<button type="button" class="btn-ghost" onclick="closeWipeModal()">✕</button>
</div>
<div class="modal-body" id="wipe-modal-body"></div>
<div class="rescan-row" style="margin:16px 0 0">
<button type="button" class="btn-danger" id="wipe-confirm-btn" onclick="doWipeFile()">Изтрий</button>
<button type="button" class="btn-ghost" onclick="closeWipeModal()">Отказ</button>
</div>
</div>
</div>
<div id="toast"></div>
<script type="application/json" id="sections-data">{{ sections_json | safe }}</script>
@@ -534,14 +562,22 @@ function switchTab(name) {
}
const SCAN_EXT = ['.zip', '.docx', '.doc', '.html', '.htm', '.pdf', '.txt'];
const PAGE_PREFIX = {{ (prefix or 'RIP') | tojson }};
let scanMode = null;
let scanFiles = [];
let fileStats = null;
let wipeTarget = null;
function scanExt(name) {
const n = String(name || '');
const i = n.lastIndexOf('.');
return i >= 0 ? n.slice(i).toLowerCase() : '';
}
function fileBase(p) {
if (!p) return '';
const parts = String(p).replace(/\\/g, '/').split('/');
return parts[parts.length - 1] || '';
}
function setScanStatus(text, kind) {
const st = document.getElementById('rescan-status');
st.textContent = text;
@@ -556,6 +592,84 @@ function folderNameFrom(files) {
const top = p.replace(/\\/g, '/').split('/')[0];
return top || (files[0] && files[0].name) || 'папка';
}
function renderFileStats() {
const meta = document.getElementById('rescan-filemeta');
const wipeBtn = document.getElementById('rescan-wipe-btn');
const editorExtra = document.getElementById('editor-file-stats');
const n = fileStats ? (fileStats.count || 0) : 0;
const fileSelected = scanMode === 'file' && scanFiles[0];
if (!fileStats || !fileSelected) {
if (meta) meta.textContent = '';
if (wipeBtn) wipeBtn.hidden = true;
if (editorExtra) editorExtra.textContent = '';
return;
}
let line = 'Извлечения: ' + n;
if (n && fileStats.first && fileStats.last) {
line += fileStats.first === fileStats.last
? ' · ' + fileStats.first
: ' · ' + fileStats.first + ' – ' + fileStats.last;
}
if (meta) {
meta.textContent = line;
if (fileStats.ok === false && fileStats.issues && fileStats.issues.length) {
meta.textContent = line + ' · ' + fileStats.issues.join('; ');
meta.classList.add('warn');
} else {
meta.classList.remove('warn');
}
}
if (wipeBtn) {
wipeBtn.hidden = n === 0;
wipeBtn.disabled = false;
}
if (editorExtra) {
editorExtra.textContent = fileStats.file ? (' · ' + fileStats.file + ': ' + n) : '';
}
}
async function loadFileStats(name) {
if (!name) {
fileStats = null;
renderFileStats();
return;
}
try {
const url = '/api/file-extractions?file=' + encodeURIComponent(name)
+ '&prefix=' + encodeURIComponent(PAGE_PREFIX);
const res = await fetch(url);
const data = await res.json().catch(() => ({}));
if (!res.ok) {
const detail = data.detail;
throw new Error(typeof detail === 'string' ? detail : ('HTTP ' + res.status));
}
fileStats = data;
} catch (e) {
const rows = ALL.filter(r => fileBase(r.source_file).toLowerCase() === String(name).toLowerCase());
const codes = rows.map(r => r.code).filter(Boolean).sort();
fileStats = {
file: name,
count: rows.length,
first: codes[0] || null,
last: codes[codes.length - 1] || null,
ok: true,
issues: [],
};
}
renderFileStats();
}
async function refreshSections() {
const p = {{ (prefix or '') | tojson }};
const url = p ? '/api/sections?prefix=' + encodeURIComponent(p) : '/api/sections';
const res = await fetch(url);
if (!res.ok) throw new Error('HTTP ' + res.status);
ALL = await res.json();
document.getElementById('total-count').textContent = ALL.length + ' секции';
const q = document.getElementById('editor-search');
if (q && q.value) filterEditor();
else renderEditor(ALL);
renderKwCloud();
doSearch();
}
function setScanSelection(mode, files) {
scanMode = mode;
scanFiles = files;
@@ -569,18 +683,22 @@ function setScanSelection(mode, files) {
if (!mode) {
sel.textContent = '';
setScanStatus('Избери файл или папка.');
loadFileStats('');
return;
}
if (mode === 'file') {
if (!files[0]) {
sel.textContent = '';
setScanStatus('Избери файл.', 'error');
loadFileStats('');
return;
}
sel.textContent = 'Файл: ' + files[0].name;
setScanStatus('Готово за старт.');
loadFileStats(files[0].name);
return;
}
loadFileStats('');
if (!files.length) {
sel.textContent = 'Папка: няма подходящи файлове';
setScanStatus('Няма .zip, .docx, .html, .pdf или .txt в папката.', 'error');
@@ -620,17 +738,64 @@ function onScanFolderPicked() {
setScanSelection('folder', files);
}
function confirmWipeFile() {
const name = scanFiles[0] && scanFiles[0].name;
if (!name) return;
const n = (fileStats && fileStats.count) || 0;
wipeTarget = name;
document.getElementById('wipe-modal-body').textContent =
'Ще се изтрият всички извлечения за «' + name + '» (' + n + ' бр.). Самият help файл не се трие.';
document.getElementById('wipe-modal').classList.add('open');
}
function closeWipeModal() {
const modal = document.getElementById('wipe-modal');
if (modal) modal.classList.remove('open');
wipeTarget = null;
}
async function doWipeFile() {
const name = wipeTarget;
if (!name) return;
const btn = document.getElementById('wipe-confirm-btn');
btn.disabled = true;
try {
const token = localStorage.getItem('RESCAN_TOKEN') || '';
const headers = {};
if (token) headers['X-Rescan-Token'] = token;
let url = '/api/file-extractions?file=' + encodeURIComponent(name)
+ '&prefix=' + encodeURIComponent(PAGE_PREFIX);
if (token) url += '&token=' + encodeURIComponent(token);
const res = await fetch(url, { method: 'DELETE', headers });
const data = await res.json().catch(() => ({}));
if (!res.ok) {
const detail = data.detail;
throw new Error(typeof detail === 'string' ? detail : (data.message || ('HTTP ' + res.status)));
}
document.getElementById('wipe-modal').classList.remove('open');
wipeTarget = null;
await refreshSections();
await loadFileStats(name);
setScanStatus('Изтрити извлечения: ' + (data.deleted || 0) + ' за ' + name + '.', 'ok');
toast('Изтрити: ' + (data.deleted || 0));
} catch (e) {
toast(e.message, true);
setScanStatus('Грешка: ' + e.message, 'error');
} finally {
btn.disabled = false;
}
}
async function startRescan() {
const btn = document.getElementById('rescan-btn');
const fileBtn = document.getElementById('rescan-pick-file');
const folderBtn = document.getElementById('rescan-pick-folder');
const wipeBtn = document.getElementById('rescan-wipe-btn');
if (!scanFiles.length) {
setScanStatus(scanMode === 'folder' ? 'Няма подходящи файлове в папката.' : 'Избери файл или папка.', 'error');
return;
}
const fd = new FormData();
scanFiles.forEach(f => fd.append('file', f));
fd.append('prefix', {{ (prefix or 'RIP') | tojson }});
fd.append('prefix', PAGE_PREFIX);
fd.append('force', document.getElementById('rescan-force').checked ? 'true' : 'false');
const token = localStorage.getItem('RESCAN_TOKEN') || '';
if (token) fd.append('token', token);
@@ -638,6 +803,7 @@ async function startRescan() {
btn.disabled = true;
fileBtn.disabled = true;
folderBtn.disabled = true;
if (wipeBtn) wipeBtn.disabled = true;
setScanStatus('Качване...', 'busy');
try {
const headers = {};
@@ -657,6 +823,7 @@ async function startRescan() {
btn.disabled = false;
fileBtn.disabled = false;
folderBtn.disabled = false;
if (wipeBtn) wipeBtn.disabled = false;
}
}
@@ -667,8 +834,25 @@ async function pollRescan(jobId) {
const data = await res.json();
const line = data.status + (data.message ? ' — ' + data.message : '');
if (data.status === 'done') {
setScanStatus('Готово. Презареждане...', 'ok');
location.reload();
const files = data.files || [];
const bad = files.some(f => f && f.ok === false);
let msg = 'Готово.';
if (typeof data.sections === 'number') msg += ' Записани: ' + data.sections;
if (files.length === 1) {
const f = files[0];
msg = 'Готово. Извлечения: ' + (f.count || 0);
if (f.first && f.last) {
msg += f.first === f.last ? ' · ' + f.first : ' · ' + f.first + ' – ' + f.last;
}
if (f.ok === false && f.issues && f.issues.length) msg += ' · ' + f.issues.join('; ');
}
setScanStatus(msg, bad ? 'error' : 'ok');
try {
await refreshSections();
if (scanMode === 'file' && scanFiles[0]) await loadFileStats(scanFiles[0].name);
} catch (e) {
setScanStatus((msg || 'Готово.') + ' Презареди страницата.', bad ? 'error' : 'ok');
}
return;
}
if (data.status === 'error') {
@@ -964,7 +1148,10 @@ function closeSection() {
document.getElementById('section-modal').classList.remove('open');
}
document.addEventListener('keydown', (e) => {
if (e.key === 'Escape') closeSection();
if (e.key === 'Escape') {
closeWipeModal();
closeSection();
}
});
function toggleSelect(code, el) {