Auto schemas

This commit is contained in:
2026-05-26 22:34:28 +12:00
parent 35d70a7746
commit d4969172c2
7 changed files with 320 additions and 132 deletions
+115 -31
View File
@@ -9,8 +9,10 @@ returns metadata (no file contents are returned).
""" """
from __future__ import annotations from __future__ import annotations
import csv
import glob import glob
import os import os
import xml.etree.ElementTree as ET
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@@ -153,23 +155,96 @@ async def update_config(reference: str, request: Request):
) )
# ── Schema inference helpers ────────────────────────────────────────────────
_DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%Y%m%d", "%d-%m-%Y"]
_DATETIME_FORMATS = ["%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"]
def _infer_type(values: list[str]) -> str:
sample = [v.strip() for v in values if v.strip()][:30]
if not sample:
return "str"
try:
[int(v.replace(",", "")) for v in sample]
return "int"
except ValueError:
pass
try:
[float(v.replace(",", "")) for v in sample]
return "float"
except ValueError:
pass
for fmt in _DATETIME_FORMATS:
try:
[datetime.strptime(v, fmt) for v in sample]
return f"datetime('{fmt}')"
except ValueError:
pass
for fmt in _DATE_FORMATS:
try:
[datetime.strptime(v, fmt) for v in sample]
return f"date('{fmt}')"
except ValueError:
pass
return "str"
def _csv_schema(path: str, delimiter: str = ",", has_header: bool = True) -> dict[str, str]:
try:
with open(path, newline="", encoding="utf-8", errors="replace") as f:
reader = csv.reader(f, delimiter=delimiter)
rows = [r for _, r in zip(range(31), reader)]
if not rows:
return {}
headers = rows[0] if has_header else [f"col_{i}" for i in range(len(rows[0]))]
data_rows = rows[1:] if has_header else rows
return {
col: _infer_type([r[i] for r in data_rows if i < len(r)])
for i, col in enumerate(headers)
if col.strip()
}
except Exception:
return {}
def _xml_schema(path: str, xpathstr: str = "./*") -> dict[str, str]:
try:
tree = ET.parse(path)
root = tree.getroot()
elements = root.findall(xpathstr) or list(root)
if not elements:
return {}
schema: dict[str, str] = {}
for el in elements[:1]:
for child in el:
schema[child.tag] = "str"
for attr in el.attrib:
schema[f"@{attr}"] = "str"
return schema
except Exception:
return {}
# ── Test URL ──────────────────────────────────────────────────────────────── # ── Test URL ────────────────────────────────────────────────────────────────
@router.post("/test-url") @router.post("/test-url")
async def test_url(request: Request): async def test_url(request: Request):
"""Return metadata for a URL pattern — no file contents ever returned.""" """Return file/DB metadata and inferred schema — no row data ever returned."""
from datetime import date as date_type
body = await request.json() body = await request.json()
url: str = body.get("url", "") url: str = body.get("url", "")
as_at_date: str = body.get("as_at_date", datetime.now().strftime("%Y%m%d")) as_at_date: str = body.get("as_at_date", datetime.now().strftime("%Y%m%d"))
csv_spec: dict = body.get("csv_spec", {})
# ── DB URL — schema inspection wired up per-connector when available ──────
if not url.startswith("file://"): if not url.startswith("file://"):
return {"type": "non-file", "message": "Only file:// URLs can be tested here. Database connections are validated at run time."} scheme = url.split("://")[0] if "://" in url else url
return {"type": "db", "scheme": scheme, "schema": {}, "found": False}
# Resolve the path part (strip file://) # ── file:// URL ──────────────────────────────────────────────────────────
raw_path = url[7:] raw_path = url[7:]
# Replace simple template variables for preview purposes
from datetime import date as date_type
try: try:
as_at = datetime.strptime(as_at_date, "%Y%m%d").date() as_at = datetime.strptime(as_at_date, "%Y%m%d").date()
except ValueError: except ValueError:
@@ -183,40 +258,49 @@ async def test_url(request: Request):
.replace("{{today.strftime('%Y-%m-%d')}}", date_type.today().strftime("%Y-%m-%d")) .replace("{{today.strftime('%Y-%m-%d')}}", date_type.today().strftime("%Y-%m-%d"))
) )
# Treat as glob pattern for anything still containing { def _file_info(p: str) -> dict:
if "*" in resolved or "?" in resolved or "{" in resolved:
matches = glob.glob(resolved)
if not matches:
return {"type": "file", "resolved": resolved, "found": False, "message": "No files matched the pattern."}
file_infos = []
for p in sorted(matches)[:10]:
try: try:
st = os.stat(p) st = os.stat(p)
file_infos.append({ return {"path": p, "size_bytes": st.st_size,
"path": p, "modified": datetime.fromtimestamp(st.st_mtime).isoformat(timespec="seconds")}
"size_bytes": st.st_size,
"modified": datetime.fromtimestamp(st.st_mtime).isoformat(timespec="seconds"),
})
except OSError: except OSError:
file_infos.append({"path": p, "error": "Could not stat file"}) return {"path": p, "error": "Could not stat file"}
return {"type": "file", "resolved": resolved, "found": True, "matches": len(matches), "files": file_infos}
def _schema_for(p: str) -> dict[str, str]:
pl = p.lower()
if pl.endswith(".xml"):
return _xml_schema(p)
delimiter = csv_spec.get("delimiter", ",") or ","
has_header = csv_spec.get("has_header", True)
return _csv_schema(p, delimiter=delimiter, has_header=has_header)
# Glob pattern
if "*" in resolved or "?" in resolved or "{" in resolved:
matches = sorted(glob.glob(resolved))
if not matches:
return {"type": "file", "resolved": resolved, "found": False,
"message": "No files matched the pattern.", "schema": {}}
schema = _schema_for(matches[0])
return {
"type": "file", "resolved": resolved, "found": True,
"matches": len(matches),
"files": [_file_info(p) for p in matches[:10]],
"schema": schema,
}
# Exact path # Exact path
p = Path(resolved) p = Path(resolved)
if not p.exists(): if not p.exists():
return {"type": "file", "resolved": resolved, "found": False, "message": f"File not found: {resolved}"} return {"type": "file", "resolved": resolved, "found": False,
"message": f"File not found: {resolved}", "schema": {}}
try: try:
st = p.stat() schema = _schema_for(str(p))
return { return {
"type": "file", "type": "file", "resolved": resolved, "found": True,
"resolved": resolved,
"found": True,
"matches": 1, "matches": 1,
"files": [{ "files": [_file_info(str(p))],
"path": str(p), "schema": schema,
"size_bytes": st.st_size,
"modified": datetime.fromtimestamp(st.st_mtime).isoformat(timespec="seconds"),
}],
} }
except OSError as e: except OSError as e:
return {"type": "file", "resolved": resolved, "found": False, "message": str(e)} return {"type": "file", "resolved": resolved, "found": False,
"message": str(e), "schema": {}}
+2
View File
@@ -10,6 +10,7 @@ from app.api.configs import router as configs_router
from app.core.settings import get_settings from app.core.settings import get_settings
from app.views.auth import router as auth_router from app.views.auth import router as auth_router
from app.views.views import router as dashboard_router from app.views.views import router as dashboard_router
from app.views.config_views import router as config_views_router
from app.views.docs import router as docs_router from app.views.docs import router as docs_router
@@ -38,5 +39,6 @@ def create_app() -> FastAPI:
app.include_router(configs_router) app.include_router(configs_router)
app.include_router(auth_router) app.include_router(auth_router)
app.include_router(dashboard_router) app.include_router(dashboard_router)
app.include_router(config_views_router)
app.include_router(docs_router) app.include_router(docs_router)
return app return app
+68
View File
@@ -0,0 +1,68 @@
"""Config list and editor views."""
from pathlib import Path
from datetime import datetime
from fastapi import APIRouter, Request
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
from app.core.refdata import ReconPatterns, ReconConfigStatus
from app.service.fake_configs import get_fake_configs
router = APIRouter(prefix="", tags=["Config_UI"])
_project_root = Path(__file__).resolve().parents[2]
templates = Jinja2Templates(directory=str(_project_root / "data" / "templates"))
_EDITOR_CONTEXT = {
"patterns": [p.value for p in ReconPatterns],
"statuses": [s.value for s in ReconConfigStatus],
"frequencies": ["Ad Hoc", "Intra Day", "Daily", "Weekly", "Monthly", "Quarterly"],
"schema_types": ["str", "int", "float", "date('%Y-%m-%d')", "datetime('%Y-%m-%d %H:%M:%S')", "bool"],
}
@router.get("/configs", response_class=HTMLResponse)
async def configs_list_view(request: Request):
now = datetime.now()
configs = get_fake_configs(now)
return templates.TemplateResponse(
request=request,
name="configs_list.html",
context={
"title": "Recon Configs",
"configs": configs,
},
)
@router.get("/configs/new", response_class=HTMLResponse)
async def config_new_view(request: Request):
return templates.TemplateResponse(
request=request,
name="config_editor.html",
context={
"title": "New Config",
"config": None,
"is_new": True,
**_EDITOR_CONTEXT,
},
)
@router.get("/configs/{reference}/edit", response_class=HTMLResponse)
async def config_edit_view(request: Request, reference: str):
now = datetime.now()
configs = get_fake_configs(now)
config = next((c for c in configs if c.reference == reference), None)
return templates.TemplateResponse(
request=request,
name="config_editor.html",
context={
"title": f"Edit — {reference}" if config else "Edit Config",
"config": config,
"reference": reference,
"is_new": False,
**_EDITOR_CONTEXT,
},
)
-56
View File
@@ -14,7 +14,6 @@ from fastapi.templating import Jinja2Templates
from app.models.recon_job import ReconJob # noqa: F401 (validates that the real model imports cleanly) from app.models.recon_job import ReconJob # noqa: F401 (validates that the real model imports cleanly)
from app.service.fake_jobs import FakeReconJob, get_fake_jobs from app.service.fake_jobs import FakeReconJob, get_fake_jobs
from app.service.fake_configs import FakeReconConfig, get_fake_configs from app.service.fake_configs import FakeReconConfig, get_fake_configs
from app.core.refdata import ReconPatterns, ReconConfigStatus
router = APIRouter(prefix="", tags=["User_Interface"]) router = APIRouter(prefix="", tags=["User_Interface"])
@@ -507,58 +506,3 @@ async def job_detail_view(request: Request, job_id: int):
}, },
) )
# ── Config editor ────────────────────────────────────────────────────────────
_EDITOR_CONTEXT = {
"patterns": [p.value for p in ReconPatterns],
"statuses": [s.value for s in ReconConfigStatus],
"frequencies": ["Ad Hoc", "Intra Day", "Daily", "Weekly", "Monthly", "Quarterly"],
"schema_types": ["str", "int", "float", "date('%Y-%m-%d')", "datetime('%Y-%m-%d %H:%M:%S')", "bool"],
}
@router.get("/configs/new", response_class=HTMLResponse)
async def config_new_view(request: Request):
return templates.TemplateResponse(
request=request,
name="config_editor.html",
context={
"title": "New Config",
"config": None,
"is_new": True,
**_EDITOR_CONTEXT,
},
)
@router.get("/configs/{reference}/edit", response_class=HTMLResponse)
async def config_edit_view(request: Request, reference: str):
now = datetime.now()
configs = get_fake_configs(now)
config = next((c for c in configs if c.reference == reference), None)
return templates.TemplateResponse(
request=request,
name="config_editor.html",
context={
"title": f"Edit — {reference}" if config else "Edit Config",
"config": config,
"reference": reference,
"is_new": False,
**_EDITOR_CONTEXT,
},
)
@router.get("/configs", response_class=HTMLResponse)
async def configs_list_view(request: Request):
now = datetime.now()
configs = get_fake_configs(now)
return templates.TemplateResponse(
request=request,
name="configs_list.html",
context={
"title": "Recon Configs",
"configs": configs,
},
)
+6
View File
@@ -0,0 +1,6 @@
account_id,customer_name,transaction_date,amount,currency,status,reference
ACC001,Alice Nguyen,2026-05-26,1500.00,NZD,MATCHED,REF-00001
ACC002,Bob Tane,2026-05-26,820.50,NZD,UNMATCHED,REF-00002
ACC003,Carol Smith,2026-05-26,3200.00,NZD,MATCHED,REF-00003
ACC004,David Park,2026-05-26,415.75,NZD,PENDING,REF-00004
ACC005,Eva Brown,2026-05-26,9900.00,NZD,MATCHED,REF-00005
1 account_id customer_name transaction_date amount currency status reference
2 ACC001 Alice Nguyen 2026-05-26 1500.00 NZD MATCHED REF-00001
3 ACC002 Bob Tane 2026-05-26 820.50 NZD UNMATCHED REF-00002
4 ACC003 Carol Smith 2026-05-26 3200.00 NZD MATCHED REF-00003
5 ACC004 David Park 2026-05-26 415.75 NZD PENDING REF-00004
6 ACC005 Eva Brown 2026-05-26 9900.00 NZD MATCHED REF-00005
+11 -3
View File
@@ -911,7 +911,8 @@ a.job-bar:hover { filter: brightness(1.1); box-shadow: 0 0 0 1px rgba(255,255,25
gap: 8px; gap: 8px;
align-items: stretch; align-items: stretch;
} }
.url-input { flex: 1; } .url-input-row .url-scheme { flex-shrink: 0; width: auto; }
.url-input-row .url-path { flex: 1; width: 0; min-width: 0; }
.url-test-result { .url-test-result {
margin-top: 6px; margin-top: 6px;
padding: 8px 10px; padding: 8px 10px;
@@ -930,7 +931,7 @@ a.job-bar:hover { filter: brightness(1.1); box-shadow: 0 0 0 1px rgba(255,255,25
.schema-builder { display: flex; flex-direction: column; gap: 6px; } .schema-builder { display: flex; flex-direction: column; gap: 6px; }
.kv-table-header { .kv-table-header {
display: grid; display: grid;
grid-template-columns: 1fr 1fr auto; grid-template-columns: 1fr 1fr auto auto;
gap: 8px; gap: 8px;
font-size: 10px; font-size: 10px;
text-transform: uppercase; text-transform: uppercase;
@@ -940,10 +941,17 @@ a.job-bar:hover { filter: brightness(1.1); box-shadow: 0 0 0 1px rgba(255,255,25
} }
.schema-row { .schema-row {
display: grid; display: grid;
grid-template-columns: 1fr 1fr auto; grid-template-columns: 1fr 1fr auto auto;
gap: 8px; gap: 8px;
align-items: center; align-items: center;
} }
.schema-row .idx-check {
width: 16px;
height: 16px;
accent-color: var(--accent);
cursor: pointer;
justify-self: center;
}
.schema-row input, .schema-row input,
.schema-row select { .schema-row select {
border-radius: 5px; border-radius: 5px;
+117 -41
View File
@@ -176,21 +176,22 @@
<div class="editor-field editor-field-full"> <div class="editor-field editor-field-full">
<label>URL <span class="req">*</span></label> <label>URL <span class="req">*</span></label>
<div class="url-input-row"> <div class="url-input-row">
<input type="text" data-field="url" placeholder="file:// or mssql:// or oracle:// ..." class="url-input"> <select class="url-scheme">
<button type="button" class="btn btn-secondary btn-sm btn-test-url">Try URL</button> <option value="file://">file://</option>
<option value="mssql://">mssql://</option>
<option value="oracle://">oracle://</option>
<option value="snowflake://">snowflake://</option>
<option value="duckdb://">duckdb://</option>
</select>
<input type="text" class="url-path" placeholder="/path/to/file or SERVER?db=DB&amp;table=TABLE">
<button type="button" class="btn btn-secondary btn-sm btn-test-url">Inspect URL</button>
</div> </div>
<span class="field-hint"> <span class="field-hint">
Supports: <code>file://</code> <code>mssql://</code> <code>oracle://</code> <code>snowflake://</code> <code>duckdb://</code><br>
Date templates: {% raw %}<code>{{as_at_date.strftime('%Y%m%d')}}</code> <code>{{today.strftime('%Y%m%d')}}</code>{% endraw %} Date templates: {% raw %}<code>{{as_at_date.strftime('%Y%m%d')}}</code> <code>{{today.strftime('%Y%m%d')}}</code>{% endraw %}
</span> </span>
<div class="url-test-result" hidden></div> <div class="url-test-result" hidden></div>
</div> </div>
<div class="editor-field editor-field-full">
<label>SQL Override</label>
<textarea data-field="sql" rows="2" placeholder="Full SQL query — replaces auto-generated SELECT. Required for duckdb://."></textarea>
</div>
<div class="editor-field editor-field-wide"> <div class="editor-field editor-field-wide">
<label>Comment</label> <label>Comment</label>
<input type="text" data-field="comment" placeholder="Optional note about this system."> <input type="text" data-field="comment" placeholder="Optional note about this system.">
@@ -203,7 +204,7 @@
<summary>Schema <span class="syscfg-section-count schema-count">0 columns</span></summary> <summary>Schema <span class="syscfg-section-count schema-count">0 columns</span></summary>
<div class="schema-builder"> <div class="schema-builder">
<div class="kv-table-header"> <div class="kv-table-header">
<span>Column Name</span><span>Type</span><span></span> <span>Column Name</span><span>Type</span><span>Index</span><span></span>
</div> </div>
<div class="schema-rows"></div> <div class="schema-rows"></div>
<button type="button" class="btn btn-secondary btn-sm btn-add-schema-row">+ Add Column</button> <button type="button" class="btn btn-secondary btn-sm btn-add-schema-row">+ Add Column</button>
@@ -263,8 +264,8 @@
<summary>Advanced</summary> <summary>Advanced</summary>
<div class="syscfg-grid"> <div class="syscfg-grid">
<div class="editor-field editor-field-full"> <div class="editor-field editor-field-full">
<label>Index Fields</label> <label>SQL Override</label>
<input type="text" data-field="index_fields" placeholder="Comma-separated field names used as join keys."> <textarea data-field="sql" rows="2" placeholder="Full SQL query — replaces auto-generated SELECT. Required for duckdb://."></textarea>
</div> </div>
<div class="editor-field editor-field-full"> <div class="editor-field editor-field-full">
<label>Obfuscate Fields</label> <label>Obfuscate Fields</label>
@@ -313,7 +314,7 @@ function buildSyscfgCard(data, role, index) {
} }
// Populate simple fields // Populate simple fields
const fields = ['name','url','day_offset','filter','sql','comment']; const fields = ['name','day_offset','filter','sql','comment'];
fields.forEach(f => { fields.forEach(f => {
const el = card.querySelector(`[data-field="${f}"]`); const el = card.querySelector(`[data-field="${f}"]`);
if (!el) return; if (!el) return;
@@ -321,17 +322,31 @@ function buildSyscfgCard(data, role, index) {
if (val !== undefined && val !== null) el.value = val; if (val !== undefined && val !== null) el.value = val;
}); });
// Split stored URL into scheme dropdown + path input
if (data.url) {
const m = data.url.match(/^([a-z]+:\/\/)([\s\S]*)$/i);
if (m) {
const schemeEl = card.querySelector('.url-scheme');
const matchingOpt = Array.from(schemeEl.options).find(o => o.value === m[1].toLowerCase());
if (matchingOpt) schemeEl.value = matchingOpt.value;
card.querySelector('.url-path').value = m[2];
} else {
card.querySelector('.url-path').value = data.url;
}
}
// Populate list fields (comma-joined) // Populate list fields (comma-joined)
['index_fields','obfuscate_fields','pci_redact_fields'].forEach(f => { ['obfuscate_fields','pci_redact_fields'].forEach(f => {
const el = card.querySelector(`[data-field="${f}"]`); const el = card.querySelector(`[data-field="${f}"]`);
if (el && Array.isArray(data[f])) el.value = data[f].join(', '); if (el && Array.isArray(data[f])) el.value = data[f].join(', ');
}); });
const fwEl = card.querySelector('[data-field="field_widths"]'); const fwEl = card.querySelector('[data-field="field_widths"]');
if (fwEl && Array.isArray(data.field_widths)) fwEl.value = data.field_widths.join(', '); if (fwEl && Array.isArray(data.field_widths)) fwEl.value = data.field_widths.join(', ');
// Schema // Schema — tick index columns derived from index_fields list
const schemaData = data.system_schema || {}; const schemaData = data.system_schema || {};
Object.entries(schemaData).forEach(([col, type]) => addSchemaRow(card, col, type)); const indexSet = new Set(data.index_fields || []);
Object.entries(schemaData).forEach(([col, type]) => addSchemaRow(card, col, type, indexSet.has(col)));
updateSchemaCount(card); updateSchemaCount(card);
// CSV spec // CSV spec
@@ -353,11 +368,12 @@ function buildSyscfgCard(data, role, index) {
} }
// URL change → show/hide specs, clear test result // URL change → show/hide specs, clear test result
const urlInput = card.querySelector('.url-input'); const onUrlChange = () => {
urlInput.addEventListener('input', () => {
syncSpecVisibility(card); syncSpecVisibility(card);
card.querySelector('.url-test-result').setAttribute('hidden', ''); card.querySelector('.url-test-result').setAttribute('hidden', '');
}); };
card.querySelector('.url-scheme').addEventListener('change', onUrlChange);
card.querySelector('.url-path').addEventListener('input', onUrlChange);
syncSpecVisibility(card); syncSpecVisibility(card);
// Schema add button // Schema add button
@@ -382,15 +398,16 @@ function setSpecCheckbox(card, spec, field, checked) {
} }
function syncSpecVisibility(card) { function syncSpecVisibility(card) {
const url = (card.querySelector('.url-input').value || '').toLowerCase(); const scheme = card.querySelector('.url-scheme').value;
const isFile = url.startsWith('file://'); const path = (card.querySelector('.url-path').value || '').toLowerCase();
const isXml = isFile && /\.xml(\b|$|\?|#)/.test(url); const isFile = scheme === 'file://';
const isXml = isFile && /\.xml(\b|$|\?|#)/.test(path);
const isCsv = isFile && !isXml; const isCsv = isFile && !isXml;
card.querySelector('.syscfg-csv-spec').toggleAttribute('hidden', !isCsv); card.querySelector('.syscfg-csv-spec').toggleAttribute('hidden', !isCsv);
card.querySelector('.syscfg-xml-spec').toggleAttribute('hidden', !isXml); card.querySelector('.syscfg-xml-spec').toggleAttribute('hidden', !isXml);
} }
function addSchemaRow(card, colName, colType) { function addSchemaRow(card, colName, colType, isIndex) {
const container = card.querySelector('.schema-rows'); const container = card.querySelector('.schema-rows');
const row = document.createElement('div'); const row = document.createElement('div');
row.className = 'schema-row'; row.className = 'schema-row';
@@ -409,13 +426,19 @@ function addSchemaRow(card, colName, colType) {
typeSelect.appendChild(opt); typeSelect.appendChild(opt);
}); });
const idxCheck = document.createElement('input');
idxCheck.type = 'checkbox';
idxCheck.className = 'idx-check';
idxCheck.checked = !!isIndex;
idxCheck.title = 'Use as index / join key';
const removeBtn = document.createElement('button'); const removeBtn = document.createElement('button');
removeBtn.type = 'button'; removeBtn.type = 'button';
removeBtn.className = 'btn btn-secondary btn-sm'; removeBtn.className = 'btn btn-secondary btn-sm';
removeBtn.textContent = '×'; removeBtn.textContent = '×';
removeBtn.addEventListener('click', () => { row.remove(); updateSchemaCount(card); }); removeBtn.addEventListener('click', () => { row.remove(); updateSchemaCount(card); });
row.append(colInput, typeSelect, removeBtn); row.append(colInput, typeSelect, idxCheck, removeBtn);
container.appendChild(row); container.appendChild(row);
} }
@@ -442,16 +465,19 @@ function readSyscfgCard(card) {
const toList = (val) => val.split(',').map(s => s.trim()).filter(Boolean); const toList = (val) => val.split(',').map(s => s.trim()).filter(Boolean);
const toIntList = (val) => val.split(',').map(s => parseInt(s.trim(), 10)).filter(n => !isNaN(n)); const toIntList = (val) => val.split(',').map(s => parseInt(s.trim(), 10)).filter(n => !isNaN(n));
const url = g('url'); const scheme = card.querySelector('.url-scheme').value;
const urlLower = url.toLowerCase(); const url = scheme + (card.querySelector('.url-path').value || '');
const isFile = urlLower.startsWith('file://'); const isFile = scheme === 'file://';
const isXml = isFile && /\.xml(\b|$)/.test(urlLower); const isXml = isFile && /\.xml(\b|$)/.test(url.toLowerCase());
const schema = {}; const schema = {};
const index_fields = [];
card.querySelectorAll('.schema-rows .schema-row').forEach(row => { card.querySelectorAll('.schema-rows .schema-row').forEach(row => {
const name = row.querySelector('input').value.trim(); const name = row.querySelector('input[type="text"]').value.trim();
const type = row.querySelector('select').value; const type = row.querySelector('select').value;
if (name) schema[name] = type; if (!name) return;
schema[name] = type;
if (row.querySelector('.idx-check')?.checked) index_fields.push(name);
}); });
const spec = (specName, fields) => { const spec = (specName, fields) => {
@@ -483,7 +509,7 @@ function readSyscfgCard(card) {
system_schema: schema, system_schema: schema,
filter: g('filter'), filter: g('filter'),
sql: g('sql'), sql: g('sql'),
index_fields: toList(g('index_fields')), index_fields,
obfuscate_fields: toList(g('obfuscate_fields')), obfuscate_fields: toList(g('obfuscate_fields')),
pci_redact_fields: toList(g('pci_redact_fields')), pci_redact_fields: toList(g('pci_redact_fields')),
field_widths: toIntList(g('field_widths')), field_widths: toIntList(g('field_widths')),
@@ -556,34 +582,39 @@ function collectPayload() {
// ── Try URL ───────────────────────────────────────────────────────────────── // ── Try URL ─────────────────────────────────────────────────────────────────
async function testUrl(card) { async function testUrl(card) {
const url = card.querySelector('.url-input').value.trim(); const url = card.querySelector('.url-scheme').value + card.querySelector('.url-path').value.trim();
const resultEl = card.querySelector('.url-test-result'); const resultEl = card.querySelector('.url-test-result');
if (!url) { showToast('Enter a URL first.', 'warn'); return; } if (!url) { showToast('Enter a URL first.', 'warn'); return; }
const btn = card.querySelector('.btn-test-url'); const btn = card.querySelector('.btn-test-url');
btn.disabled = true; btn.textContent = 'Checking…'; btn.disabled = true; btn.textContent = 'Inspecting…';
resultEl.removeAttribute('hidden'); resultEl.removeAttribute('hidden');
resultEl.className = 'url-test-result url-test-loading'; resultEl.className = 'url-test-result url-test-loading';
resultEl.textContent = 'Checking URL…'; resultEl.textContent = 'Inspecting URL…';
const csvSpec = {
delimiter: card.querySelector('[data-spec="csv"][data-field="delimiter"]')?.value || ',',
has_header: card.querySelector('[data-spec="csv"][data-field="header"]')?.checked !== false,
};
try { try {
const resp = await fetch('/api/configs/test-url', { const resp = await fetch('/api/configs/test-url', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ url }), body: JSON.stringify({ url, csv_spec: csvSpec }),
}); });
const data = await resp.json(); const data = await resp.json();
renderUrlTestResult(resultEl, data); renderUrlTestResult(resultEl, data, card);
} catch (err) { } catch (err) {
resultEl.className = 'url-test-result url-test-error'; resultEl.className = 'url-test-result url-test-error';
resultEl.textContent = `Error: ${err.message}`; resultEl.textContent = `Error: ${err.message}`;
} finally { } finally {
btn.disabled = false; btn.textContent = 'Try URL'; btn.disabled = false; btn.textContent = 'Inspect URL';
} }
} }
function renderUrlTestResult(el, data) { function renderUrlTestResult(el, data, card) {
if (data.type === 'non-file') { el.textContent = '';
el.className = 'url-test-result url-test-info';
el.textContent = data.message; if (data.type === 'db') {
el.setAttribute('hidden', '');
return; return;
} }
if (!data.found) { if (!data.found) {
@@ -591,14 +622,59 @@ function renderUrlTestResult(el, data) {
el.textContent = `Not found: ${data.message || data.resolved}`; el.textContent = `Not found: ${data.message || data.resolved}`;
return; return;
} }
el.className = 'url-test-result url-test-ok'; el.className = 'url-test-result url-test-ok';
// File stats
const lines = [`${data.matches} file${data.matches !== 1 ? 's' : ''} matched`]; const lines = [`${data.matches} file${data.matches !== 1 ? 's' : ''} matched`];
(data.files || []).forEach(f => { (data.files || []).forEach(f => {
if (f.error) { lines.push(`${f.path}: ${f.error}`); return; } if (f.error) { lines.push(`${f.path}: ${f.error}`); return; }
const kb = (f.size_bytes / 1024).toFixed(1); const kb = (f.size_bytes / 1024).toFixed(1);
lines.push(` ${f.path.split('/').pop()} ${kb} KB (modified ${f.modified})`); lines.push(` ${f.path.split('/').pop()} ${kb} KB (modified ${f.modified})`);
}); });
el.textContent = lines.join('\n'); const pre = document.createElement('span');
pre.textContent = lines.join('\n');
el.appendChild(pre);
// Schema
const schema = data.schema || {};
const colCount = Object.keys(schema).length;
if (colCount > 0) {
const schemaLine = document.createElement('div');
schemaLine.style.cssText = 'margin-top:6px; display:flex; align-items:center; gap:10px; flex-wrap:wrap;';
const label = document.createElement('span');
label.textContent = ` ${colCount} column${colCount !== 1 ? 's' : ''} detected`;
const applyBtn = document.createElement('button');
applyBtn.type = 'button';
applyBtn.className = 'btn btn-primary btn-sm';
applyBtn.textContent = 'Apply Schema';
applyBtn.addEventListener('click', () => {
applySchema(card, schema);
applyBtn.textContent = '✓ Applied';
applyBtn.disabled = true;
});
schemaLine.append(label, applyBtn);
el.appendChild(schemaLine);
}
}
function applySchema(card, schema) {
const container = card.querySelector('.schema-rows');
// Preserve existing index ticks by column name before clearing
const existingIndex = new Set();
container.querySelectorAll('.schema-row').forEach(row => {
if (row.querySelector('.idx-check')?.checked)
existingIndex.add(row.querySelector('input[type="text"]').value.trim());
});
container.innerHTML = '';
Object.entries(schema).forEach(([col, type]) =>
addSchemaRow(card, col, type, existingIndex.has(col))
);
updateSchemaCount(card);
// Open the schema section so the user sees it
card.querySelector('.syscfg-section details, details.syscfg-section')?.setAttribute('open', '');
card.querySelector('.schema-builder')?.closest('details')?.setAttribute('open', '');
showToast(`Schema applied — ${Object.keys(schema).length} columns.`, 'ok');
} }
// ── Toast ─────────────────────────────────────────────────────────────────── // ── Toast ───────────────────────────────────────────────────────────────────