Auto schemas
This commit is contained in:
@@ -176,21 +176,22 @@
|
||||
<div class="editor-field editor-field-full">
|
||||
<label>URL <span class="req">*</span></label>
|
||||
<div class="url-input-row">
|
||||
<input type="text" data-field="url" placeholder="file:// or mssql:// or oracle:// ..." class="url-input">
|
||||
<button type="button" class="btn btn-secondary btn-sm btn-test-url">Try URL</button>
|
||||
<select class="url-scheme">
|
||||
<option value="file://">file://</option>
|
||||
<option value="mssql://">mssql://</option>
|
||||
<option value="oracle://">oracle://</option>
|
||||
<option value="snowflake://">snowflake://</option>
|
||||
<option value="duckdb://">duckdb://</option>
|
||||
</select>
|
||||
<input type="text" class="url-path" placeholder="/path/to/file or SERVER?db=DB&table=TABLE">
|
||||
<button type="button" class="btn btn-secondary btn-sm btn-test-url">Inspect URL</button>
|
||||
</div>
|
||||
<span class="field-hint">
|
||||
Supports: <code>file://</code> <code>mssql://</code> <code>oracle://</code> <code>snowflake://</code> <code>duckdb://</code><br>
|
||||
Date templates: {% raw %}<code>{{as_at_date.strftime('%Y%m%d')}}</code> <code>{{today.strftime('%Y%m%d')}}</code>{% endraw %}
|
||||
</span>
|
||||
<div class="url-test-result" hidden></div>
|
||||
</div>
|
||||
|
||||
<div class="editor-field editor-field-full">
|
||||
<label>SQL Override</label>
|
||||
<textarea data-field="sql" rows="2" placeholder="Full SQL query — replaces auto-generated SELECT. Required for duckdb://."></textarea>
|
||||
</div>
|
||||
|
||||
<div class="editor-field editor-field-wide">
|
||||
<label>Comment</label>
|
||||
<input type="text" data-field="comment" placeholder="Optional note about this system.">
|
||||
@@ -203,7 +204,7 @@
|
||||
<summary>Schema <span class="syscfg-section-count schema-count">0 columns</span></summary>
|
||||
<div class="schema-builder">
|
||||
<div class="kv-table-header">
|
||||
<span>Column Name</span><span>Type</span><span></span>
|
||||
<span>Column Name</span><span>Type</span><span>Index</span><span></span>
|
||||
</div>
|
||||
<div class="schema-rows"></div>
|
||||
<button type="button" class="btn btn-secondary btn-sm btn-add-schema-row">+ Add Column</button>
|
||||
@@ -263,8 +264,8 @@
|
||||
<summary>Advanced</summary>
|
||||
<div class="syscfg-grid">
|
||||
<div class="editor-field editor-field-full">
|
||||
<label>Index Fields</label>
|
||||
<input type="text" data-field="index_fields" placeholder="Comma-separated field names used as join keys.">
|
||||
<label>SQL Override</label>
|
||||
<textarea data-field="sql" rows="2" placeholder="Full SQL query — replaces auto-generated SELECT. Required for duckdb://."></textarea>
|
||||
</div>
|
||||
<div class="editor-field editor-field-full">
|
||||
<label>Obfuscate Fields</label>
|
||||
@@ -313,7 +314,7 @@ function buildSyscfgCard(data, role, index) {
|
||||
}
|
||||
|
||||
// Populate simple fields
|
||||
const fields = ['name','url','day_offset','filter','sql','comment'];
|
||||
const fields = ['name','day_offset','filter','sql','comment'];
|
||||
fields.forEach(f => {
|
||||
const el = card.querySelector(`[data-field="${f}"]`);
|
||||
if (!el) return;
|
||||
@@ -321,17 +322,31 @@ function buildSyscfgCard(data, role, index) {
|
||||
if (val !== undefined && val !== null) el.value = val;
|
||||
});
|
||||
|
||||
// Split stored URL into scheme dropdown + path input
|
||||
if (data.url) {
|
||||
const m = data.url.match(/^([a-z]+:\/\/)([\s\S]*)$/i);
|
||||
if (m) {
|
||||
const schemeEl = card.querySelector('.url-scheme');
|
||||
const matchingOpt = Array.from(schemeEl.options).find(o => o.value === m[1].toLowerCase());
|
||||
if (matchingOpt) schemeEl.value = matchingOpt.value;
|
||||
card.querySelector('.url-path').value = m[2];
|
||||
} else {
|
||||
card.querySelector('.url-path').value = data.url;
|
||||
}
|
||||
}
|
||||
|
||||
// Populate list fields (comma-joined)
|
||||
['index_fields','obfuscate_fields','pci_redact_fields'].forEach(f => {
|
||||
['obfuscate_fields','pci_redact_fields'].forEach(f => {
|
||||
const el = card.querySelector(`[data-field="${f}"]`);
|
||||
if (el && Array.isArray(data[f])) el.value = data[f].join(', ');
|
||||
});
|
||||
const fwEl = card.querySelector('[data-field="field_widths"]');
|
||||
if (fwEl && Array.isArray(data.field_widths)) fwEl.value = data.field_widths.join(', ');
|
||||
|
||||
// Schema
|
||||
// Schema — tick index columns derived from index_fields list
|
||||
const schemaData = data.system_schema || {};
|
||||
Object.entries(schemaData).forEach(([col, type]) => addSchemaRow(card, col, type));
|
||||
const indexSet = new Set(data.index_fields || []);
|
||||
Object.entries(schemaData).forEach(([col, type]) => addSchemaRow(card, col, type, indexSet.has(col)));
|
||||
updateSchemaCount(card);
|
||||
|
||||
// CSV spec
|
||||
@@ -353,11 +368,12 @@ function buildSyscfgCard(data, role, index) {
|
||||
}
|
||||
|
||||
// URL change → show/hide specs, clear test result
|
||||
const urlInput = card.querySelector('.url-input');
|
||||
urlInput.addEventListener('input', () => {
|
||||
const onUrlChange = () => {
|
||||
syncSpecVisibility(card);
|
||||
card.querySelector('.url-test-result').setAttribute('hidden', '');
|
||||
});
|
||||
};
|
||||
card.querySelector('.url-scheme').addEventListener('change', onUrlChange);
|
||||
card.querySelector('.url-path').addEventListener('input', onUrlChange);
|
||||
syncSpecVisibility(card);
|
||||
|
||||
// Schema add button
|
||||
@@ -382,15 +398,16 @@ function setSpecCheckbox(card, spec, field, checked) {
|
||||
}
|
||||
|
||||
function syncSpecVisibility(card) {
|
||||
const url = (card.querySelector('.url-input').value || '').toLowerCase();
|
||||
const isFile = url.startsWith('file://');
|
||||
const isXml = isFile && /\.xml(\b|$|\?|#)/.test(url);
|
||||
const scheme = card.querySelector('.url-scheme').value;
|
||||
const path = (card.querySelector('.url-path').value || '').toLowerCase();
|
||||
const isFile = scheme === 'file://';
|
||||
const isXml = isFile && /\.xml(\b|$|\?|#)/.test(path);
|
||||
const isCsv = isFile && !isXml;
|
||||
card.querySelector('.syscfg-csv-spec').toggleAttribute('hidden', !isCsv);
|
||||
card.querySelector('.syscfg-xml-spec').toggleAttribute('hidden', !isXml);
|
||||
}
|
||||
|
||||
function addSchemaRow(card, colName, colType) {
|
||||
function addSchemaRow(card, colName, colType, isIndex) {
|
||||
const container = card.querySelector('.schema-rows');
|
||||
const row = document.createElement('div');
|
||||
row.className = 'schema-row';
|
||||
@@ -409,13 +426,19 @@ function addSchemaRow(card, colName, colType) {
|
||||
typeSelect.appendChild(opt);
|
||||
});
|
||||
|
||||
const idxCheck = document.createElement('input');
|
||||
idxCheck.type = 'checkbox';
|
||||
idxCheck.className = 'idx-check';
|
||||
idxCheck.checked = !!isIndex;
|
||||
idxCheck.title = 'Use as index / join key';
|
||||
|
||||
const removeBtn = document.createElement('button');
|
||||
removeBtn.type = 'button';
|
||||
removeBtn.className = 'btn btn-secondary btn-sm';
|
||||
removeBtn.textContent = '×';
|
||||
removeBtn.addEventListener('click', () => { row.remove(); updateSchemaCount(card); });
|
||||
|
||||
row.append(colInput, typeSelect, removeBtn);
|
||||
row.append(colInput, typeSelect, idxCheck, removeBtn);
|
||||
container.appendChild(row);
|
||||
}
|
||||
|
||||
@@ -442,16 +465,19 @@ function readSyscfgCard(card) {
|
||||
const toList = (val) => val.split(',').map(s => s.trim()).filter(Boolean);
|
||||
const toIntList = (val) => val.split(',').map(s => parseInt(s.trim(), 10)).filter(n => !isNaN(n));
|
||||
|
||||
const url = g('url');
|
||||
const urlLower = url.toLowerCase();
|
||||
const isFile = urlLower.startsWith('file://');
|
||||
const isXml = isFile && /\.xml(\b|$)/.test(urlLower);
|
||||
const scheme = card.querySelector('.url-scheme').value;
|
||||
const url = scheme + (card.querySelector('.url-path').value || '');
|
||||
const isFile = scheme === 'file://';
|
||||
const isXml = isFile && /\.xml(\b|$)/.test(url.toLowerCase());
|
||||
|
||||
const schema = {};
|
||||
const index_fields = [];
|
||||
card.querySelectorAll('.schema-rows .schema-row').forEach(row => {
|
||||
const name = row.querySelector('input').value.trim();
|
||||
const name = row.querySelector('input[type="text"]').value.trim();
|
||||
const type = row.querySelector('select').value;
|
||||
if (name) schema[name] = type;
|
||||
if (!name) return;
|
||||
schema[name] = type;
|
||||
if (row.querySelector('.idx-check')?.checked) index_fields.push(name);
|
||||
});
|
||||
|
||||
const spec = (specName, fields) => {
|
||||
@@ -483,7 +509,7 @@ function readSyscfgCard(card) {
|
||||
system_schema: schema,
|
||||
filter: g('filter'),
|
||||
sql: g('sql'),
|
||||
index_fields: toList(g('index_fields')),
|
||||
index_fields,
|
||||
obfuscate_fields: toList(g('obfuscate_fields')),
|
||||
pci_redact_fields: toList(g('pci_redact_fields')),
|
||||
field_widths: toIntList(g('field_widths')),
|
||||
@@ -556,34 +582,39 @@ function collectPayload() {
|
||||
|
||||
// ── Try URL ─────────────────────────────────────────────────────────────────
|
||||
async function testUrl(card) {
|
||||
const url = card.querySelector('.url-input').value.trim();
|
||||
const url = card.querySelector('.url-scheme').value + card.querySelector('.url-path').value.trim();
|
||||
const resultEl = card.querySelector('.url-test-result');
|
||||
if (!url) { showToast('Enter a URL first.', 'warn'); return; }
|
||||
const btn = card.querySelector('.btn-test-url');
|
||||
btn.disabled = true; btn.textContent = 'Checking…';
|
||||
btn.disabled = true; btn.textContent = 'Inspecting…';
|
||||
resultEl.removeAttribute('hidden');
|
||||
resultEl.className = 'url-test-result url-test-loading';
|
||||
resultEl.textContent = 'Checking URL…';
|
||||
resultEl.textContent = 'Inspecting URL…';
|
||||
const csvSpec = {
|
||||
delimiter: card.querySelector('[data-spec="csv"][data-field="delimiter"]')?.value || ',',
|
||||
has_header: card.querySelector('[data-spec="csv"][data-field="header"]')?.checked !== false,
|
||||
};
|
||||
try {
|
||||
const resp = await fetch('/api/configs/test-url', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ url }),
|
||||
body: JSON.stringify({ url, csv_spec: csvSpec }),
|
||||
});
|
||||
const data = await resp.json();
|
||||
renderUrlTestResult(resultEl, data);
|
||||
renderUrlTestResult(resultEl, data, card);
|
||||
} catch (err) {
|
||||
resultEl.className = 'url-test-result url-test-error';
|
||||
resultEl.textContent = `Error: ${err.message}`;
|
||||
} finally {
|
||||
btn.disabled = false; btn.textContent = 'Try URL';
|
||||
btn.disabled = false; btn.textContent = 'Inspect URL';
|
||||
}
|
||||
}
|
||||
|
||||
function renderUrlTestResult(el, data) {
|
||||
if (data.type === 'non-file') {
|
||||
el.className = 'url-test-result url-test-info';
|
||||
el.textContent = data.message;
|
||||
function renderUrlTestResult(el, data, card) {
|
||||
el.textContent = '';
|
||||
|
||||
if (data.type === 'db') {
|
||||
el.setAttribute('hidden', '');
|
||||
return;
|
||||
}
|
||||
if (!data.found) {
|
||||
@@ -591,14 +622,59 @@ function renderUrlTestResult(el, data) {
|
||||
el.textContent = `Not found: ${data.message || data.resolved}`;
|
||||
return;
|
||||
}
|
||||
|
||||
el.className = 'url-test-result url-test-ok';
|
||||
|
||||
// File stats
|
||||
const lines = [`✓ ${data.matches} file${data.matches !== 1 ? 's' : ''} matched`];
|
||||
(data.files || []).forEach(f => {
|
||||
if (f.error) { lines.push(` ⚠ ${f.path}: ${f.error}`); return; }
|
||||
const kb = (f.size_bytes / 1024).toFixed(1);
|
||||
lines.push(` ${f.path.split('/').pop()} ${kb} KB (modified ${f.modified})`);
|
||||
});
|
||||
el.textContent = lines.join('\n');
|
||||
const pre = document.createElement('span');
|
||||
pre.textContent = lines.join('\n');
|
||||
el.appendChild(pre);
|
||||
|
||||
// Schema
|
||||
const schema = data.schema || {};
|
||||
const colCount = Object.keys(schema).length;
|
||||
if (colCount > 0) {
|
||||
const schemaLine = document.createElement('div');
|
||||
schemaLine.style.cssText = 'margin-top:6px; display:flex; align-items:center; gap:10px; flex-wrap:wrap;';
|
||||
const label = document.createElement('span');
|
||||
label.textContent = ` ${colCount} column${colCount !== 1 ? 's' : ''} detected`;
|
||||
const applyBtn = document.createElement('button');
|
||||
applyBtn.type = 'button';
|
||||
applyBtn.className = 'btn btn-primary btn-sm';
|
||||
applyBtn.textContent = 'Apply Schema';
|
||||
applyBtn.addEventListener('click', () => {
|
||||
applySchema(card, schema);
|
||||
applyBtn.textContent = '✓ Applied';
|
||||
applyBtn.disabled = true;
|
||||
});
|
||||
schemaLine.append(label, applyBtn);
|
||||
el.appendChild(schemaLine);
|
||||
}
|
||||
}
|
||||
|
||||
function applySchema(card, schema) {
|
||||
const container = card.querySelector('.schema-rows');
|
||||
// Preserve existing index ticks by column name before clearing
|
||||
const existingIndex = new Set();
|
||||
container.querySelectorAll('.schema-row').forEach(row => {
|
||||
if (row.querySelector('.idx-check')?.checked)
|
||||
existingIndex.add(row.querySelector('input[type="text"]').value.trim());
|
||||
});
|
||||
container.innerHTML = '';
|
||||
Object.entries(schema).forEach(([col, type]) =>
|
||||
addSchemaRow(card, col, type, existingIndex.has(col))
|
||||
);
|
||||
updateSchemaCount(card);
|
||||
// Open the schema section so the user sees it
|
||||
card.querySelector('.syscfg-section details, details.syscfg-section')?.setAttribute('open', '');
|
||||
card.querySelector('.schema-builder')?.closest('details')?.setAttribute('open', '');
|
||||
showToast(`Schema applied — ${Object.keys(schema).length} columns.`, 'ok');
|
||||
}
|
||||
|
||||
// ── Toast ───────────────────────────────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user