Auto schemas

2026-05-26 22:34:28 +12:00
parent 35d70a7746
commit d4969172c2
7 changed files with 320 additions and 132 deletions
@@ -0,0 +1,6 @@
+account_id,customer_name,transaction_date,amount,currency,status,reference
+ACC001,Alice Nguyen,2026-05-26,1500.00,NZD,MATCHED,REF-00001
+ACC002,Bob Tane,2026-05-26,820.50,NZD,UNMATCHED,REF-00002
+ACC003,Carol Smith,2026-05-26,3200.00,NZD,MATCHED,REF-00003
+ACC004,David Park,2026-05-26,415.75,NZD,PENDING,REF-00004
+ACC005,Eva Brown,2026-05-26,9900.00,NZD,MATCHED,REF-00005
@@ -911,7 +911,8 @@ a.job-bar:hover { filter: brightness(1.1); box-shadow: 0 0 0 1px rgba(255,255,25
    gap: 8px;
    align-items: stretch;
 }
-.url-input { flex: 1; }
+.url-input-row .url-scheme { flex-shrink: 0; width: auto; }
+.url-input-row .url-path   { flex: 1; width: 0; min-width: 0; }
 .url-test-result {
    margin-top: 6px;
    padding: 8px 10px;
@@ -930,7 +931,7 @@ a.job-bar:hover { filter: brightness(1.1); box-shadow: 0 0 0 1px rgba(255,255,25
 .schema-builder { display: flex; flex-direction: column; gap: 6px; }
 .kv-table-header {
    display: grid;
-    grid-template-columns: 1fr 1fr auto;
+    grid-template-columns: 1fr 1fr auto auto;
    gap: 8px;
    font-size: 10px;
    text-transform: uppercase;
@@ -940,10 +941,17 @@ a.job-bar:hover { filter: brightness(1.1); box-shadow: 0 0 0 1px rgba(255,255,25
 }
 .schema-row {
    display: grid;
-    grid-template-columns: 1fr 1fr auto;
+    grid-template-columns: 1fr 1fr auto auto;
    gap: 8px;
    align-items: center;
 }
+.schema-row .idx-check {
+    width: 16px;
+    height: 16px;
+    accent-color: var(--accent);
+    cursor: pointer;
+    justify-self: center;
+}
 .schema-row input,
 .schema-row select {
    border-radius: 5px;
@@ -176,21 +176,22 @@
      <div class="editor-field editor-field-full">
        <label>URL <span class="req">*</span></label>
        <div class="url-input-row">
-          <input type="text" data-field="url" placeholder="file:// or mssql:// or oracle:// ..." class="url-input">
-          <button type="button" class="btn btn-secondary btn-sm btn-test-url">Try URL</button>
+          <select class="url-scheme">
+            <option value="file://">file://</option>
+            <option value="mssql://">mssql://</option>
+            <option value="oracle://">oracle://</option>
+            <option value="snowflake://">snowflake://</option>
+            <option value="duckdb://">duckdb://</option>
+          </select>
+          <input type="text" class="url-path" placeholder="/path/to/file or SERVER?db=DB&amp;table=TABLE">
+          <button type="button" class="btn btn-secondary btn-sm btn-test-url">Inspect URL</button>
        </div>
        <span class="field-hint">
-          Supports: <code>file://</code> <code>mssql://</code> <code>oracle://</code> <code>snowflake://</code> <code>duckdb://</code><br>
          Date templates: {% raw %}<code>{{as_at_date.strftime('%Y%m%d')}}</code> <code>{{today.strftime('%Y%m%d')}}</code>{% endraw %}
        </span>
        <div class="url-test-result" hidden></div>
      </div>

-      <div class="editor-field editor-field-full">
-        <label>SQL Override</label>
-        <textarea data-field="sql" rows="2" placeholder="Full SQL query — replaces auto-generated SELECT. Required for duckdb://."></textarea>
-      </div>
-
      <div class="editor-field editor-field-wide">
        <label>Comment</label>
        <input type="text" data-field="comment" placeholder="Optional note about this system.">
@@ -203,7 +204,7 @@
      <summary>Schema <span class="syscfg-section-count schema-count">0 columns</span></summary>
      <div class="schema-builder">
        <div class="kv-table-header">
-          <span>Column Name</span><span>Type</span><span></span>
+          <span>Column Name</span><span>Type</span><span>Index</span><span></span>
        </div>
        <div class="schema-rows"></div>
        <button type="button" class="btn btn-secondary btn-sm btn-add-schema-row">+ Add Column</button>
@@ -263,8 +264,8 @@
      <summary>Advanced</summary>
      <div class="syscfg-grid">
        <div class="editor-field editor-field-full">
-          <label>Index Fields</label>
-          <input type="text" data-field="index_fields" placeholder="Comma-separated field names used as join keys.">
+          <label>SQL Override</label>
+          <textarea data-field="sql" rows="2" placeholder="Full SQL query — replaces auto-generated SELECT. Required for duckdb://."></textarea>
        </div>
        <div class="editor-field editor-field-full">
          <label>Obfuscate Fields</label>
@@ -313,7 +314,7 @@ function buildSyscfgCard(data, role, index) {
  }

  // Populate simple fields
-  const fields = ['name','url','day_offset','filter','sql','comment'];
+  const fields = ['name','day_offset','filter','sql','comment'];
  fields.forEach(f => {
    const el = card.querySelector(`[data-field="${f}"]`);
    if (!el) return;
@@ -321,17 +322,31 @@ function buildSyscfgCard(data, role, index) {
    if (val !== undefined && val !== null) el.value = val;
  });

+  // Split stored URL into scheme dropdown + path input
+  if (data.url) {
+    const m = data.url.match(/^([a-z]+:\/\/)([\s\S]*)$/i);
+    if (m) {
+      const schemeEl = card.querySelector('.url-scheme');
+      const matchingOpt = Array.from(schemeEl.options).find(o => o.value === m[1].toLowerCase());
+      if (matchingOpt) schemeEl.value = matchingOpt.value;
+      card.querySelector('.url-path').value = m[2];
+    } else {
+      card.querySelector('.url-path').value = data.url;
+    }
+  }
+
  // Populate list fields (comma-joined)
-  ['index_fields','obfuscate_fields','pci_redact_fields'].forEach(f => {
+  ['obfuscate_fields','pci_redact_fields'].forEach(f => {
    const el = card.querySelector(`[data-field="${f}"]`);
    if (el && Array.isArray(data[f])) el.value = data[f].join(', ');
  });
  const fwEl = card.querySelector('[data-field="field_widths"]');
  if (fwEl && Array.isArray(data.field_widths)) fwEl.value = data.field_widths.join(', ');

-  // Schema
+  // Schema — tick index columns derived from index_fields list
  const schemaData = data.system_schema || {};
-  Object.entries(schemaData).forEach(([col, type]) => addSchemaRow(card, col, type));
+  const indexSet = new Set(data.index_fields || []);
+  Object.entries(schemaData).forEach(([col, type]) => addSchemaRow(card, col, type, indexSet.has(col)));
  updateSchemaCount(card);

  // CSV spec
@@ -353,11 +368,12 @@ function buildSyscfgCard(data, role, index) {
  }

  // URL change → show/hide specs, clear test result
-  const urlInput = card.querySelector('.url-input');
-  urlInput.addEventListener('input', () => {
+  const onUrlChange = () => {
    syncSpecVisibility(card);
    card.querySelector('.url-test-result').setAttribute('hidden', '');
-  });
+  };
+  card.querySelector('.url-scheme').addEventListener('change', onUrlChange);
+  card.querySelector('.url-path').addEventListener('input', onUrlChange);
  syncSpecVisibility(card);

  // Schema add button
@@ -382,15 +398,16 @@ function setSpecCheckbox(card, spec, field, checked) {
 }

 function syncSpecVisibility(card) {
-  const url = (card.querySelector('.url-input').value || '').toLowerCase();
-  const isFile = url.startsWith('file://');
-  const isXml = isFile && /\.xml(\b|$|\?|#)/.test(url);
+  const scheme = card.querySelector('.url-scheme').value;
+  const path = (card.querySelector('.url-path').value || '').toLowerCase();
+  const isFile = scheme === 'file://';
+  const isXml = isFile && /\.xml(\b|$|\?|#)/.test(path);
  const isCsv = isFile && !isXml;
  card.querySelector('.syscfg-csv-spec').toggleAttribute('hidden', !isCsv);
  card.querySelector('.syscfg-xml-spec').toggleAttribute('hidden', !isXml);
 }

-function addSchemaRow(card, colName, colType) {
+function addSchemaRow(card, colName, colType, isIndex) {
  const container = card.querySelector('.schema-rows');
  const row = document.createElement('div');
  row.className = 'schema-row';
@@ -409,13 +426,19 @@ function addSchemaRow(card, colName, colType) {
    typeSelect.appendChild(opt);
  });

+  const idxCheck = document.createElement('input');
+  idxCheck.type = 'checkbox';
+  idxCheck.className = 'idx-check';
+  idxCheck.checked = !!isIndex;
+  idxCheck.title = 'Use as index / join key';
+
  const removeBtn = document.createElement('button');
  removeBtn.type = 'button';
  removeBtn.className = 'btn btn-secondary btn-sm';
  removeBtn.textContent = '×';
  removeBtn.addEventListener('click', () => { row.remove(); updateSchemaCount(card); });

-  row.append(colInput, typeSelect, removeBtn);
+  row.append(colInput, typeSelect, idxCheck, removeBtn);
  container.appendChild(row);
 }

@@ -442,16 +465,19 @@ function readSyscfgCard(card) {
  const toList = (val) => val.split(',').map(s => s.trim()).filter(Boolean);
  const toIntList = (val) => val.split(',').map(s => parseInt(s.trim(), 10)).filter(n => !isNaN(n));

-  const url = g('url');
-  const urlLower = url.toLowerCase();
-  const isFile = urlLower.startsWith('file://');
-  const isXml = isFile && /\.xml(\b|$)/.test(urlLower);
+  const scheme = card.querySelector('.url-scheme').value;
+  const url = scheme + (card.querySelector('.url-path').value || '');
+  const isFile = scheme === 'file://';
+  const isXml = isFile && /\.xml(\b|$)/.test(url.toLowerCase());

  const schema = {};
+  const index_fields = [];
  card.querySelectorAll('.schema-rows .schema-row').forEach(row => {
-    const name = row.querySelector('input').value.trim();
+    const name = row.querySelector('input[type="text"]').value.trim();
    const type = row.querySelector('select').value;
-    if (name) schema[name] = type;
+    if (!name) return;
+    schema[name] = type;
+    if (row.querySelector('.idx-check')?.checked) index_fields.push(name);
  });

  const spec = (specName, fields) => {
@@ -483,7 +509,7 @@ function readSyscfgCard(card) {
    system_schema: schema,
    filter: g('filter'),
    sql: g('sql'),
-    index_fields: toList(g('index_fields')),
+    index_fields,
    obfuscate_fields: toList(g('obfuscate_fields')),
    pci_redact_fields: toList(g('pci_redact_fields')),
    field_widths: toIntList(g('field_widths')),
@@ -556,34 +582,39 @@ function collectPayload() {

 // ── Try URL ─────────────────────────────────────────────────────────────────
 async function testUrl(card) {
-  const url = card.querySelector('.url-input').value.trim();
+  const url = card.querySelector('.url-scheme').value + card.querySelector('.url-path').value.trim();
  const resultEl = card.querySelector('.url-test-result');
  if (!url) { showToast('Enter a URL first.', 'warn'); return; }
  const btn = card.querySelector('.btn-test-url');
-  btn.disabled = true; btn.textContent = 'Checking…';
+  btn.disabled = true; btn.textContent = 'Inspecting…';
  resultEl.removeAttribute('hidden');
  resultEl.className = 'url-test-result url-test-loading';
-  resultEl.textContent = 'Checking URL…';
+  resultEl.textContent = 'Inspecting URL…';
+  const csvSpec = {
+    delimiter: card.querySelector('[data-spec="csv"][data-field="delimiter"]')?.value || ',',
+    has_header: card.querySelector('[data-spec="csv"][data-field="header"]')?.checked !== false,
+  };
  try {
    const resp = await fetch('/api/configs/test-url', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ url }),
+      body: JSON.stringify({ url, csv_spec: csvSpec }),
    });
    const data = await resp.json();
-    renderUrlTestResult(resultEl, data);
+    renderUrlTestResult(resultEl, data, card);
  } catch (err) {
    resultEl.className = 'url-test-result url-test-error';
    resultEl.textContent = `Error: ${err.message}`;
  } finally {
-    btn.disabled = false; btn.textContent = 'Try URL';
+    btn.disabled = false; btn.textContent = 'Inspect URL';
  }
 }

-function renderUrlTestResult(el, data) {
-  if (data.type === 'non-file') {
-    el.className = 'url-test-result url-test-info';
-    el.textContent = data.message;
+function renderUrlTestResult(el, data, card) {
+  el.textContent = '';
+
+  if (data.type === 'db') {
+    el.setAttribute('hidden', '');
    return;
  }
  if (!data.found) {
@@ -591,14 +622,59 @@ function renderUrlTestResult(el, data) {
    el.textContent = `Not found: ${data.message || data.resolved}`;
    return;
  }
+
  el.className = 'url-test-result url-test-ok';
+
+  // File stats
  const lines = [`✓ ${data.matches} file${data.matches !== 1 ? 's' : ''} matched`];
  (data.files || []).forEach(f => {
    if (f.error) { lines.push(`  ⚠ ${f.path}: ${f.error}`); return; }
    const kb = (f.size_bytes / 1024).toFixed(1);
    lines.push(`  ${f.path.split('/').pop()}  ${kb} KB  (modified ${f.modified})`);
  });
-  el.textContent = lines.join('\n');
+  const pre = document.createElement('span');
+  pre.textContent = lines.join('\n');
+  el.appendChild(pre);
+
+  // Schema
+  const schema = data.schema || {};
+  const colCount = Object.keys(schema).length;
+  if (colCount > 0) {
+    const schemaLine = document.createElement('div');
+    schemaLine.style.cssText = 'margin-top:6px; display:flex; align-items:center; gap:10px; flex-wrap:wrap;';
+    const label = document.createElement('span');
+    label.textContent = `  ${colCount} column${colCount !== 1 ? 's' : ''} detected`;
+    const applyBtn = document.createElement('button');
+    applyBtn.type = 'button';
+    applyBtn.className = 'btn btn-primary btn-sm';
+    applyBtn.textContent = 'Apply Schema';
+    applyBtn.addEventListener('click', () => {
+      applySchema(card, schema);
+      applyBtn.textContent = '✓ Applied';
+      applyBtn.disabled = true;
+    });
+    schemaLine.append(label, applyBtn);
+    el.appendChild(schemaLine);
+  }
+}
+
+function applySchema(card, schema) {
+  const container = card.querySelector('.schema-rows');
+  // Preserve existing index ticks by column name before clearing
+  const existingIndex = new Set();
+  container.querySelectorAll('.schema-row').forEach(row => {
+    if (row.querySelector('.idx-check')?.checked)
+      existingIndex.add(row.querySelector('input[type="text"]').value.trim());
+  });
+  container.innerHTML = '';
+  Object.entries(schema).forEach(([col, type]) =>
+    addSchemaRow(card, col, type, existingIndex.has(col))
+  );
+  updateSchemaCount(card);
+  // Open the schema section so the user sees it
+  card.querySelector('.syscfg-section details, details.syscfg-section')?.setAttribute('open', '');
+  card.querySelector('.schema-builder')?.closest('details')?.setAttribute('open', '');
+  showToast(`Schema applied — ${Object.keys(schema).length} columns.`, 'ok');
 }

 // ── Toast ───────────────────────────────────────────────────────────────────