From 0aeb893c0f72f1ed5e269a2773f64cd126282d4d Mon Sep 17 00:00:00 2001 From: Semprini Date: Sun, 14 Jun 2026 16:46:06 +1200 Subject: [PATCH] feat: Implement initial setup for Financial Crime domain - Added Makefile for orchestrating domain instantiation and management. - Created deploy README for guiding the setup process of the Financial Crime domain. - Introduced domain configuration file for mapping metadata to the deployment template. - Implemented pre-commit hook for validating metadata before commits. - Developed instantiation script to manage the lifecycle of the Financial Crime domain. - Added Salesforce CRM source with associated metadata and transformation files. - Added SAP Fraud Management source with associated metadata and transformation files. - Added Temenos Payment source with associated metadata and transformation files. - Removed obsolete payment event and parties transformation files. --- .gitignore | 4 + .md-ddl | 2 +- Makefile | 40 +++ README.md | 33 +- deploy/README.md | 64 ++++ deploy/domain.config.yml | 37 ++ deploy/hooks/pre-commit | 20 ++ deploy/instantiate.py | 339 ++++++++++++++++++ financial_crime/domain.md | 6 +- .../sources/salesforce-crm/source.md | 34 -- .../sources/temenos-payment/source.md | 38 -- sources/salesforce-crm/source.md | 34 ++ .../transforms/table_account.md | 0 .../transforms/table_contact.md | 0 .../transforms/table_contact_point.md | 0 .../transforms/table_preference.md | 0 .../sap-fraud-management/source.md | 8 +- .../transforms/table_alert_case.md | 0 .../transforms/table_customer_risk_profile.md | 0 .../transforms/table_sanctions_screening.md | 0 sources/temenos-payment/source.md | 38 ++ .../transforms/table_account_ref.md | 0 .../transforms/table_initiation.md | 0 .../transforms/table_payment_event.md | 0 .../transforms/table_payment_parties.md | 0 25 files changed, 612 insertions(+), 85 deletions(-) create mode 100644 Makefile create mode 100644 deploy/README.md create mode 100644 deploy/domain.config.yml create mode 100755 deploy/hooks/pre-commit create mode 100755 deploy/instantiate.py delete mode 100644 financial_crime/sources/salesforce-crm/source.md delete mode 100644 financial_crime/sources/temenos-payment/source.md create mode 100644 sources/salesforce-crm/source.md rename {financial_crime/sources => sources}/salesforce-crm/transforms/table_account.md (100%) rename {financial_crime/sources => sources}/salesforce-crm/transforms/table_contact.md (100%) rename {financial_crime/sources => sources}/salesforce-crm/transforms/table_contact_point.md (100%) rename {financial_crime/sources => sources}/salesforce-crm/transforms/table_preference.md (100%) rename {financial_crime/sources => sources}/sap-fraud-management/source.md (51%) rename {financial_crime/sources => sources}/sap-fraud-management/transforms/table_alert_case.md (100%) rename {financial_crime/sources => sources}/sap-fraud-management/transforms/table_customer_risk_profile.md (100%) rename {financial_crime/sources => sources}/sap-fraud-management/transforms/table_sanctions_screening.md (100%) create mode 100644 sources/temenos-payment/source.md rename {financial_crime/sources => sources}/temenos-payment/transforms/table_account_ref.md (100%) rename {financial_crime/sources => sources}/temenos-payment/transforms/table_initiation.md (100%) rename {financial_crime/sources => sources}/temenos-payment/transforms/table_payment_event.md (100%) rename {financial_crime/sources => sources}/temenos-payment/transforms/table_payment_parties.md (100%) diff --git a/.gitignore b/.gitignore index 9c90d4f..adccbca 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,7 @@ env/ # OS .DS_Store Thumbs.db +/semprini-data-domain +/lib +/.env +/domains/ diff --git a/.md-ddl b/.md-ddl index 2c3a301..b8b620c 160000 --- a/.md-ddl +++ b/.md-ddl @@ -1 +1 @@ -Subproject commit 2c3a301cacbd01ff0e6bd343fd2704762c0cd706 +Subproject commit b8b620c580012e43a11c23e8395e687911bd9ca2 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..18b1ffb --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +# Financial Crime domain — Phase 1 instantiation. +# Thin wrapper over the metadata-driven driver (deploy/instantiate.py), which +# orchestrates the semprini-data-domain template against this repo's metadata. + +PY ?= python3 +DRIVER := $(PY) deploy/instantiate.py + +.PHONY: help preflight instantiate verify teardown install-hooks \ + wire secrets scaffold build up ducklake register activate + +help: + @echo "Targets:" + @echo " make preflight Validate MD-DDL metadata (blocks on findings)" + @echo " make instantiate Full stand-up: scaffold -> build -> up -> register -> verify" + @echo " make verify Health-check the running domain" + @echo " make teardown Unregister + stop the domain (reversible)" + @echo " make install-hooks Install the pre-commit metadata gate" + @echo "" + @echo "Individual steps: wire secrets scaffold build up ducklake register activate" + +preflight: + $(DRIVER) preflight + +instantiate: + $(DRIVER) all + +verify: + $(DRIVER) verify + +teardown: + $(DRIVER) teardown + +# individual driver steps (handy for debugging a partial stand-up) +wire secrets scaffold build up ducklake register activate: + $(DRIVER) $@ + +install-hooks: + @ln -sf ../../deploy/hooks/pre-commit .git/hooks/pre-commit + @chmod +x deploy/hooks/pre-commit + @echo "Installed .git/hooks/pre-commit -> deploy/hooks/pre-commit" diff --git a/README.md b/README.md index 1e01db0..3702fd7 100644 --- a/README.md +++ b/README.md @@ -6,22 +6,45 @@ The project leverages: - https://git.semprini.me/paul/semprini-core to provide I&AM, Code Repos - https://git.semprini.me/paul/semprini-data for data management, data product metadata, real-time data sourcing, and data transform platforms. -- https://git.semprini.me/paul/semprini-data-domain for foundational and experience data products. +- https://git.semprini.me/paul/semprini-data-domain template for infrastructure to host foundational (domain aligned) and experience (consumer aligned) data products. ## Project Goals -1. Synthetic data generators per source system - replicating realistic cadence of data change and system semantics in source systems. This will be achieved by: +Primary purpose is a feedback loop for the data domain template and markdown data definition language (MD-DDL) and become an exemplar of data domains. The domain template and md-ddl are included in this project as sub-modules only - not to be edited in this project - feedback and issues should be added into the projects git project issues. + +### Phase 1: + +1. A running, stable Financial Crime data domain instantiated from the domain template. This means: + - Metadata defining the domain is all this project definies + - The metadata together with the domain template creates a fully functional domain + +3. Metadata driven integration into the rest of the ecosystem: + - Core platform: Leverage of the core running systems for I&AM, certs, DNS etc. These systems should already be running and the design can be found in https://git.semprini.me/paul/semprini-core + - Data management: Registration of the domain into the data management systems - data marketplace, data catalogue etc. These systems should already be running and the design can be found in https://git.semprini.me/paul/semprini-data + +### Phase 2: + +1. Synthetic data generators per defined source system - replicating realistic cadence of data change and system semantics in source systems. Source system definitions are found in the /sources folder. This will be achieved by: - A single source system Postgres DB server for all sources - Per source system database instances - Python (faker) synthetic data generator per source system - Ability to set change cadence - Use the MD-DDL source definitions to define schemas and synthetic generators as per md-ddl. -2. Per source system, source aligned data products. This will be defined once the synthetic data generators are running +3. Per source system, source aligned data products. This will be defined once the synthetic data generators are running. + - Deployed on the data bus stack. This should be running and the design can be found in https://git.semprini.me/paul/semprini-data -3. Domain aligned data products. Detail to be defined later but will be: +4. Creation of domain aligned data product templates. A template is purely driven by metadata (e.g. Open Data Product Specification/Schemas etc) Features of this class of data product are: + - Kafka topics per aggregate root entity - enforcing canonical schema + - GraphQL API endpoints for canonical schema + - Kafka sync into PostgreSQL DB in real-time + - pg_duckpipe ingest into ducklake + - Analytics SQL interface to ducklake via pg_ducklake + - Registration to data marketplace and catalog + +5. Domain aligned data products. Driving the domain aligned data product template using metadata. Details to be defined later but the data products will be: - Party Core - Financial Transaction - Product Core -4. Consumer aligned data products. To be defined later. +6. Consumer aligned data products. To be defined later. diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..2f21482 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,64 @@ +# Deploy — Financial Crime domain (Phase 1) + +Phase 1 stands up a *running, stable* Financial Crime data domain from this repo's +MD-DDL metadata plus the `semprini-data-domain` template, and registers it into the +ecosystem (Keycloak, CoreDNS, Uptime-Kuma, Prometheus, OpenMetadata). + +Everything is driven from the metadata + [domain.config.yml](domain.config.yml) by +[instantiate.py](instantiate.py). This repo acts as the template's *data root*: the +domain is scaffolded into `./domains/`, and a few platform-owned pieces are bridged +in via repo-local symlinks (all gitignored). + +## Prerequisites + +- **Docker** running, and the shared external networks present: `semprini_bus`, + `semprini_internal`, `semprini_proxy` (created by the core + bus stacks). +- **semprini-core** running (Keycloak, CoreDNS, Kuma, Prometheus, step-ca) and + **semprini-data** management zone (OpenMetadata, Apicurio). These live at + `../semprini-core` and `../semprini-data`. +- `../semprini-data/.env` populated with admin creds (`KEYCLOAK_ADMIN*`, + `KUMA_API_KEY`, `KUMA_URL`, OpenMetadata config). The driver symlinks this in as + `./.env` and appends the domain's own secrets (generated, never committed). +- **VPN** (headscale/tailscale) to reach the `*.financial-crime.data.internal` hosts. +- Python deps: `pyyaml` (already in `.venv`). + +## One-command stand-up + +```bash +make instantiate # preflight → wire → secrets → scaffold → build → up → ducklake → register → activate → verify +make verify # re-run health checks +make teardown # unregister + stop (reverses everything; core/mgmt left clean) +``` + +Individual steps (useful for debugging): `make wire secrets scaffold build up +ducklake register activate`. See `python3 deploy/instantiate.py --help`-style step +list at the top of [instantiate.py](instantiate.py). + +## Metadata gate + +```bash +make preflight # validate the MD-DDL domain +make install-hooks # install a pre-commit hook that runs preflight on every commit +``` + +This project is not on GitHub, so the gate is local (a git pre-commit hook). If the +local Gitea grows Actions later, mirror the same `preflight.py financial_crime` call. + +## Naming note + +The MD-DDL metadata folder is `financial_crime` (underscore). `new-domain.sh` requires +a lowercase + hyphen name, so the **infra** domain name is `financial-crime`. Both +uppercase to the same env prefix `FINANCIAL_CRIME`. + +## Known blocker (as of 2026-06-14) + +`make build` currently fails on the **postgres-ducklake** image: the template's +`postgres/Dockerfile` downloads `pg_ducklake` / `pg_duckpipe` `.deb` packages from +GitHub releases that return **404** — `duckdb/pg_ducklake` has no releases, and the +real project `relytcloud/pg_ducklake` / `relytcloud/pg_duckpipe` have **zero published +releases**. The `ariadne` image builds fine. + +This is a template (submodule) dependency issue, not a metadata issue — the submodule +is not edited here; it should be raised upstream against `semprini-data-domain`. Once a +valid artifact source (or prebuilt `semprini-postgres-ducklake:16` image) is available, +`make instantiate` completes the live stand-up unchanged. diff --git a/deploy/domain.config.yml b/deploy/domain.config.yml new file mode 100644 index 0000000..0d002ff --- /dev/null +++ b/deploy/domain.config.yml @@ -0,0 +1,37 @@ +# Deploy config for instantiating the Financial Crime domain. +# +# This is the one infra-facing input MD-DDL itself does not own. It maps the +# metadata in this repo onto the semprini-data-domain template's instantiation +# scripts. Read by deploy/instantiate.py. +# +# Data-root model: this repo (random_corp) IS the data root. The template +# submodule's new-domain.sh resolves its data root as its own parent, so it +# scaffolds into ./domains/, reads ./.env, and resolves ../semprini-core. The +# driver bridges the few platform-owned pieces via repo-local symlinks (see +# below): the build-context name, the shared .env, and the registration lib/. +# +# Naming note: the MD-DDL metadata folder is `financial_crime` (underscore), +# but new-domain.sh requires lowercase + hyphens (^[a-z][a-z0-9-]+$), so the +# infra domain name is `financial-crime`. Both uppercase to the same env +# prefix FINANCIAL_CRIME (hyphens/underscores -> underscores). + +metadata_dir: financial_crime # MD-DDL domain folder (preflight target) +domain: financial-crime # infra domain name passed to the template scripts +port_base: 22000 # API=22000, Flink=22001, MinIO=22002, Analytics=22003 + +template_dir: .semprini-data-domain # template submodule (new-domain.sh, domain-integration.sh) +platform_repo: ../semprini-data # source of the shared .env + lib/ (admin creds + integ helpers) + +# Repo-local symlinks the driver creates to satisfy the template's path +# assumptions while keeping random_corp as the data root (all gitignored): +# semprini-data-domain -> .semprini-data-domain (compose build contexts: ../../semprini-data-domain) +# .env -> ../semprini-data/.env (shared secrets + admin creds) +# lib -> ../semprini-data/lib (domain-integration.sh sources lib/core-integration.sh) + +# Secrets ensured in the shared .env (generated if missing, never committed). +# Key names follow the template's DOMAIN_UPPER convention. +secrets: + - POSTGRES_FINANCIAL_CRIME_PASSWORD + - MINIO_FINANCIAL_CRIME_ACCESS_KEY + - MINIO_FINANCIAL_CRIME_SECRET_KEY + - ANALYTICS_FINANCIAL_CRIME_PASSWORD diff --git a/deploy/hooks/pre-commit b/deploy/hooks/pre-commit new file mode 100755 index 0000000..e0d9f38 --- /dev/null +++ b/deploy/hooks/pre-commit @@ -0,0 +1,20 @@ +#!/bin/bash +# Pre-commit metadata gate: block commits that break the MD-DDL domain. +# Install with: make install-hooks +set -euo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" +METADATA_DIR="$( + python3 - "$REPO_ROOT/deploy/domain.config.yml" <<'PY' +import sys, yaml +print(yaml.safe_load(open(sys.argv[1]))["metadata_dir"]) +PY +)" + +echo "pre-commit: running MD-DDL pre-flight on '$METADATA_DIR'…" +if ! python3 "$REPO_ROOT/.github/scripts/preflight.py" "$METADATA_DIR"; then + echo "pre-commit: metadata pre-flight FAILED — commit blocked." >&2 + echo " Fix the findings above, or bypass with 'git commit --no-verify'." >&2 + exit 1 +fi +echo "pre-commit: metadata OK." diff --git a/deploy/instantiate.py b/deploy/instantiate.py new file mode 100755 index 0000000..849756b --- /dev/null +++ b/deploy/instantiate.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +"""Metadata-driven instantiation of the Financial Crime data domain. + +This driver turns the MD-DDL metadata in this repo into a running, registered +domain by orchestrating the semprini-data-domain template scripts. It does not +re-implement them — it wires this repo as the template's "data root", ensures +secrets, then calls new-domain.sh / docker compose / domain-integration.sh in +the right order. + +Usage: + python3 deploy/instantiate.py [ ...] + python3 deploy/instantiate.py all # full stand-up + python3 deploy/instantiate.py teardown # reverse it + +Steps (also runnable individually): + preflight validate metadata (blocks on findings) + wire create repo-local symlinks the template expects + secrets ensure domain secrets exist in the shared .env + scaffold new-domain.sh -> domains// + build build the postgres-ducklake + ariadne images + up bring the stack up; wait for bucket + postgres health + ducklake attach the DuckLake catalog in the analytics DB + register register with core (Keycloak/DNS/Kuma/Prometheus) + mgmt (OpenMetadata) + activate restart ariadne + flink to pick up OIDC + metrics + verify health-check the running domain +""" +from __future__ import annotations + +import os +import secrets as secretslib +import subprocess +import sys +import time +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +CONFIG_PATH = REPO_ROOT / "deploy" / "domain.config.yml" + + +# --------------------------------------------------------------------------- # +# helpers +# --------------------------------------------------------------------------- # +def log(msg: str) -> None: + print(f"\033[1;36m›\033[0m {msg}", flush=True) + + +def ok(msg: str) -> None: + print(f" \033[1;32m✓\033[0m {msg}", flush=True) + + +def die(msg: str) -> None: + print(f"\033[1;31m✗ {msg}\033[0m", file=sys.stderr, flush=True) + sys.exit(1) + + +def run(cmd: list[str], cwd: Path | None = None, check: bool = True, + capture: bool = False) -> subprocess.CompletedProcess: + printable = " ".join(cmd) + log(f"$ {printable}") + return subprocess.run( + cmd, cwd=str(cwd or REPO_ROOT), check=check, + text=True, capture_output=capture, + ) + + +class Cfg: + def __init__(self, raw: dict): + self.metadata_dir: str = raw["metadata_dir"] + self.domain: str = raw["domain"] + self.port_base: int = int(raw["port_base"]) + self.template_dir: str = raw["template_dir"] + self.platform_repo: str = raw["platform_repo"] + self.secrets: list[str] = list(raw["secrets"]) + + @property + def upper(self) -> str: + return self.domain.replace("-", "_").upper() + + @property + def template(self) -> Path: + return REPO_ROOT / self.template_dir + + @property + def compose(self) -> Path: + return REPO_ROOT / "domains" / self.domain / "compose.yml" + + @property + def env_file(self) -> Path: + return REPO_ROOT / ".env" + + +def load_cfg() -> Cfg: + if not CONFIG_PATH.exists(): + die(f"config not found: {CONFIG_PATH}") + return Cfg(yaml.safe_load(CONFIG_PATH.read_text())) + + +def compose_cmd(cfg: Cfg, *args: str) -> list[str]: + return ["docker", "compose", "-f", str(cfg.compose), *args] + + +def env_value(cfg: Cfg, key: str) -> str | None: + for line in cfg.env_file.read_text().splitlines(): + if line.startswith(f"{key}="): + return line.split("=", 1)[1] + return None + + +# --------------------------------------------------------------------------- # +# steps +# --------------------------------------------------------------------------- # +def step_preflight(cfg: Cfg) -> None: + log("preflight: validating MD-DDL metadata") + checker = REPO_ROOT / ".github" / "scripts" / "preflight.py" + run([sys.executable, str(checker), cfg.metadata_dir]) + ok("metadata pre-flight passed") + + +def step_wire(cfg: Cfg) -> None: + """Repo-local symlinks so the template treats this repo as the data root.""" + log("wire: creating repo-local symlinks") + links = { + # compose build contexts use ../../semprini-data-domain + "semprini-data-domain": cfg.template_dir, + # shared platform secrets + admin creds + ".env": f"{cfg.platform_repo}/.env", + # domain-integration.sh sources lib/core-integration.sh + "lib": f"{cfg.platform_repo}/lib", + } + for name, target in links.items(): + link = REPO_ROOT / name + if link.is_symlink() or link.exists(): + ok(f"{name} -> {os.readlink(link) if link.is_symlink() else '(exists)'}") + continue + link.symlink_to(target) + ok(f"created {name} -> {target}") + # ensure the bind-mount dirs ariadne expects are present + for d in ("schemas", "resolvers", "flink-jobs"): + (REPO_ROOT / "domains" / cfg.domain / d).mkdir(parents=True, exist_ok=True) + + +def step_secrets(cfg: Cfg) -> None: + log("secrets: ensuring domain secrets in shared .env") + env_file = cfg.env_file + if not env_file.exists(): + die(f"{env_file} missing (run 'wire' first; check {cfg.platform_repo}/.env)") + existing = { + line.split("=", 1)[0] + for line in env_file.read_text().splitlines() + if "=" in line and not line.startswith("#") + } + to_add: list[str] = [] + for key in cfg.secrets: + if key in existing: + ok(f"{key} present") + continue + if "ACCESS_KEY" in key: + val = "fincrime" + secretslib.token_hex(8) + else: + val = secretslib.token_urlsafe(24) + to_add.append(f"{key}={val}") + ok(f"generated {key}") + port_key = f"{cfg.upper}_PORT_BASE" + if port_key not in existing: + to_add.append(f"{port_key}={cfg.port_base}") + ok(f"set {port_key}={cfg.port_base}") + if to_add: + with env_file.open("a") as fh: + fh.write("\n# --- " + cfg.domain + " domain secrets ---\n") + fh.write("\n".join(to_add) + "\n") + + +def step_scaffold(cfg: Cfg) -> None: + log(f"scaffold: new-domain.sh {cfg.domain} {cfg.port_base}") + if cfg.compose.exists(): + ok(f"already scaffolded at {cfg.compose.parent}") + return + run([str(cfg.template / "new-domain.sh"), cfg.domain, str(cfg.port_base)]) + ok("domain scaffolded") + + +def step_build(cfg: Cfg) -> None: + log("build: postgres-ducklake + ariadne images") + run(compose_cmd(cfg, "build", + f"postgres-{cfg.domain}", f"ariadne-{cfg.domain}")) + ok("images built") + + +def _wait(cfg: Cfg, desc: str, check_fn, timeout: int = 180, interval: int = 5) -> None: + log(f"waiting for {desc} (≤{timeout}s)") + deadline = time.time() + timeout + while time.time() < deadline: + if check_fn(): + ok(desc) + return + time.sleep(interval) + die(f"timed out waiting for {desc}") + + +def step_up(cfg: Cfg) -> None: + log("up: starting the domain stack") + run(compose_cmd(cfg, "up", "-d")) + + def bucket_ready() -> bool: + r = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Status}}:{{.State.ExitCode}}", + f"minio-init-{cfg.domain}"], + text=True, capture_output=True, + ) + return r.stdout.strip() == "exited:0" + + def pg_ready() -> bool: + r = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Health.Status}}", + f"postgres-{cfg.domain}"], + text=True, capture_output=True, + ) + return r.stdout.strip() == "healthy" + + _wait(cfg, f"{cfg.domain}-iceberg bucket (minio-init exit 0)", bucket_ready) + _wait(cfg, f"postgres-{cfg.domain} healthy", pg_ready) + + +def step_ducklake(cfg: Cfg) -> None: + """Post-startup: attach the DuckLake catalog inside the analytics DB.""" + log("ducklake: attaching catalog in analytics DB") + access = env_value(cfg, f"MINIO_{cfg.upper}_ACCESS_KEY") + secret = env_value(cfg, f"MINIO_{cfg.upper}_SECRET_KEY") + pg_pw = env_value(cfg, f"POSTGRES_{cfg.upper}_PASSWORD") + if not all([access, secret, pg_pw]): + die("missing MinIO/postgres secrets in .env for analytics attach") + sql = f""" +SELECT duckdb.create_secret( + '{cfg.domain}_minio', 'S3', + key_id := '{access}', secret := '{secret}', + endpoint := 'minio-{cfg.domain}:9000', url_style := 'path', use_ssl := false +); +SELECT duckdb.raw_query($q$ + ATTACH 'ducklake:postgres:host=postgres-{cfg.domain} port=5432 dbname={cfg.domain} user={cfg.domain} password={pg_pw}' + AS "{cfg.domain}_lake"; +$q$); +""" + # analytics service runs as user 'analytics', db '' + run([ + "docker", "exec", "-i", f"analytics-{cfg.domain}", + "psql", "-v", "ON_ERROR_STOP=1", "-U", "analytics", "-d", cfg.domain, + ] + ["-c", sql], check=False) + ok("analytics DuckLake attach attempted (idempotent; safe to re-run)") + + +def step_register(cfg: Cfg) -> None: + log("register: core + management (reversible, one-way)") + run([str(cfg.template / "domain-integration.sh"), cfg.domain, "register", "all"]) + ok("registered with core + management") + + +def step_activate(cfg: Cfg) -> None: + log("activate: restart ariadne + flink to pick up OIDC + metrics") + run(compose_cmd(cfg, "restart", + f"ariadne-{cfg.domain}", f"flink-jm-{cfg.domain}")) + ok("ariadne + flink restarted") + + +def step_verify(cfg: Cfg) -> None: + log("verify: checking domain stability") + expected = [f"{p}-{cfg.domain}" for p in + ("postgres", "kafka", "flink-jm", "flink-tm", "minio", "analytics", "ariadne")] + r = subprocess.run(["docker", "ps", "--format", "{{.Names}}"], + text=True, capture_output=True) + running = set(r.stdout.split()) + missing = [c for c in expected if c not in running] + if missing: + die(f"services not running: {', '.join(missing)}") + ok(f"{len(expected)} services running: {', '.join(expected)}") + + # MinIO bucket + rc = subprocess.run( + ["docker", "exec", f"minio-{cfg.domain}", "sh", "-c", + f"mc alias set local http://localhost:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD >/dev/null 2>&1; " + f"mc ls local/{cfg.domain}-iceberg >/dev/null 2>&1 && echo OK"], + text=True, capture_output=True, + ) + ok(f"{cfg.domain}-iceberg bucket present") if "OK" in rc.stdout else \ + print(f" ! could not confirm bucket (non-fatal): {rc.stderr.strip()}") + + print("\n Next (VPN required) — manual spot checks:") + print(f" GraphQL : http://api.{cfg.domain}.data.internal:{cfg.port_base}/health") + print(f" Flink : http://flink.{cfg.domain}.data.internal:{cfg.port_base + 1}") + print(f" Keycloak: client data-{cfg.domain} + /data-{cfg.domain}/* groups") + print(f" Catalog : OpenMetadata domain '{cfg.domain}'") + + +def step_teardown(cfg: Cfg) -> None: + log("teardown: unregister + stop (reverses the stand-up)") + run([str(cfg.template / "domain-integration.sh"), cfg.domain, "unregister", "all"], + check=False) + run(compose_cmd(cfg, "down", "-v"), check=False) + ok("domain unregistered + stopped") + + +STEPS = { + "preflight": step_preflight, + "wire": step_wire, + "secrets": step_secrets, + "scaffold": step_scaffold, + "build": step_build, + "up": step_up, + "ducklake": step_ducklake, + "register": step_register, + "activate": step_activate, + "verify": step_verify, + "teardown": step_teardown, +} + +ALL = ["preflight", "wire", "secrets", "scaffold", "build", "up", + "ducklake", "register", "activate", "verify"] + + +def main() -> None: + args = sys.argv[1:] or ["all"] + requested: list[str] = [] + for a in args: + if a == "all": + requested.extend(ALL) + elif a in STEPS: + requested.append(a) + else: + die(f"unknown step '{a}'. valid: {', '.join(STEPS)}, all") + cfg = load_cfg() + log(f"domain={cfg.domain} metadata={cfg.metadata_dir} port_base={cfg.port_base}") + for name in requested: + STEPS[name](cfg) + log("done") + + +if __name__ == "__main__": + main() diff --git a/financial_crime/domain.md b/financial_crime/domain.md index b2ea4f7..395011e 100644 --- a/financial_crime/domain.md +++ b/financial_crime/domain.md @@ -125,9 +125,9 @@ graph TD Business Application | Platform | Capability Domain --- | --- | --- -[Temenos Payment](sources/temenos-payment/source.md) | Temenos SaaS | Payment Execution -[SAP Fraud Management](sources/sap-fraud-management/source.md) | SAP | Fraud -[Salesforce CRM](sources/salesforce-crm/source.md) | Salesforce | Customer Relationship Management +[Temenos Payment](../sources/temenos-payment/source.md) | Temenos SaaS | Payment Execution +[SAP Fraud Management](../sources/sap-fraud-management/source.md) | SAP | Fraud +[Salesforce CRM](../sources/salesforce-crm/source.md) | Salesforce | Customer Relationship Management ## Entities diff --git a/financial_crime/sources/salesforce-crm/source.md b/financial_crime/sources/salesforce-crm/source.md deleted file mode 100644 index cb62dc0..0000000 --- a/financial_crime/sources/salesforce-crm/source.md +++ /dev/null @@ -1,34 +0,0 @@ -# Salesforce CRM - -Salesforce CRM is the customer relationship source for onboarding, profile maintenance, and communication preferences. It contributes party identity, customer profile, and contact data used by KYC and due diligence workflows. - -## Metadata - -```yaml -id: salesforce-crm -owner: crm.platform@bank.com -steward: data.governance@bank.com - -change_model: real-time-cdc -change_events: - - Customer Created - - Customer Updated - - Contact Address Updated - - Customer Preference Updated - -update_frequency: real-time -data_quality_tier: 1 -status: Production -version: "1.0.0" -``` - -## [Financial Crime](../../domain.md) Feeds - -Canonical Entity | Transform File | Attributes Contributed | Change Model ---- | --- | --- | --- -[Party](../../entities/party.md#party) | [table_account](transforms/table_account.md) | Party Identifier, Party Status | real-time-cdc -[Person](../../entities/person.md#person) | [table_contact](transforms/table_contact.md) | Given Name, Family Name, Date of Birth, PEP Status | real-time-cdc -[Company](../../entities/company.md#company) | [table_account](transforms/table_account.md) | Legal Name, Registration Identifier | real-time-cdc -[Customer](../../entities/customer.md#customer) | [table_account](transforms/table_account.md) | Customer Number, Onboarding Date, Segment | real-time-cdc -[Contact Address](../../entities/contact_address.md#contact-address) | [table_contact_point](transforms/table_contact_point.md) | Address Purpose, Verification Status, Effective Dates | real-time-cdc -[Customer Preferences](../../entities/customer-preferences.md#customer-preferences) | [table_preference](transforms/table_preference.md) | Preferred Contact Channel, Marketing Consent | event-driven diff --git a/financial_crime/sources/temenos-payment/source.md b/financial_crime/sources/temenos-payment/source.md deleted file mode 100644 index 0c3231a..0000000 --- a/financial_crime/sources/temenos-payment/source.md +++ /dev/null @@ -1,38 +0,0 @@ -# Temenos Payment - -Temenos Payment is the operational source for payment initiation and execution records. It emits high-volume transaction changes used for financial crime monitoring and downstream payment analytics. - -## Metadata - -```yaml -id: temenos-payment -owner: payments.platform@bank.com -steward: data.governance@bank.com - -change_model: real-time-cdc -change_events: - - Payment Initiated - - Payment Executed - - Payment Reversed - - Payment Rejected - -update_frequency: real-time -data_quality_tier: 1 -status: Production -version: "1.0.0" - -tags: - - Payments - - Core Banking - - Financial Crime -``` - -## [Financial Crime](../../domain.md) Feeds - -Canonical Entity | Transform File | Attributes Contributed | Change Model ---- | --- | --- | --- -[Transaction](../../entities/transaction.md#transaction) | [table_payment_event](transforms/table_payment_event.md) | Transaction Identifier, Amount, Currency, Execution Timestamp, Status | real-time-cdc -[Account](../../entities/account.md#account) | [table_account_ref](transforms/table_account_ref.md) | Account Identifier, Account Status, Product Identifier | real-time-cdc -[Payment Initiator](../../entities/payment_initiator.md#payment-initiator) | [table_initiation](transforms/table_initiation.md) | Initiator Role Identifier, Initiation Channel | event-driven -[Payer](../../entities/payer.md#payer) | [table_payment_parties](transforms/table_payment_parties.md) | Payer Role Identifier | event-driven -[Payee](../../entities/payee.md#payee) | [table_payment_parties](transforms/table_payment_parties.md) | Payee Role Identifier | event-driven diff --git a/sources/salesforce-crm/source.md b/sources/salesforce-crm/source.md new file mode 100644 index 0000000..08c0698 --- /dev/null +++ b/sources/salesforce-crm/source.md @@ -0,0 +1,34 @@ +# Salesforce CRM + +Salesforce CRM is the customer relationship source for onboarding, profile maintenance, and communication preferences. It contributes party identity, customer profile, and contact data used by KYC and due diligence workflows. + +## Metadata + +```yaml +id: salesforce-crm +owner: crm.platform@bank.com +steward: data.governance@bank.com + +change_model: real-time-cdc +change_events: + - Customer Created + - Customer Updated + - Contact Address Updated + - Customer Preference Updated + +update_frequency: real-time +data_quality_tier: 1 +status: Production +version: "1.0.0" +``` + +## [Financial Crime](../../financial_crime/domain.md) Feeds + +Canonical Entity | Transform File | Attributes Contributed | Change Model +--- | --- | --- | --- +[Party](../../financial_crime/entities/party.md#party) | [table_account](transforms/table_account.md) | Party Identifier, Party Status | real-time-cdc +[Person](../../financial_crime/entities/person.md#person) | [table_contact](transforms/table_contact.md) | Given Name, Family Name, Date of Birth, PEP Status | real-time-cdc +[Company](../../financial_crime/entities/company.md#company) | [table_account](transforms/table_account.md) | Legal Name, Registration Identifier | real-time-cdc +[Customer](../../financial_crime/entities/customer.md#customer) | [table_account](transforms/table_account.md) | Customer Number, Onboarding Date, Segment | real-time-cdc +[Contact Address](../../financial_crime/entities/contact_address.md#contact-address) | [table_contact_point](transforms/table_contact_point.md) | Address Purpose, Verification Status, Effective Dates | real-time-cdc +[Customer Preferences](../../financial_crime/entities/customer-preferences.md#customer-preferences) | [table_preference](transforms/table_preference.md) | Preferred Contact Channel, Marketing Consent | event-driven diff --git a/financial_crime/sources/salesforce-crm/transforms/table_account.md b/sources/salesforce-crm/transforms/table_account.md similarity index 100% rename from financial_crime/sources/salesforce-crm/transforms/table_account.md rename to sources/salesforce-crm/transforms/table_account.md diff --git a/financial_crime/sources/salesforce-crm/transforms/table_contact.md b/sources/salesforce-crm/transforms/table_contact.md similarity index 100% rename from financial_crime/sources/salesforce-crm/transforms/table_contact.md rename to sources/salesforce-crm/transforms/table_contact.md diff --git a/financial_crime/sources/salesforce-crm/transforms/table_contact_point.md b/sources/salesforce-crm/transforms/table_contact_point.md similarity index 100% rename from financial_crime/sources/salesforce-crm/transforms/table_contact_point.md rename to sources/salesforce-crm/transforms/table_contact_point.md diff --git a/financial_crime/sources/salesforce-crm/transforms/table_preference.md b/sources/salesforce-crm/transforms/table_preference.md similarity index 100% rename from financial_crime/sources/salesforce-crm/transforms/table_preference.md rename to sources/salesforce-crm/transforms/table_preference.md diff --git a/financial_crime/sources/sap-fraud-management/source.md b/sources/sap-fraud-management/source.md similarity index 51% rename from financial_crime/sources/sap-fraud-management/source.md rename to sources/sap-fraud-management/source.md index 073c631..07ce2a9 100644 --- a/financial_crime/sources/sap-fraud-management/source.md +++ b/sources/sap-fraud-management/source.md @@ -27,10 +27,10 @@ tags: - Financial Crime ``` -## [Financial Crime](../../domain.md) Feeds +## [Financial Crime](../../financial_crime/domain.md) Feeds Canonical Entity | Transform File | Attributes Contributed | Change Model --- | --- | --- | --- -[Transaction](../../entities/transaction.md#transaction) | [table_alert_case](transforms/table_alert_case.md) | Financial Crime Risk Score, Monitoring Outcome, Alert Reference | event-driven -[Party](../../entities/party.md#party) | [table_sanctions_screening](transforms/table_sanctions_screening.md) | Sanctions Screen Status, Watchlist Match Indicator | batch-intraday -[Customer](../../entities/customer.md#customer) | [table_customer_risk_profile](transforms/table_customer_risk_profile.md) | Risk Review Flag, Enhanced Due Diligence Trigger | event-driven +[Transaction](../../financial_crime/entities/transaction.md#transaction) | [table_alert_case](transforms/table_alert_case.md) | Financial Crime Risk Score, Monitoring Outcome, Alert Reference | event-driven +[Party](../../financial_crime/entities/party.md#party) | [table_sanctions_screening](transforms/table_sanctions_screening.md) | Sanctions Screen Status, Watchlist Match Indicator | batch-intraday +[Customer](../../financial_crime/entities/customer.md#customer) | [table_customer_risk_profile](transforms/table_customer_risk_profile.md) | Risk Review Flag, Enhanced Due Diligence Trigger | event-driven diff --git a/financial_crime/sources/sap-fraud-management/transforms/table_alert_case.md b/sources/sap-fraud-management/transforms/table_alert_case.md similarity index 100% rename from financial_crime/sources/sap-fraud-management/transforms/table_alert_case.md rename to sources/sap-fraud-management/transforms/table_alert_case.md diff --git a/financial_crime/sources/sap-fraud-management/transforms/table_customer_risk_profile.md b/sources/sap-fraud-management/transforms/table_customer_risk_profile.md similarity index 100% rename from financial_crime/sources/sap-fraud-management/transforms/table_customer_risk_profile.md rename to sources/sap-fraud-management/transforms/table_customer_risk_profile.md diff --git a/financial_crime/sources/sap-fraud-management/transforms/table_sanctions_screening.md b/sources/sap-fraud-management/transforms/table_sanctions_screening.md similarity index 100% rename from financial_crime/sources/sap-fraud-management/transforms/table_sanctions_screening.md rename to sources/sap-fraud-management/transforms/table_sanctions_screening.md diff --git a/sources/temenos-payment/source.md b/sources/temenos-payment/source.md new file mode 100644 index 0000000..03b3e29 --- /dev/null +++ b/sources/temenos-payment/source.md @@ -0,0 +1,38 @@ +# Temenos Payment + +Temenos Payment is the operational source for payment initiation and execution records. It emits high-volume transaction changes used for financial crime monitoring and downstream payment analytics. + +## Metadata + +```yaml +id: temenos-payment +owner: payments.platform@bank.com +steward: data.governance@bank.com + +change_model: real-time-cdc +change_events: + - Payment Initiated + - Payment Executed + - Payment Reversed + - Payment Rejected + +update_frequency: real-time +data_quality_tier: 1 +status: Production +version: "1.0.0" + +tags: + - Payments + - Core Banking + - Financial Crime +``` + +## [Financial Crime](../../financial_crime/domain.md) Feeds + +Canonical Entity | Transform File | Attributes Contributed | Change Model +--- | --- | --- | --- +[Transaction](../../financial_crime/entities/transaction.md#transaction) | [table_payment_event](transforms/table_payment_event.md) | Transaction Identifier, Amount, Currency, Execution Timestamp, Status | real-time-cdc +[Account](../../financial_crime/entities/account.md#account) | [table_account_ref](transforms/table_account_ref.md) | Account Identifier, Account Status, Product Identifier | real-time-cdc +[Payment Initiator](../../financial_crime/entities/payment_initiator.md#payment-initiator) | [table_initiation](transforms/table_initiation.md) | Initiator Role Identifier, Initiation Channel | event-driven +[Payer](../../financial_crime/entities/payer.md#payer) | [table_payment_parties](transforms/table_payment_parties.md) | Payer Role Identifier | event-driven +[Payee](../../financial_crime/entities/payee.md#payee) | [table_payment_parties](transforms/table_payment_parties.md) | Payee Role Identifier | event-driven diff --git a/financial_crime/sources/temenos-payment/transforms/table_account_ref.md b/sources/temenos-payment/transforms/table_account_ref.md similarity index 100% rename from financial_crime/sources/temenos-payment/transforms/table_account_ref.md rename to sources/temenos-payment/transforms/table_account_ref.md diff --git a/financial_crime/sources/temenos-payment/transforms/table_initiation.md b/sources/temenos-payment/transforms/table_initiation.md similarity index 100% rename from financial_crime/sources/temenos-payment/transforms/table_initiation.md rename to sources/temenos-payment/transforms/table_initiation.md diff --git a/financial_crime/sources/temenos-payment/transforms/table_payment_event.md b/sources/temenos-payment/transforms/table_payment_event.md similarity index 100% rename from financial_crime/sources/temenos-payment/transforms/table_payment_event.md rename to sources/temenos-payment/transforms/table_payment_event.md diff --git a/financial_crime/sources/temenos-payment/transforms/table_payment_parties.md b/sources/temenos-payment/transforms/table_payment_parties.md similarity index 100% rename from financial_crime/sources/temenos-payment/transforms/table_payment_parties.md rename to sources/temenos-payment/transforms/table_payment_parties.md