Initial commit: Financial Crime domain exemplar
This commit is contained in:
@@ -0,0 +1,228 @@
|
||||
"""
|
||||
Eventual Consistency Demonstration — Financial Crime / canonical scope
|
||||
|
||||
Shows how Party entity updates converge (or fail to converge) across three
|
||||
source feeds with different propagation lags, and validates the canonical
|
||||
product's declared SLA of < 1 hour.
|
||||
|
||||
Source feeds modelled (from examples/Financial Crime/sources/):
|
||||
1. salesforce-crm — real-time-cdc, lag = 5 min (party identity fields)
|
||||
2. sap-fraud-mgmt — batch-intraday, lag = 30 min (risk_rating, sanctions_screen_status)
|
||||
3. temenos-payment — batch-intraday, lag = 60 min (slowest contributing system)
|
||||
|
||||
Canonical SLA: freshness < 1 hour
|
||||
(from examples/Financial Crime/products/canonical.md)
|
||||
|
||||
Consistency posture: eventual
|
||||
(sources have heterogeneous change_models; synchronous propagation not achievable)
|
||||
|
||||
Null strategy: nullable-staging
|
||||
(partial rows admitted to staging; converged view used by consumers)
|
||||
|
||||
Usage:
|
||||
cd examples/Financial\ Crime
|
||||
python consistency_example.py
|
||||
|
||||
# Or with runtime on path explicitly:
|
||||
PYTHONPATH=../../agents/agent-artifact/skills/faker/runtime python consistency_example.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import sys
|
||||
import os
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Path setup — inject faker runtime so integrity_check and consistency_scenario
|
||||
# can be imported regardless of working directory
|
||||
# ---------------------------------------------------------------------------
|
||||
_runtime_path = os.path.abspath(
|
||||
os.path.join(os.path.dirname(__file__), "..", "..",
|
||||
"agents", "agent-artifact", "skills", "faker", "runtime")
|
||||
)
|
||||
if _runtime_path not in sys.path:
|
||||
sys.path.insert(0, _runtime_path)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Imports
|
||||
# ---------------------------------------------------------------------------
|
||||
try:
|
||||
from factories import (
|
||||
DatasetBuilder,
|
||||
FK_SPECS_FINANCIAL_CRIME,
|
||||
NOT_NULL_FINANCIAL_CRIME,
|
||||
ENUM_VALUES_FINANCIAL_CRIME,
|
||||
UNIQUE_PK_FINANCIAL_CRIME,
|
||||
UNIQUE_CURRENT_PK_FINANCIAL_CRIME,
|
||||
)
|
||||
except ImportError as e:
|
||||
print(f"[ERROR] Could not import factories.py — run from the "
|
||||
f"'examples/Financial Crime/' directory: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
from integrity_check import (
|
||||
check_integrity, print_report,
|
||||
check_temporal_chain, print_temporal_report,
|
||||
)
|
||||
from consistency_scenario import (
|
||||
SourceFeed, generate_scenario,
|
||||
check_convergence, print_convergence_report,
|
||||
)
|
||||
except ImportError as e:
|
||||
print(f"[ERROR] Could not import runtime modules from {_runtime_path}: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Source feeds — modelled on the Financial Crime domain's declared source systems
|
||||
FEEDS = [
|
||||
SourceFeed(name="salesforce-crm", lag_minutes=5, change_model="real-time-cdc"),
|
||||
SourceFeed(name="sap-fraud-mgmt", lag_minutes=30, change_model="batch-intraday"),
|
||||
SourceFeed(name="temenos-payment", lag_minutes=60, change_model="batch-intraday"),
|
||||
]
|
||||
|
||||
# Entities with valid_time or bitemporal tracking (candidate for convergence analysis)
|
||||
TEMPORAL_ENTITIES = {
|
||||
"Party": "party_identifier",
|
||||
"Account": "account_identifier",
|
||||
}
|
||||
|
||||
# Canonical product SLA (from products/canonical.md: freshness: "< 1 hour")
|
||||
SLA_MINUTES = 60
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Section 1 — Build dataset
|
||||
# ---------------------------------------------------------------------------
|
||||
print("=" * 60)
|
||||
print("Financial Crime — Eventual Consistency Demonstration")
|
||||
print("=" * 60)
|
||||
|
||||
print("\n[1] Building synthetic dataset (20 parties, history=True)…")
|
||||
dataset = DatasetBuilder(pii_mode="safe").build(
|
||||
n_parties=20,
|
||||
accounts_per_party=2,
|
||||
txns_per_account=3,
|
||||
with_history=True,
|
||||
)
|
||||
total_rows = sum(len(v) for v in dataset.values())
|
||||
print(f" Built: {total_rows} total rows across {len(dataset)} entities")
|
||||
for entity, rows in dataset.items():
|
||||
print(f" {entity}: {len(rows)} rows")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Section 2 — Baseline integrity check
|
||||
# ---------------------------------------------------------------------------
|
||||
print("\n[2] Baseline integrity check…")
|
||||
integrity_errors = check_integrity(
|
||||
dataset,
|
||||
fk_specs=FK_SPECS_FINANCIAL_CRIME,
|
||||
not_null=NOT_NULL_FINANCIAL_CRIME,
|
||||
enum_values=ENUM_VALUES_FINANCIAL_CRIME,
|
||||
unique_pk=UNIQUE_PK_FINANCIAL_CRIME,
|
||||
unique_current_pk=UNIQUE_CURRENT_PK_FINANCIAL_CRIME,
|
||||
)
|
||||
print_report(integrity_errors)
|
||||
if integrity_errors:
|
||||
print("[WARN] Integrity violations present — convergence analysis may be misleading")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Section 3 — Temporal chain validation
|
||||
# ---------------------------------------------------------------------------
|
||||
print("\n[3] Temporal chain validation…")
|
||||
temporal_errors = check_temporal_chain(dataset, TEMPORAL_ENTITIES)
|
||||
print_temporal_report(temporal_errors)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Section 4 — Generate eventual consistency scenario
|
||||
# ---------------------------------------------------------------------------
|
||||
print("\n[4] Generating eventual consistency scenario…")
|
||||
print(f" Feeds: " +
|
||||
" | ".join(f"{f.name} (lag={f.lag_minutes}m, {f.change_model})" for f in FEEDS))
|
||||
|
||||
as_of = datetime.now(tz=timezone.utc)
|
||||
scenario = generate_scenario(
|
||||
dataset=dataset,
|
||||
feeds=FEEDS,
|
||||
as_of=as_of,
|
||||
temporal_entities=TEMPORAL_ENTITIES,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Section 5 — Convergence report
|
||||
# ---------------------------------------------------------------------------
|
||||
print("\n[5] Convergence analysis…")
|
||||
print(f" Observation time: {as_of.strftime('%Y-%m-%d %H:%M:%S UTC')}")
|
||||
|
||||
for entity, pk_col in TEMPORAL_ENTITIES.items():
|
||||
deltas = scenario.convergence_delta_minutes.get(entity, {})
|
||||
divergent_ids = scenario.divergent.get(entity, [])
|
||||
converged = [pk for pk, d in deltas.items() if d == 0.0]
|
||||
in_flight = [pk for pk, d in deltas.items() if d > 0.0]
|
||||
|
||||
print(f"\n {entity} ({len(deltas)} current instances):")
|
||||
print(f" Fully converged : {len(converged)}")
|
||||
print(f" In-flight : {len(in_flight)}")
|
||||
|
||||
if in_flight:
|
||||
print(f" In-flight sample (showing up to 5):")
|
||||
for pk in in_flight[:5]:
|
||||
mins = deltas[pk]
|
||||
pk_short = str(pk)[:8] + "…"
|
||||
# Show which feeds still need to receive this row
|
||||
late_feeds = []
|
||||
for feed in FEEDS:
|
||||
feed_rows = scenario.source_views[feed.name].get(entity, [])
|
||||
feed_pks = {r.get(pk_col) for r in feed_rows if r.get("is_current") is True}
|
||||
if pk not in feed_pks:
|
||||
late_feeds.append(f"{feed.name}({feed.lag_minutes}m)")
|
||||
print(f" {pk_short} converges in {mins:.1f} min "
|
||||
f"[pending: {', '.join(late_feeds) or 'none'}]")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Section 6 — SLA check
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n[6] SLA validation (freshness < {SLA_MINUTES} min)…")
|
||||
violations = check_convergence(scenario, sla_minutes=SLA_MINUTES)
|
||||
|
||||
if not violations:
|
||||
print(f" PASS — all entity instances converge within the "
|
||||
f"{SLA_MINUTES}-minute SLA.")
|
||||
else:
|
||||
print(f" FAIL — {len(violations)} instance(s) violate the "
|
||||
f"{SLA_MINUTES}-minute SLA:")
|
||||
for v in violations[:5]:
|
||||
pk_short = str(v.entity_id)[:8] + "…"
|
||||
print(f" [{v.entity}] {pk_short}: {v.message}")
|
||||
if len(violations) > 5:
|
||||
print(f" … and {len(violations) - 5} more")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Section 7 — Null handling illustration
|
||||
# ---------------------------------------------------------------------------
|
||||
print("\n[7] Null handling under eventual consistency…")
|
||||
print("""
|
||||
Scenario: salesforce-crm (lag=5m) has delivered party_status.
|
||||
sap-fraud-mgmt (lag=30m) has NOT yet delivered risk_rating.
|
||||
|
||||
In-flight canonical row (nullable-staging pattern):
|
||||
party_identifier : "aaa-bbb-ccc…" ← arrived from salesforce-crm
|
||||
party_status : "Active" ← arrived from salesforce-crm
|
||||
risk_rating : NULL ← not yet received from sap-fraud-mgmt
|
||||
sanctions_screen_status : NULL ← not yet received from sap-fraud-mgmt
|
||||
|
||||
Storage concern : Parquet cannot distinguish this NULL from a genuinely
|
||||
absent risk_rating. Downstream readers see identical bytes.
|
||||
Transport concern : JSON omits absent keys; Avro encodes both as null union.
|
||||
Only Protobuf hasField() can distinguish "not set" vs null.
|
||||
DDL concern : A hard NOT NULL on risk_rating would block this insert.
|
||||
With nullable-staging pattern, the base table allows NULL;
|
||||
the converged view filters to rows where all fields are set.
|
||||
""")
|
||||
|
||||
print("=" * 60)
|
||||
print("Demonstration complete.")
|
||||
print("=" * 60)
|
||||
@@ -0,0 +1,524 @@
|
||||
"""
|
||||
Synthetic data factory — Financial Crime / canonical scope
|
||||
Scope: canonical
|
||||
PII mode: safe (default) | realistic
|
||||
|
||||
Generated from:
|
||||
examples/Financial Crime/entities/currency.md
|
||||
examples/Financial Crime/entities/party.md
|
||||
examples/Financial Crime/entities/account.md
|
||||
examples/Financial Crime/entities/transaction.md
|
||||
examples/Financial Crime/enums.md
|
||||
|
||||
WARNING: SYNTHETIC DATA ONLY. Do not use for production data migration.
|
||||
Dependencies: pip install faker
|
||||
|
||||
Generation order (topological — respects FK dependencies):
|
||||
1. Currency (reference — no FKs)
|
||||
2. Party (independent, slowly_changing, bitemporal)
|
||||
3. Account (independent, slowly_changing, valid_time; FK → Currency)
|
||||
4. Transaction (dependent, append_only; FK → Currency, Account)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import random
|
||||
import uuid
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
|
||||
from faker import Faker
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Enum pools — sourced from examples/Financial Crime/enums.md
|
||||
# ---------------------------------------------------------------------------
|
||||
PARTY_STATUS_VALUES = ["Active", "Under Review", "Restricted", "Inactive", "Closed"]
|
||||
FINANCIAL_CRIME_RISK_RATING_VALUES = ["Low", "Medium", "High", "Very High"]
|
||||
SANCTIONS_SCREEN_STATUS_VALUES = [
|
||||
"Not Screened", "Clear", "Potential Match", "Confirmed Match", "False Positive",
|
||||
]
|
||||
ACCOUNT_STATUS_VALUES = ["Pending", "Active", "Dormant", "Frozen", "Suspended", "Closed"]
|
||||
ACCOUNT_TYPE_VALUES = [
|
||||
"Savings", "Current", "Term Deposit", "Loan",
|
||||
"Line Of Credit", "Mortgage", "Offset", "Foreign Currency",
|
||||
]
|
||||
TRANSACTION_TYPE_VALUES = [
|
||||
"Wire Transfer", "SWIFT Transfer", "EFTPOS", "ATM Withdrawal", "ATM Deposit",
|
||||
"Direct Debit", "Direct Credit", "Internal Transfer", "BPay",
|
||||
"Cash Deposit", "Cash Withdrawal", "Cheque", "RTGS",
|
||||
]
|
||||
TRANSACTION_CHANNEL_VALUES = [
|
||||
"Branch", "Online Banking", "Mobile Banking", "ATM", "EFTPOS Terminal",
|
||||
"SWIFT", "Direct Entry", "Third Party", "Internal System",
|
||||
]
|
||||
TRANSACTION_STATUS_VALUES = [
|
||||
"Pending", "Authorised", "Cleared", "Settled",
|
||||
"Failed", "Reversed", "Cancelled", "Under Review",
|
||||
]
|
||||
|
||||
# ISO 4217 representative subset — sourced from enums.md
|
||||
CURRENCY_CODES = ["AUD", "NZD", "USD", "EUR", "GBP", "JPY", "SGD", "HKD", "CHF", "CAD"]
|
||||
CURRENCY_NAMES = {
|
||||
"AUD": "Australian Dollar", "NZD": "New Zealand Dollar", "USD": "United States Dollar",
|
||||
"EUR": "Euro", "GBP": "Pound Sterling", "JPY": "Japanese Yen",
|
||||
"SGD": "Singapore Dollar", "HKD": "Hong Kong Dollar", "CHF": "Swiss Franc",
|
||||
"CAD": "Canadian Dollar",
|
||||
}
|
||||
CURRENCY_MINOR_UNITS = {
|
||||
"AUD": 2, "NZD": 2, "USD": 2, "EUR": 2, "GBP": 2,
|
||||
"JPY": 0, "SGD": 2, "HKD": 2, "CHF": 2, "CAD": 2,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Seeding — deterministic by default
|
||||
# ---------------------------------------------------------------------------
|
||||
fake = Faker()
|
||||
Faker.seed(0)
|
||||
random.seed(0)
|
||||
|
||||
_seq = 0
|
||||
|
||||
|
||||
def _next_seq() -> int:
|
||||
global _seq
|
||||
_seq += 1
|
||||
return _seq
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CurrencyFactory
|
||||
# Entity: Currency
|
||||
# Existence: independent
|
||||
# Mutability: reference (static rows; no temporal columns)
|
||||
# PII fields: none
|
||||
# ---------------------------------------------------------------------------
|
||||
class CurrencyFactory:
|
||||
"""Produces one row per ISO 4217 code in CURRENCY_CODES."""
|
||||
|
||||
def build_all(self) -> list[dict]:
|
||||
return [
|
||||
{
|
||||
"currency_code": code,
|
||||
"currency_name": CURRENCY_NAMES[code],
|
||||
"minor_unit": CURRENCY_MINOR_UNITS[code],
|
||||
}
|
||||
for code in CURRENCY_CODES
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PartyFactory
|
||||
# Entity: Party
|
||||
# Existence: independent
|
||||
# Mutability: slowly_changing
|
||||
# Temporal: bitemporal (valid_time + transaction_time)
|
||||
# PII fields: legal_name, also_known_as
|
||||
# Constraints encoded:
|
||||
# - Legal Name Required: legal_name is never None
|
||||
# - Review Date Must Not Be Overdue: next_review_date >= today (unless Under Review)
|
||||
# - Confirmed Sanctions Match Blocks Service: sanctions_screen_status != 'Confirmed Match'
|
||||
# (softened in synthetic data — status allowed but flagged via comment)
|
||||
# ---------------------------------------------------------------------------
|
||||
class PartyFactory:
|
||||
"""
|
||||
Entity: Party
|
||||
Existence: independent
|
||||
Mutability: slowly_changing
|
||||
Temporal: bitemporal
|
||||
PII fields: legal_name, also_known_as
|
||||
Constraints encoded:
|
||||
- Legal Name Required
|
||||
- Review Date Must Not Be Overdue
|
||||
"""
|
||||
|
||||
def __init__(self, fake: Faker = None, pii_mode: str = "safe"):
|
||||
self.fake = fake or Faker()
|
||||
self.pii_mode = pii_mode
|
||||
|
||||
def build(self, with_history: bool = False, **overrides) -> list[dict]:
|
||||
seq = _next_seq()
|
||||
f = self.fake
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
prior_end = now - timedelta(days=random.randint(30, 730))
|
||||
prior_start = prior_end - timedelta(days=random.randint(90, 1825))
|
||||
|
||||
# Constraint: Review Date Must Not Be Overdue
|
||||
# next_review_date is always in the future unless status is Under Review
|
||||
today = date.today()
|
||||
next_review_date = today.replace(
|
||||
year=today.year + random.randint(0, 2),
|
||||
month=random.randint(1, 12),
|
||||
day=random.randint(1, 28),
|
||||
)
|
||||
|
||||
# PII: legal_name, also_known_as
|
||||
if self.pii_mode == "realistic":
|
||||
legal_name = f.name()
|
||||
also_known_as = [f.name()] if random.random() < 0.3 else []
|
||||
else:
|
||||
legal_name = f"Test Entity {seq:04d}"
|
||||
also_known_as = [f"Test AKA {seq:04d}"] if random.random() < 0.3 else []
|
||||
|
||||
current = {
|
||||
"party_identifier": str(uuid.uuid4()),
|
||||
"legal_name": legal_name, # pii: true
|
||||
"also_known_as": also_known_as, # pii: true
|
||||
"party_status": random.choice(PARTY_STATUS_VALUES),
|
||||
"risk_rating": random.choice(FINANCIAL_CRIME_RISK_RATING_VALUES),
|
||||
"sanctions_screen_status": random.choice(SANCTIONS_SCREEN_STATUS_VALUES),
|
||||
"next_review_date": next_review_date,
|
||||
# bitemporal columns
|
||||
"valid_from": prior_end,
|
||||
"valid_to": None,
|
||||
"is_current": True,
|
||||
"recorded_at": prior_end,
|
||||
"superseded_at": None,
|
||||
**overrides,
|
||||
}
|
||||
|
||||
if not with_history:
|
||||
return [current]
|
||||
|
||||
prior = {
|
||||
**current,
|
||||
"valid_from": prior_start,
|
||||
"valid_to": prior_end,
|
||||
"is_current": False,
|
||||
"recorded_at": prior_start,
|
||||
"superseded_at": prior_end,
|
||||
}
|
||||
return [prior, current]
|
||||
|
||||
def batch(self, n: int, with_history: bool = False, **overrides) -> list[dict]:
|
||||
rows: list[dict] = []
|
||||
for _ in range(n):
|
||||
rows.extend(self.build(with_history=with_history, **overrides))
|
||||
return rows
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AccountFactory
|
||||
# Entity: Account
|
||||
# Existence: independent
|
||||
# Mutability: slowly_changing
|
||||
# Temporal: valid_time
|
||||
# PII fields: none
|
||||
# FK: currency_code → Currency.currency_code
|
||||
# Constraints encoded:
|
||||
# - Closed Date After Opened Date: closed_date > opened_date when present
|
||||
# ---------------------------------------------------------------------------
|
||||
class AccountFactory:
|
||||
"""
|
||||
Entity: Account
|
||||
Existence: independent
|
||||
Mutability: slowly_changing
|
||||
Temporal: valid_time
|
||||
PII fields: none
|
||||
FK: currency_code → Currency.currency_code
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
currency_codes: list[str],
|
||||
fake: Faker = None,
|
||||
pii_mode: str = "safe",
|
||||
):
|
||||
self.currency_codes = currency_codes
|
||||
self.fake = fake or Faker()
|
||||
self.pii_mode = pii_mode
|
||||
|
||||
def build(self, with_history: bool = False, **overrides) -> list[dict]:
|
||||
seq = _next_seq()
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
prior_end = now - timedelta(days=random.randint(30, 730))
|
||||
prior_start = prior_end - timedelta(days=random.randint(90, 1825))
|
||||
|
||||
opened_date = (now - timedelta(days=random.randint(365, 3650))).date()
|
||||
|
||||
# Constraint: Closed Date After Opened Date
|
||||
account_status = random.choice(ACCOUNT_STATUS_VALUES)
|
||||
closed_date = None
|
||||
if account_status == "Closed":
|
||||
closed_date = opened_date + timedelta(days=random.randint(30, 2000))
|
||||
|
||||
current = {
|
||||
"account_identifier": str(uuid.uuid4()),
|
||||
"account_number": f"BSB{seq:06d}",
|
||||
"account_type": random.choice(ACCOUNT_TYPE_VALUES),
|
||||
"account_status": account_status,
|
||||
"opened_date": opened_date,
|
||||
"closed_date": closed_date,
|
||||
"currency_code": random.choice(self.currency_codes), # FK → Currency
|
||||
# valid_time columns
|
||||
"valid_from": prior_end,
|
||||
"valid_to": None,
|
||||
"is_current": True,
|
||||
**overrides,
|
||||
}
|
||||
|
||||
if not with_history:
|
||||
return [current]
|
||||
|
||||
prior = {
|
||||
**current,
|
||||
"valid_from": prior_start,
|
||||
"valid_to": prior_end,
|
||||
"is_current": False,
|
||||
}
|
||||
return [prior, current]
|
||||
|
||||
def batch(self, n: int, with_history: bool = False, **overrides) -> list[dict]:
|
||||
rows: list[dict] = []
|
||||
for _ in range(n):
|
||||
rows.extend(self.build(with_history=with_history, **overrides))
|
||||
return rows
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TransactionFactory
|
||||
# Entity: Transaction
|
||||
# Existence: dependent
|
||||
# Mutability: append_only
|
||||
# Temporal: transaction_time (recorded_at only — no valid_time)
|
||||
# PII fields: none
|
||||
# FK (required): currency_code → Currency.currency_code
|
||||
# FK (nullable): debit_account_identifier → Account.account_identifier
|
||||
# credit_account_identifier → Account.account_identifier
|
||||
# Constraints encoded:
|
||||
# - Amount Must Be Positive: amount = abs(generated) with floor of 0.01
|
||||
# - Settlement After Initiation: settlement_date_time >= transaction_date_time
|
||||
# - Settled Transaction Has Settlement Time: if status == Settled, settlement is set
|
||||
# ---------------------------------------------------------------------------
|
||||
class TransactionFactory:
|
||||
"""
|
||||
Entity: Transaction
|
||||
Existence: dependent
|
||||
Mutability: append_only
|
||||
Temporal: transaction_time
|
||||
PII fields: none
|
||||
FK (required): currency_code → Currency.currency_code
|
||||
FK (nullable): debit_account_identifier, credit_account_identifier → Account.account_identifier
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
currency_codes: list[str],
|
||||
account_identifiers: list[str],
|
||||
fake: Faker = None,
|
||||
pii_mode: str = "safe",
|
||||
):
|
||||
self.currency_codes = currency_codes
|
||||
self.account_identifiers = account_identifiers
|
||||
self.fake = fake or Faker()
|
||||
self.pii_mode = pii_mode
|
||||
|
||||
def build(self, **overrides) -> dict:
|
||||
seq = _next_seq()
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
|
||||
transaction_date_time = now - timedelta(
|
||||
days=random.randint(0, 365),
|
||||
hours=random.randint(0, 23),
|
||||
minutes=random.randint(0, 59),
|
||||
)
|
||||
|
||||
transaction_status = random.choice(TRANSACTION_STATUS_VALUES)
|
||||
|
||||
# Constraint: Settlement After Initiation
|
||||
# Constraint: Settled Transaction Has Settlement Time
|
||||
if transaction_status in ("Settled", "Reversed"):
|
||||
settlement_date_time = transaction_date_time + timedelta(
|
||||
hours=random.randint(0, 48)
|
||||
)
|
||||
elif transaction_status in ("Pending", "Authorised", "Under Review"):
|
||||
settlement_date_time = None
|
||||
else:
|
||||
settlement_date_time = None
|
||||
|
||||
# Constraint: Amount Must Be Positive
|
||||
amount = max(Decimal("0.01"), round(Decimal(str(random.uniform(1.00, 50000.00))), 2))
|
||||
|
||||
# FK (nullable): randomly assign debit/credit accounts from pool
|
||||
debit_account = random.choice(self.account_identifiers) if random.random() > 0.1 else None
|
||||
credit_account = random.choice(self.account_identifiers) if random.random() > 0.1 else None
|
||||
|
||||
return {
|
||||
"transaction_identifier": str(uuid.uuid4()),
|
||||
"transaction_date_time": transaction_date_time,
|
||||
"settlement_date_time": settlement_date_time,
|
||||
"amount": amount,
|
||||
"transaction_type": random.choice(TRANSACTION_TYPE_VALUES),
|
||||
"transaction_channel": random.choice(TRANSACTION_CHANNEL_VALUES),
|
||||
"transaction_status": transaction_status,
|
||||
"reference": f"REF-{seq:08d}",
|
||||
"currency_code": random.choice(self.currency_codes), # FK → Currency
|
||||
"debit_account_identifier": debit_account, # FK → Account (nullable)
|
||||
"credit_account_identifier": credit_account, # FK → Account (nullable)
|
||||
**overrides,
|
||||
}
|
||||
|
||||
def batch(self, n: int, **overrides) -> list[dict]:
|
||||
return [self.build(**overrides) for _ in range(n)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DatasetBuilder
|
||||
# Generates a referentially consistent dataset across all four entities.
|
||||
# Generation order: Currency → Party → Account → Transaction
|
||||
# ---------------------------------------------------------------------------
|
||||
class DatasetBuilder:
|
||||
"""
|
||||
Generates a referentially consistent Financial Crime dataset.
|
||||
|
||||
Generation order (topological):
|
||||
1. Currency — reference; no FKs
|
||||
2. Party — independent; no FK dependencies in core attributes
|
||||
3. Account — FK → Currency.currency_code
|
||||
4. Transaction — FK → Currency.currency_code, Account.account_identifier
|
||||
|
||||
Parameters
|
||||
----------
|
||||
n_parties : Party root records (current rows)
|
||||
accounts_per_party : Account records created per Party (approximate)
|
||||
txns_per_account : Transaction records created per active Account
|
||||
with_history : Include prior SCD rows for Party and Account
|
||||
pii_mode : "safe" (default) or "realistic"
|
||||
"""
|
||||
|
||||
def __init__(self, fake: Faker = None, pii_mode: str = "safe"):
|
||||
self.fake = fake or Faker()
|
||||
self.pii_mode = pii_mode
|
||||
|
||||
def build(
|
||||
self,
|
||||
n_parties: int = 10,
|
||||
accounts_per_party: int = 2,
|
||||
txns_per_account: int = 5,
|
||||
with_history: bool = False,
|
||||
) -> dict[str, list[dict]]:
|
||||
|
||||
# --- 1. Currency (reference) ---
|
||||
currencies = CurrencyFactory().build_all()
|
||||
currency_codes = [c["currency_code"] for c in currencies]
|
||||
|
||||
# --- 2. Party ---
|
||||
party_factory = PartyFactory(fake=self.fake, pii_mode=self.pii_mode)
|
||||
parties: list[dict] = []
|
||||
for _ in range(n_parties):
|
||||
parties.extend(party_factory.build(with_history=with_history))
|
||||
|
||||
# --- 3. Account (FK → Currency) ---
|
||||
account_factory = AccountFactory(
|
||||
currency_codes=currency_codes,
|
||||
fake=self.fake,
|
||||
pii_mode=self.pii_mode,
|
||||
)
|
||||
accounts: list[dict] = []
|
||||
for _ in range(n_parties * accounts_per_party):
|
||||
accounts.extend(account_factory.build(with_history=with_history))
|
||||
|
||||
# FK pool: only current account rows supply IDs for Transaction FKs
|
||||
active_account_ids = [
|
||||
a["account_identifier"] for a in accounts if a.get("is_current", True)
|
||||
]
|
||||
|
||||
# --- 4. Transaction (FK → Currency, FK → Account) ---
|
||||
txn_factory = TransactionFactory(
|
||||
currency_codes=currency_codes,
|
||||
account_identifiers=active_account_ids,
|
||||
fake=self.fake,
|
||||
pii_mode=self.pii_mode,
|
||||
)
|
||||
transactions = txn_factory.batch(
|
||||
n=len(active_account_ids) * txns_per_account
|
||||
)
|
||||
|
||||
return {
|
||||
"Currency": currencies,
|
||||
"Party": parties,
|
||||
"Account": accounts,
|
||||
"Transaction": transactions,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integrity spec — used by integrity_check.py and test_factories.py
|
||||
# ---------------------------------------------------------------------------
|
||||
FK_SPECS_FINANCIAL_CRIME = None # populated below after import guard
|
||||
NOT_NULL_FINANCIAL_CRIME = {
|
||||
"Party": ["party_identifier", "legal_name"],
|
||||
"Account": ["account_identifier", "account_number", "currency_code"],
|
||||
"Transaction": ["transaction_identifier", "amount", "currency_code"],
|
||||
}
|
||||
ENUM_VALUES_FINANCIAL_CRIME = {
|
||||
"Party": {
|
||||
"party_status": PARTY_STATUS_VALUES,
|
||||
"risk_rating": FINANCIAL_CRIME_RISK_RATING_VALUES,
|
||||
"sanctions_screen_status": SANCTIONS_SCREEN_STATUS_VALUES,
|
||||
},
|
||||
"Account": {
|
||||
"account_status": ACCOUNT_STATUS_VALUES,
|
||||
"account_type": ACCOUNT_TYPE_VALUES,
|
||||
},
|
||||
"Transaction": {
|
||||
"transaction_type": TRANSACTION_TYPE_VALUES,
|
||||
"transaction_channel": TRANSACTION_CHANNEL_VALUES,
|
||||
"transaction_status": TRANSACTION_STATUS_VALUES,
|
||||
},
|
||||
}
|
||||
# unique_pk: all rows — for reference and append_only entities
|
||||
UNIQUE_PK_FINANCIAL_CRIME = {
|
||||
"Currency": "currency_code",
|
||||
"Transaction": "transaction_identifier",
|
||||
}
|
||||
# unique_current_pk: current rows only — for SCD2/bitemporal entities
|
||||
# History rows legitimately repeat the entity identifier across prior/current pairs.
|
||||
UNIQUE_CURRENT_PK_FINANCIAL_CRIME = {
|
||||
"Party": "party_identifier",
|
||||
"Account": "account_identifier",
|
||||
}
|
||||
|
||||
try:
|
||||
from integrity_check import FKSpec
|
||||
FK_SPECS_FINANCIAL_CRIME = [
|
||||
FKSpec("Account", "currency_code", "Currency", "currency_code"),
|
||||
FKSpec("Transaction", "currency_code", "Currency", "currency_code"),
|
||||
FKSpec("Transaction", "debit_account_identifier", "Account", "account_identifier", nullable=True),
|
||||
FKSpec("Transaction", "credit_account_identifier", "Account", "account_identifier", nullable=True),
|
||||
]
|
||||
except ImportError:
|
||||
FK_SPECS_FINANCIAL_CRIME = None # integrity_check.py not on path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point — quick smoke test and integrity check
|
||||
# ---------------------------------------------------------------------------
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
|
||||
dataset = DatasetBuilder(pii_mode="safe").build(
|
||||
n_parties=5, accounts_per_party=2, txns_per_account=3, with_history=True
|
||||
)
|
||||
|
||||
for name, rows in dataset.items():
|
||||
print(f"\n=== {name} ({len(rows)} rows) ===")
|
||||
print(json.dumps(rows[0], indent=2, default=str))
|
||||
|
||||
# Run integrity checks if runtime is available
|
||||
try:
|
||||
from integrity_check import check_integrity, print_report
|
||||
print("\n--- Integrity check ---")
|
||||
errors = check_integrity(
|
||||
dataset,
|
||||
fk_specs=FK_SPECS_FINANCIAL_CRIME,
|
||||
not_null=NOT_NULL_FINANCIAL_CRIME,
|
||||
enum_values=ENUM_VALUES_FINANCIAL_CRIME,
|
||||
unique_pk=UNIQUE_PK_FINANCIAL_CRIME,
|
||||
unique_current_pk=UNIQUE_CURRENT_PK_FINANCIAL_CRIME,
|
||||
)
|
||||
print_report(errors)
|
||||
except ImportError:
|
||||
print(
|
||||
"\n[integrity_check not found] "
|
||||
"Copy integrity_check.py from agents/agent-artifact/skills/faker/runtime/ "
|
||||
"alongside this file to enable integrity validation."
|
||||
)
|
||||
Reference in New Issue
Block a user