diff --git a/backend/src/walacor_service.py b/backend/src/walacor_service.py index 0af10e8d..81785acc 100644 --- a/backend/src/walacor_service.py +++ b/backend/src/walacor_service.py @@ -273,12 +273,15 @@ def store_document_hash(self, loan_id: str, document_type: str, document_hash: s if len(document_hash) != 64: raise ValueError("document_hash must be a 64-character SHA-256 hash") - # HYBRID APPROACH: Only essential blockchain data goes to Walacor + # HYBRID APPROACH: Match Walacor schema fields exactly blockchain_data = { + "loan_id": loan_id, + "document_type": document_type, "document_hash": document_hash, - "seal_timestamp": datetime.now().isoformat(), - "etid": self.LOAN_DOCUMENTS_ETID, - "integrity_seal": f"SEAL_{document_hash[:16]}_{int(datetime.now().timestamp())}" + "file_size": file_size, + "upload_timestamp": int(datetime.now().timestamp()), # DATETIME_EPOCH format + "uploaded_by": uploaded_by, + "file_path": file_path } # Store in Walacor or local blockchain (only essential data) @@ -500,16 +503,20 @@ def seal_loan_document(self, loan_id: str, loan_data: Dict[str, Any], envelope_json = json.dumps(envelope, sort_keys=True, separators=(',', ':')) document_hash = hashlib.sha256(envelope_json.encode('utf-8')).hexdigest() - # Create blockchain data for Walacor + # Create blockchain data matching loan_documents schema + # Extract uploaded_by from borrower data or use default + uploaded_by = loan_data.get("created_by", "system") + if borrower_data and "full_name" in borrower_data: + uploaded_by = borrower_data["full_name"] + blockchain_data = { - "document_hash": document_hash, "loan_id": loan_id, - "seal_timestamp": envelope["sealed_timestamp"], - "etid": self.LOAN_DOCUMENTS_ETID, - "integrity_seal": f"LOAN_SEAL_{document_hash[:16]}_{int(datetime.now().timestamp())}", - "envelope_size": len(envelope_json), - "borrower_data_included": True, - "file_count": len(files) + "document_type": "loan_packet", + "document_hash": document_hash, + "file_size": len(envelope_json), + "upload_timestamp": int(datetime.now().timestamp()), # DATETIME_EPOCH format + "uploaded_by": uploaded_by, + "file_path": f"/loans/{loan_id}/packet.json" } # Store hash in Walacor blockchain or local simulation @@ -627,12 +634,14 @@ def log_audit_event(self, document_id: str, event_type: str, user: str, if not all([document_id, event_type, user]): raise ValueError("document_id, event_type, and user are required") - # HYBRID APPROACH: Only essential audit data goes to blockchain + # Match audit_logs schema fields exactly blockchain_audit_data = { "document_id": document_id, "event_type": event_type, - "audit_timestamp": datetime.now().isoformat(), - "audit_hash": f"AUDIT_{document_id}_{event_type}_{int(datetime.now().timestamp())}" + "user": user, + "timestamp": int(datetime.now().timestamp()), # DATETIME_EPOCH format + "ip_address": ip_address if ip_address else "", + "details": details if details else "" } # Store in Walacor or local blockchain (only essential audit data) @@ -774,13 +783,13 @@ def create_provenance_link(self, parent_doc_id: str, child_doc_id: str, if parent_doc_id == child_doc_id: raise ValueError("parent_doc_id and child_doc_id must be different") - # Prepare provenance data + # Match document_provenance schema fields exactly provenance_data = { "parent_doc_id": parent_doc_id, "child_doc_id": child_doc_id, "relationship_type": relationship_type, - "timestamp": datetime.now().isoformat(), - "description": description + "timestamp": int(datetime.now().timestamp()), # DATETIME_EPOCH format + "description": description if description else "" } # Store in Walacor @@ -821,23 +830,23 @@ def create_attestation(self, document_id: str, attestor_name: str, RuntimeError: If Walacor operation fails """ try: - # Validate inputs - if not all([document_id, attestor_name, attestation_type, status, signature]): - raise ValueError("document_id, attestor_name, attestation_type, status, and signature are required") - + # Validate required inputs (signature and notes are optional per schema) + if not all([document_id, attestor_name, attestation_type, status]): + raise ValueError("document_id, attestor_name, attestation_type, and status are required") + valid_statuses = ["pending", "approved", "rejected"] if status not in valid_statuses: raise ValueError(f"status must be one of: {', '.join(valid_statuses)}") - - # Prepare attestation data + + # Match attestations schema fields exactly attestation_data = { "document_id": document_id, "attestor_name": attestor_name, "attestation_type": attestation_type, "status": status, - "timestamp": datetime.now().isoformat(), - "signature": signature, - "notes": notes + "timestamp": int(datetime.now().timestamp()), # DATETIME_EPOCH format + "signature": signature if signature else "", + "notes": notes if notes else "" } # Store in Walacor diff --git a/scripts/check_walacor_schemas.py b/scripts/check_walacor_schemas.py new file mode 100755 index 00000000..a0121b8e --- /dev/null +++ b/scripts/check_walacor_schemas.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +""" +Check Walacor Schemas Script + +This script lists all existing schemas on your Walacor tenant and checks +if the required schemas for IntegrityX already exist. +""" + +import os +import sys +from pathlib import Path +from dotenv import load_dotenv + +# Add backend/src directory to Python path +script_dir = Path(__file__).parent +project_root = script_dir.parent +backend_src_dir = project_root / "backend" / "src" +sys.path.insert(0, str(backend_src_dir)) + +try: + from walacor_sdk import WalacorService +except ImportError as e: + print(f"āŒ Import Error: {e}") + print("Please ensure walacor-python-sdk is installed") + sys.exit(1) + + +def main(): + print("=" * 70) + print("WALACOR SCHEMA CHECKER") + print("=" * 70) + print() + + # Load environment + env_path = project_root / "backend" / ".env" + if not env_path.exists(): + print(f"āŒ .env file not found at: {env_path}") + sys.exit(1) + + load_dotenv(env_path) + + host = os.getenv('WALACOR_HOST') + username = os.getenv('WALACOR_USERNAME') + password = os.getenv('WALACOR_PASSWORD') + + if not all([host, username, password]): + print("āŒ Missing Walacor credentials in .env") + sys.exit(1) + + # Connect to Walacor + print(f"šŸ“” Connecting to Walacor at: {host}") + try: + wal = WalacorService( + server=f"http://{host}/api", + username=username, + password=password + ) + + schemas = wal.schema.get_list_with_latest_version() + print(f"āœ… Connected successfully!") + print(f" Found {len(schemas)} schemas on tenant") + print() + + except Exception as e: + print(f"āŒ Failed to connect: {e}") + sys.exit(1) + + # Required schemas for IntegrityX + required_schemas = { + 100001: "loan_documents", + 100002: "document_provenance", + 100003: "attestations", + 100004: "audit_logs" + } + + # Check for meta-schema (needed to create schemas) + print("šŸ” Checking for Schema Creation Meta-Schema:") + meta_schema_found = False + for schema in schemas: + schema_data = schema.model_dump() + if schema_data.get('ETId') == 50: + print(f" āœ… Found ETId 50 (Schema Creation) - SV {schema_data.get('SV', 'Unknown')}") + meta_schema_found = True + break + + if not meta_schema_found: + print(f" āŒ ETId 50 (Schema Creation Meta-Schema) NOT FOUND") + print(f" šŸ’” This is why schema creation is failing") + print() + + # Check for required IntegrityX schemas + print("šŸ” Checking for Required IntegrityX Schemas:") + found_schemas = {} + + for schema in schemas: + schema_data = schema.model_dump() + etid = schema_data.get('ETId') + + if etid in required_schemas: + table_name = schema_data.get('TableName', 'Unknown') + sv = schema_data.get('SV', 'Unknown') + found_schemas[etid] = (table_name, sv) + print(f" āœ… ETId {etid} ({required_schemas[etid]}) - Found as '{table_name}' (SV {sv})") + + # Check for missing schemas + missing_schemas = set(required_schemas.keys()) - set(found_schemas.keys()) + + if missing_schemas: + print() + print("āš ļø Missing Required Schemas:") + for etid in sorted(missing_schemas): + print(f" āŒ ETId {etid} ({required_schemas[etid]}) - NOT FOUND") + + print() + print("=" * 70) + print("ALL SCHEMAS ON TENANT:") + print("=" * 70) + + # List all schemas + for i, schema in enumerate(schemas, 1): + schema_data = schema.model_dump() + etid = schema_data.get('ETId', 'Unknown') + sv = schema_data.get('SV', 'Unknown') + table_name = schema_data.get('TableName', 'Unknown') + family = schema_data.get('Family', 'Unknown') + + print(f"{i:2}. ETId={etid:<6} SV={sv:<3} Table={table_name:<30} Family={family}") + + print() + print("=" * 70) + print("SUMMARY:") + print("=" * 70) + + if not meta_schema_found: + print("āŒ Cannot create schemas programmatically (ETId 50 missing)") + print("šŸ’” RECOMMENDATION: Create schemas manually via Walacor Dashboard") + elif missing_schemas: + print(f"āš ļø {len(missing_schemas)} required schemas are missing") + print("šŸ’” RECOMMENDATION: Run initialize_schemas.py to create them") + else: + print("āœ… All required schemas exist!") + print("šŸŽ‰ Your IntegrityX system is ready to use") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\n\nāš ļø Interrupted by user") + sys.exit(1) + except Exception as e: + print(f"\n\nāŒ Unexpected error: {e}") + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/scripts/initialize_schemas.py b/scripts/initialize_schemas.py index 143c0ca2..3eeb7669 100755 --- a/scripts/initialize_schemas.py +++ b/scripts/initialize_schemas.py @@ -45,7 +45,7 @@ from schemas import LoanSchemas except ImportError as e: print(f"āŒ Import Error: {e}") - print("Please ensure walacor-python-sdk is installed and src/schemas.py exists") + print("Please ensure walacor-python-sdk is installed and backend/src/schemas.py exists") sys.exit(1) diff --git a/scripts/inspect_walacor_schemas.py b/scripts/inspect_walacor_schemas.py new file mode 100755 index 00000000..6cd36989 --- /dev/null +++ b/scripts/inspect_walacor_schemas.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python +""" +Detailed Walacor Schema Inspector + +This script connects to Walacor and displays the complete field definitions +for all IntegrityX schemas, allowing comparison with expected structure. +""" + +import os +import sys +from pathlib import Path +from dotenv import load_dotenv + +# Add backend/src directory to Python path +script_dir = Path(__file__).parent +project_root = script_dir.parent +backend_src_dir = project_root / "backend" / "src" +sys.path.insert(0, str(backend_src_dir)) + +try: + from walacor_sdk import WalacorService +except ImportError as e: + print(f"āŒ Import Error: {e}") + print("Please ensure walacor-python-sdk is installed") + sys.exit(1) + + +def main(): + print("=" * 80) + print("WALACOR SCHEMA FIELD INSPECTOR") + print("=" * 80) + print() + + # Load environment + env_path = project_root / "backend" / ".env" + if not env_path.exists(): + print(f"āŒ .env file not found at: {env_path}") + sys.exit(1) + + load_dotenv(env_path) + + host = os.getenv('WALACOR_HOST') + username = os.getenv('WALACOR_USERNAME') + password = os.getenv('WALACOR_PASSWORD') + + if not all([host, username, password]): + print("āŒ Missing Walacor credentials") + sys.exit(1) + + # Connect to Walacor + print(f"šŸ“” Connecting to Walacor at: {host}") + try: + wal = WalacorService( + server=f"http://{host}/api", + username=username, + password=password + ) + print(f"āœ… Connected successfully!") + print() + except Exception as e: + print(f"āŒ Failed to connect: {e}") + sys.exit(1) + + # Target schemas + target_schemas = { + 100001: "loan_documents", + 100002: "document_provenance", + 100003: "attestations", + 100004: "audit_logs" + } + + # Get all schemas first + try: + all_schemas = wal.schema.get_list_with_latest_version() + except Exception as e: + print(f"āŒ Failed to get schema list: {e}") + sys.exit(1) + + # Inspect each schema + for etid, expected_name in target_schemas.items(): + print("=" * 80) + print(f"SCHEMA: {expected_name} (ETId {etid})") + print("=" * 80) + + try: + # Find schema in list + schema_detail = None + for schema in all_schemas: + if schema.ETId == etid: + schema_detail = schema + break + + if not schema_detail: + print(f"āŒ Schema not found in Walacor!") + print() + continue + + schema_data = schema_detail.model_dump() + + # Basic info + print(f"Table Name: {schema_data.get('TableName', 'N/A')}") + print(f"Schema Version: {schema_data.get('SV', 'N/A')}") + print(f"Family: {schema_data.get('Family', 'N/A')}") + print() + + # Fields + print("FIELDS:") + print("-" * 80) + fields = schema_data.get('Fields', []) + + if not fields: + print(" āš ļø No fields defined!") + else: + print(f"{'Field Name':<25} {'Type':<20} {'Required':<10} {'Max Length':<12}") + print("-" * 80) + for field in fields: + field_name = field.get('FieldName', 'Unknown') + data_type = field.get('DataType', 'Unknown') + required = "Yes" if field.get('Required', False) else "No" + max_length = field.get('MaxLength', 'N/A') + + print(f"{field_name:<25} {data_type:<20} {required:<10} {str(max_length):<12}") + + print() + + # Indexes + print("INDEXES:") + print("-" * 80) + indexes = schema_data.get('Indexes', []) + + if not indexes: + print(" āš ļø No indexes defined!") + else: + for idx in indexes: + idx_name = idx.get('IndexValue', 'Unknown') + idx_fields = idx.get('Fields', []) + print(f" ā€¢ {idx_name}: {', '.join(idx_fields)}") + + print() + + except Exception as e: + print(f"āŒ Failed to fetch schema details: {e}") + print() + + # Now show what we EXPECT + print() + print("=" * 80) + print("EXPECTED SCHEMA DEFINITIONS (from backend/src/schemas.py)") + print("=" * 80) + print() + + print("šŸ“‹ EXPECTED: loan_documents (100001)") + print("-" * 80) + print("Fields:") + print(" ā€¢ loan_id (TEXT, Required, max 2048)") + print(" ā€¢ document_type (TEXT, Required)") + print(" ā€¢ document_hash (TEXT, Required)") + print(" ā€¢ file_size (INTEGER, Required)") + print(" ā€¢ upload_timestamp (DATETIME_EPOCH, Required)") + print(" ā€¢ uploaded_by (TEXT, Required)") + print(" ā€¢ file_path (TEXT, Required)") + print("Indexes:") + print(" ā€¢ idx_loan_id: loan_id") + print(" ā€¢ idx_document_hash: document_hash") + print() + + print("šŸ“‹ EXPECTED: document_provenance (100002)") + print("-" * 80) + print("Fields:") + print(" ā€¢ parent_doc_id (TEXT, Required)") + print(" ā€¢ child_doc_id (TEXT, Required)") + print(" ā€¢ relationship_type (TEXT, Required)") + print(" ā€¢ timestamp (DATETIME_EPOCH, Required)") + print(" ā€¢ description (TEXT, Optional)") + print("Indexes:") + print(" ā€¢ idx_parent_doc_id: parent_doc_id") + print(" ā€¢ idx_child_doc_id: child_doc_id") + print() + + print("šŸ“‹ EXPECTED: attestations (100003)") + print("-" * 80) + print("Fields:") + print(" ā€¢ document_id (TEXT, Required)") + print(" ā€¢ attestor_name (TEXT, Required)") + print(" ā€¢ attestation_type (TEXT, Required)") + print(" ā€¢ status (TEXT, Required)") + print(" ā€¢ timestamp (DATETIME_EPOCH, Required)") + print(" ā€¢ signature (TEXT, Optional)") + print(" ā€¢ notes (TEXT, Optional)") + print("Indexes:") + print(" ā€¢ idx_document_id: document_id") + print() + + print("šŸ“‹ EXPECTED: audit_logs (100004)") + print("-" * 80) + print("Fields:") + print(" ā€¢ document_id (TEXT, Required)") + print(" ā€¢ event_type (TEXT, Required)") + print(" ā€¢ user (TEXT, Required)") + print(" ā€¢ timestamp (DATETIME_EPOCH, Required)") + print(" ā€¢ ip_address (TEXT, Optional)") + print(" ā€¢ details (TEXT, Optional)") + print("Indexes:") + print(" ā€¢ idx_document_id: document_id") + print(" ā€¢ idx_timestamp: timestamp") + print() + + print("=" * 80) + print("NEXT STEPS:") + print("=" * 80) + print("1. Compare the ACTUAL fields (above) with EXPECTED fields") + print("2. If they don't match, update schemas in Walacor Dashboard") + print("3. Go to http://{} and navigate to Schemas".format(host)) + print("4. Update each schema's SV 2 to match the EXPECTED definitions") + print() + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\n\nāš ļø Interrupted by user") + sys.exit(1) + except Exception as e: + print(f"\n\nāŒ Unexpected error: {e}") + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/scripts/test_walacor_insert.py b/scripts/test_walacor_insert.py new file mode 100755 index 00000000..5e214e2c --- /dev/null +++ b/scripts/test_walacor_insert.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python +""" +Test Walacor Data Insertion + +This script directly tests inserting data into Walacor schemas to diagnose issues. +""" + +import os +import sys +import json +from pathlib import Path +from datetime import datetime +from dotenv import load_dotenv + +# Add backend/src directory to Python path +script_dir = Path(__file__).parent +project_root = script_dir.parent +backend_src_dir = project_root / "backend" / "src" +sys.path.insert(0, str(backend_src_dir)) + +try: + from walacor_sdk import WalacorService +except ImportError as e: + print(f"āŒ Import Error: {e}") + sys.exit(1) + + +def main(): + print("=" * 80) + print("WALACOR DATA INSERTION TEST") + print("=" * 80) + print() + + # Load environment + env_path = project_root / "backend" / ".env" + if not env_path.exists(): + print(f"āŒ .env file not found at: {env_path}") + sys.exit(1) + + load_dotenv(env_path) + + host = os.getenv('WALACOR_HOST') + username = os.getenv('WALACOR_USERNAME') + password = os.getenv('WALACOR_PASSWORD') + + if not all([host, username, password]): + print("āŒ Missing Walacor credentials") + sys.exit(1) + + # Connect to Walacor + print(f"šŸ“” Connecting to Walacor at: {host}") + try: + wal = WalacorService( + server=f"http://{host}/api", + username=username, + password=password + ) + print(f"āœ… Connected successfully!") + print() + except Exception as e: + print(f"āŒ Failed to connect: {e}") + sys.exit(1) + + # Test 1: Insert into loan_documents (ETId 100001) + print("=" * 80) + print("TEST 1: Insert into loan_documents (ETId 100001)") + print("=" * 80) + + test_data = { + "loan_id": "TEST-LOAN-001", + "document_type": "test_document", + "document_hash": "a" * 64, # 64-char hash + "file_size": 1024000, + "upload_timestamp": int(datetime.now().timestamp()), + "uploaded_by": "test_user", + "file_path": "/test/path/document.pdf" + } + + print("Data to insert:") + print(json.dumps(test_data, indent=2)) + print() + + try: + result = wal.data_requests.insert_single_record( + jsonRecord=json.dumps(test_data), + ETId=100001 + ) + print("āœ… INSERT SUCCESSFUL!") + print("Response:") + print(json.dumps(result, indent=2, default=str)) + print() + except Exception as e: + print(f"āŒ INSERT FAILED!") + print(f"Error type: {type(e).__name__}") + print(f"Error message: {str(e)}") + print() + + # Try to get more details from the exception + print("Exception attributes:") + for attr in dir(e): + if not attr.startswith('_'): + try: + val = getattr(e, attr) + if not callable(val): + print(f" {attr}: {val}") + except: + pass + print() + + # Try to get response details + if hasattr(e, 'response'): + print("Response details:") + print(f" Status: {e.response.status_code if hasattr(e.response, 'status_code') else 'N/A'}") + try: + print(f" Body: {e.response.text if hasattr(e.response, 'text') else 'N/A'}") + except: + print(f" Body: [Unable to read]") + try: + print(f" Headers: {dict(e.response.headers) if hasattr(e.response, 'headers') else 'N/A'}") + except: + print(f" Headers: [Unable to read]") + print() + + # Test 2: Insert into audit_logs (ETId 100004) + print("=" * 80) + print("TEST 2: Insert into audit_logs (ETId 100004)") + print("=" * 80) + + audit_data = { + "document_id": "DOC-001", + "event_type": "test_event", + "user": "test_user@example.com", + "timestamp": int(datetime.now().timestamp()), + "ip_address": "127.0.0.1", + "details": "Test audit log entry" + } + + print("Data to insert:") + print(json.dumps(audit_data, indent=2)) + print() + + try: + result = wal.data_requests.insert_single_record( + jsonRecord=json.dumps(audit_data), + ETId=100004 + ) + print("āœ… INSERT SUCCESSFUL!") + print("Response:") + print(json.dumps(result, indent=2, default=str)) + print() + except Exception as e: + print(f"āŒ INSERT FAILED!") + print(f"Error type: {type(e).__name__}") + print(f"Error message: {str(e)}") + print() + + # Try to get more details + if hasattr(e, 'response'): + print("Response details:") + print(f" Status: {e.response.status_code if hasattr(e.response, 'status_code') else 'N/A'}") + print(f" Body: {e.response.text if hasattr(e.response, 'text') else 'N/A'}") + print() + + # Test 3: Try to query existing data (read test) + print("=" * 80) + print("TEST 3: Try to query loan_documents schema") + print("=" * 80) + + try: + # Try to get existing records + query_result = wal.data_requests.get_records( + ETId=100001, + limit=5 + ) + print("āœ… QUERY SUCCESSFUL!") + print(f"Found {len(query_result) if isinstance(query_result, list) else 'unknown'} records") + if query_result: + print("Sample record:") + print(json.dumps(query_result[0] if isinstance(query_result, list) else query_result, indent=2, default=str)) + print() + except Exception as e: + print(f"āŒ QUERY FAILED!") + print(f"Error: {str(e)}") + print() + + print("=" * 80) + print("DIAGNOSIS") + print("=" * 80) + print() + print("šŸ” 500 Internal Server Error means:") + print(" āœ… Connection to Walacor works") + print(" āœ… Authentication is valid") + print(" āœ… Request format is correct") + print(" āŒ Walacor server crashed when processing the insert") + print() + print("šŸ’” Possible causes:") + print(" 1. Schema fields not properly initialized in Walacor backend database") + print(" 2. Missing database tables/columns for the schema") + print(" 3. Schema version mismatch (client expects SV 2, server has different version)") + print(" 4. Walacor server bug or misconfiguration") + print(" 5. Database constraints (foreign keys, triggers) failing") + print() + print("šŸ“‹ Recommended actions:") + print(" 1. Check Walacor server logs for detailed error") + print(" 2. Contact Walacor support with this error") + print(" 3. Try inserting via Walacor Dashboard UI to see if it works") + print(" 4. Verify schema is fully initialized (not just created)") + print() + + print("=" * 80) + print("TEST COMPLETE") + print("=" * 80) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\n\nāš ļø Interrupted by user") + sys.exit(1) + except Exception as e: + print(f"\n\nāŒ Unexpected error: {e}") + import traceback + traceback.print_exc() + sys.exit(1)