Add coinbase parsing

galt-tr · galt-tr · commit a9dfe7709408 · 2025-08-04T11:24:31.000-04:00
diff --git a/cmd/main.go b/cmd/main.go
@@ -41,10 +41,26 @@ func main() {
 	}
 
 	// Initialize SQLite DB with GORM
-	db, err := gorm.Open(sqlite.Open(databasePath), &gorm.Config{})
+	// Enable WAL mode and optimize for concurrent reads
+	dsn := fmt.Sprintf("%s?_journal_mode=WAL&_busy_timeout=5000&_synchronous=NORMAL&cache=shared", databasePath)
+	db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
 	if err != nil {
 		log.Fatalf("failed to connect to database [%s]: %v", databasePath, err)
 	}
+	
+	// Configure connection pool for SQLite
+	sqlDB, err := db.DB()
+	if err != nil {
+		log.Fatalf("failed to get underlying SQL database: %v", err)
+	}
+	
+	// Set max open connections to 1 for SQLite (prevents database locked errors)
+	// WAL mode allows multiple readers even with single writer
+	sqlDB.SetMaxOpenConns(1)
+	// Set max idle connections
+	sqlDB.SetMaxIdleConns(1)
+	// Set max lifetime of a connection
+	sqlDB.SetConnMaxLifetime(time.Hour)
 	// Auto-migrate all schemas
 	err = model.MigrateAll(db)
 	if err != nil {
diff --git a/pkg/model/best_block_request.go b/pkg/model/best_block_request.go
@@ -4,10 +4,10 @@ import (
 	"time"
 )
 
-// BestBlockRequest represents a request for the best block information from a peer
+// BestBlockRequest represents a best block request message
 type BestBlockRequest struct {
 	ID         uint      `gorm:"primaryKey"`
 	Network    string    `gorm:"index;not null"`
 	PeerID     string    `gorm:"index;not null"`
-	ReceivedAt time.Time
+	ReceivedAt time.Time `gorm:"index"`
 }
diff --git a/pkg/model/block.go b/pkg/model/block.go
@@ -13,7 +13,7 @@ type Block struct {
 	DataHubURL string    `gorm:"type:text" json:"DataHubURL"`
 	PeerID     string    `gorm:"index;not null" json:"PeerID"`
 	Header     string    `gorm:"type:text" json:"Header"`
-	ReceivedAt time.Time `json:"ReceivedAt"`
+	ReceivedAt time.Time `gorm:"index" json:"ReceivedAt"`
 }
 
 type BlockHeader struct {
diff --git a/pkg/model/handshake.go b/pkg/model/handshake.go
@@ -15,5 +15,5 @@ type Handshake struct {
 	DataHubURL string    `gorm:"type:text"`
 	UserAgent  string    `gorm:"type:text"`
 	Services   uint64    
-	ReceivedAt time.Time
+	ReceivedAt time.Time `gorm:"index"`
 }
diff --git a/pkg/model/message.go b/pkg/model/message.go
@@ -9,5 +9,5 @@ type Message struct {
 	Topic      string    `gorm:"index;not null"`
 	Data       string    `gorm:"type:text;not null"`
 	Peer       string    `gorm:"type:text"`
-	ReceivedAt time.Time
+	ReceivedAt time.Time `gorm:"index"`
 }
diff --git a/pkg/model/mining_on.go b/pkg/model/mining_on.go
@@ -16,5 +16,5 @@ type MiningOn struct {
 	Miner        string    `gorm:"index"`
 	SizeInBytes  uint64    
 	TxCount      uint64    
-	ReceivedAt   time.Time
+	ReceivedAt   time.Time `gorm:"index"`
 }
diff --git a/pkg/model/models.go b/pkg/model/models.go
@@ -8,6 +8,7 @@ func MigrateAll(db *gorm.DB) error {
 		&Message{},      // Keep existing generic message table
 		&Block{},
 		&BlockHeader{},  // New table for parsed block headers
+		&BestBlockRequest{},
 		&MiningOn{},
 		&Subtree{},
 		&Handshake{},
diff --git a/pkg/model/rejected_tx.go b/pkg/model/rejected_tx.go
@@ -11,5 +11,5 @@ type RejectedTx struct {
 	TxID       string    `gorm:"index;not null"`
 	Reason     string    `gorm:"type:text"`
 	PeerID     string    `gorm:"index;not null"`
-	ReceivedAt time.Time
+	ReceivedAt time.Time `gorm:"index"`
 }
diff --git a/pkg/model/subtree.go b/pkg/model/subtree.go
@@ -11,5 +11,5 @@ type Subtree struct {
 	Hash       string    `gorm:"index;not null"`
 	DataHubURL string    `gorm:"type:text"`
 	PeerID     string    `gorm:"index;not null"`
-	ReceivedAt time.Time
+	ReceivedAt time.Time `gorm:"index"`
 }
diff --git a/pkg/parser/coinbase_parser.go b/pkg/parser/coinbase_parser.go
@@ -2,7 +2,6 @@ package parser
 
 import (
 	"encoding/hex"
-	"fmt"
 	"strings"
 	"unicode"
 )
@@ -37,10 +36,10 @@ func ParseCoinbaseScript(scriptHex string) string {
 
 	// Clean up the text
 	result := strings.TrimSpace(text.String())
-	
+
 	// Remove excessive spaces
 	result = strings.Join(strings.Fields(result), " ")
-	
+
 	return result
 }
 
@@ -57,7 +56,7 @@ func ExtractMinerInfo(coinbaseText string) string {
 	}
 
 	lowerText := strings.ToLower(coinbaseText)
-	
+
 	for poolName, patterns := range poolPatterns {
 		for _, pattern := range patterns {
 			if strings.Contains(lowerText, strings.ToLower(pattern)) {
@@ -87,18 +86,18 @@ func CalculateBlockReward(height uint32) uint64 {
 	// BSV block reward schedule
 	// Initial reward: 50 BSV = 5,000,000,000 satoshis
 	// Halving every 210,000 blocks
-	
+
 	const initialReward = uint64(5000000000) // 50 BSV in satoshis
 	const halvingInterval = uint32(210000)
-	
+
 	halvings := height / halvingInterval
-	
+
 	// Block reward halves every 210,000 blocks
 	reward := initialReward
 	for i := uint32(0); i < halvings; i++ {
 		reward = reward / 2
 	}
-	
+
 	return reward
 }
 
@@ -157,10 +156,10 @@ func CleanCoinbaseText(text string) string {
 		}
 		return -1
 	}, text)
-	
+
 	// Trim and remove excessive whitespace
 	cleaned = strings.TrimSpace(cleaned)
 	cleaned = strings.Join(strings.Fields(cleaned), " ")
-	
+
 	return cleaned
-}
+}
diff --git a/pkg/service/coinbase_service.go b/pkg/service/coinbase_service.go
@@ -36,7 +36,12 @@ func (s *CoinbaseService) ProcessBlockCoinbase(block *model.Block) error {
 	// Check if we already have coinbase data for this block
 	var blockHeader model.BlockHeader
 	if err := s.db.Where("hash = ? AND network = ?", block.Hash, block.Network).First(&blockHeader).Error; err != nil {
-		s.log.Errorf("Failed to find block header for %s: %v", block.Hash, err)
+		if err == gorm.ErrRecordNotFound {
+			// Block header doesn't exist yet - this is expected for blocks received before the header parsing feature was added
+			s.log.Debugf("Block header not found for %s (network: %s), skipping coinbase processing", block.Hash, block.Network)
+			return nil
+		}
+		s.log.Errorf("Failed to query block header for %s: %v", block.Hash, err)
 		return err
 	}
 
@@ -127,23 +132,29 @@ func (s *CoinbaseService) extractCoinbaseData(tx *datahub.Transaction, height ui
 
 // ProcessPendingBlocks processes blocks that don't have coinbase data yet
 func (s *CoinbaseService) ProcessPendingBlocks() {
-	// Find blocks without coinbase data
+	// Find blocks that have block headers but no coinbase data
 	var blocks []model.Block
-	query := s.db.Joins("LEFT JOIN block_headers ON blocks.hash = block_headers.hash AND blocks.network = block_headers.network").
+	query := s.db.Joins("INNER JOIN block_headers ON blocks.hash = block_headers.hash AND blocks.network = block_headers.network").
 		Where("block_headers.coinbase_tx_id IS NULL OR block_headers.coinbase_tx_id = ''").
 		Where("blocks.data_hub_url != ''").
+		Order("blocks.height DESC"). // Process newest blocks first
 		Limit(10) // Process 10 at a time
 
 	if err := query.Find(&blocks).Error; err != nil {
 		s.log.Errorf("Failed to find pending blocks: %v", err)
 		return
 	}
 
-	s.log.Infof("Found %d blocks to process for coinbase data", len(blocks))
+	if len(blocks) > 0 {
+		s.log.Infof("Found %d blocks to process for coinbase data", len(blocks))
+	}
 
 	for _, block := range blocks {
 		if err := s.ProcessBlockCoinbase(&block); err != nil {
-			s.log.Errorf("Failed to process coinbase for block %s: %v", block.Hash, err)
+			// Only log actual errors, not expected "not found" cases
+			if err != gorm.ErrRecordNotFound {
+				s.log.Errorf("Failed to process coinbase for block %s: %v", block.Hash, err)
+			}
 			// Continue with next block
 		}
 		
diff --git a/scripts/README.md b/scripts/README.md
@@ -0,0 +1,47 @@
+# Database Optimization Scripts
+
+## Production Database Optimization
+
+If you're experiencing "database is locked" errors in production with high message volumes, use these optimization scripts.
+
+### Quick Start
+
+```bash
+# Run the optimization script on your production database
+./optimize_production_db.sh /path/to/your/teranode.db
+```
+
+### What the Optimization Does
+
+1. **Enables WAL Mode**: Write-Ahead Logging allows multiple concurrent readers even while writes are happening
+2. **Optimizes SQLite Settings**: Adjusts cache size, synchronous mode, and memory-mapped I/O
+3. **Creates Missing Indexes**: Ensures all commonly queried fields have indexes
+4. **Runs VACUUM and ANALYZE**: Optimizes database structure and updates statistics
+
+### Manual Optimization
+
+If you prefer to run the SQL commands manually:
+
+```bash
+sqlite3 /path/to/your/teranode.db < optimize_db.sql
+```
+
+### Application Configuration
+
+The application (main.go) has been updated to automatically use optimal SQLite settings:
+- WAL mode enabled by default
+- Connection pool limited to 1 connection (required for SQLite)
+- Busy timeout set to 5 seconds
+- Synchronous mode set to NORMAL
+
+### Monitoring Performance
+
+After optimization, you can check the database settings:
+
+```bash
+sqlite3 /path/to/your/teranode.db "PRAGMA journal_mode; PRAGMA synchronous; PRAGMA cache_size;"
+```
+
+### Rollback
+
+The optimization script automatically creates a backup before making changes. If you need to rollback, simply restore the backup file.
diff --git a/scripts/optimize_db.sql b/scripts/optimize_db.sql
@@ -0,0 +1,98 @@
+-- SQLite Database Optimization Script for Teranode P2P
+-- This script optimizes the database for high-volume production use
+-- Backup your database before running this script!
+
+-- Enable Write-Ahead Logging (WAL) mode
+-- This allows multiple readers concurrent with a single writer
+PRAGMA journal_mode=WAL;
+
+-- Set synchronous mode to NORMAL (good balance of safety and performance)
+PRAGMA synchronous=NORMAL;
+
+-- Increase cache size (negative value means KB, -20000 = 20MB)
+PRAGMA cache_size=-20000;
+
+-- Enable memory-mapped I/O (helps with read performance)
+PRAGMA mmap_size=268435456; -- 256MB
+
+-- Optimize database
+VACUUM;
+ANALYZE;
+
+-- Create indexes if they don't exist
+-- Note: GORM auto-migration should have created most of these,
+-- but this ensures they exist for production optimization
+
+-- Messages table indexes
+CREATE INDEX IF NOT EXISTS idx_messages_topic ON messages(topic);
+CREATE INDEX IF NOT EXISTS idx_messages_received_at ON messages(received_at);
+
+-- Blocks table indexes
+CREATE INDEX IF NOT EXISTS idx_blocks_network ON blocks(network);
+CREATE INDEX IF NOT EXISTS idx_blocks_hash ON blocks(hash);
+CREATE INDEX IF NOT EXISTS idx_blocks_height ON blocks(height);
+CREATE INDEX IF NOT EXISTS idx_blocks_peer_id ON blocks(peer_id);
+CREATE INDEX IF NOT EXISTS idx_blocks_received_at ON blocks(received_at);
+
+-- Block headers table indexes
+CREATE INDEX IF NOT EXISTS idx_block_headers_network ON block_headers(network);
+CREATE INDEX IF NOT EXISTS idx_block_headers_hash ON block_headers(hash);
+CREATE INDEX IF NOT EXISTS idx_block_headers_height ON block_headers(height);
+CREATE INDEX IF NOT EXISTS idx_block_headers_timestamp ON block_headers(timestamp);
+CREATE INDEX IF NOT EXISTS idx_block_headers_previous_hash ON block_headers(previous_hash);
+CREATE INDEX IF NOT EXISTS idx_block_headers_received_at ON block_headers(received_at);
+CREATE INDEX IF NOT EXISTS idx_block_headers_miner_address ON block_headers(miner_address);
+CREATE INDEX IF NOT EXISTS idx_block_headers_coinbase_tx_id ON block_headers(coinbase_tx_id);
+
+-- Best block requests table indexes
+CREATE INDEX IF NOT EXISTS idx_best_block_requests_network ON best_block_requests(network);
+CREATE INDEX IF NOT EXISTS idx_best_block_requests_peer_id ON best_block_requests(peer_id);
+CREATE INDEX IF NOT EXISTS idx_best_block_requests_received_at ON best_block_requests(received_at);
+
+-- Mining on table indexes
+CREATE INDEX IF NOT EXISTS idx_mining_ons_network ON mining_ons(network);
+CREATE INDEX IF NOT EXISTS idx_mining_ons_hash ON mining_ons(hash);
+CREATE INDEX IF NOT EXISTS idx_mining_ons_previous_hash ON mining_ons(previous_hash);
+CREATE INDEX IF NOT EXISTS idx_mining_ons_peer_id ON mining_ons(peer_id);
+CREATE INDEX IF NOT EXISTS idx_mining_ons_height ON mining_ons(height);
+CREATE INDEX IF NOT EXISTS idx_mining_ons_miner ON mining_ons(miner);
+CREATE INDEX IF NOT EXISTS idx_mining_ons_received_at ON mining_ons(received_at);
+
+-- Subtrees table indexes
+CREATE INDEX IF NOT EXISTS idx_subtrees_network ON subtrees(network);
+CREATE INDEX IF NOT EXISTS idx_subtrees_hash ON subtrees(hash);
+CREATE INDEX IF NOT EXISTS idx_subtrees_peer_id ON subtrees(peer_id);
+CREATE INDEX IF NOT EXISTS idx_subtrees_received_at ON subtrees(received_at);
+
+-- Handshakes table indexes
+CREATE INDEX IF NOT EXISTS idx_handshakes_network ON handshakes(network);
+CREATE INDEX IF NOT EXISTS idx_handshakes_type ON handshakes(type);
+CREATE INDEX IF NOT EXISTS idx_handshakes_peer_id ON handshakes(peer_id);
+CREATE INDEX IF NOT EXISTS idx_handshakes_best_height ON handshakes(best_height);
+CREATE INDEX IF NOT EXISTS idx_handshakes_best_hash ON handshakes(best_hash);
+CREATE INDEX IF NOT EXISTS idx_handshakes_received_at ON handshakes(received_at);
+
+-- Rejected transactions table indexes
+CREATE INDEX IF NOT EXISTS idx_rejected_txes_network ON rejected_txes(network);
+CREATE INDEX IF NOT EXISTS idx_rejected_txes_tx_id ON rejected_txes(tx_id);
+CREATE INDEX IF NOT EXISTS idx_rejected_txes_peer_id ON rejected_txes(peer_id);
+CREATE INDEX IF NOT EXISTS idx_rejected_txes_received_at ON rejected_txes(received_at);
+
+-- Compound indexes for common query patterns
+CREATE INDEX IF NOT EXISTS idx_messages_topic_received_at ON messages(topic, received_at);
+CREATE INDEX IF NOT EXISTS idx_blocks_network_height ON blocks(network, height);
+CREATE INDEX IF NOT EXISTS idx_block_headers_network_height ON block_headers(network, height);
+CREATE INDEX IF NOT EXISTS idx_block_headers_network_timestamp ON block_headers(network, timestamp);
+
+-- Update statistics
+ANALYZE;
+
+-- Check integrity
+PRAGMA integrity_check;
+
+-- Display current settings
+SELECT 'Journal Mode: ' || journal_mode FROM pragma_journal_mode();
+SELECT 'Synchronous: ' || synchronous FROM pragma_synchronous();
+SELECT 'Cache Size: ' || cache_size FROM pragma_cache_size();
+SELECT 'Page Count: ' || page_count FROM pragma_page_count();
+SELECT 'Page Size: ' || page_size FROM pragma_page_size();
diff --git a/scripts/optimize_production_db.sh b/scripts/optimize_production_db.sh

Original file line number	Diff line number	Diff line change
`@@ -4,10 +4,10 @@ import (`
`4`	`4`	`"time"`
`5`	`5`	`)`
`6`	`6`
`7`		`-// BestBlockRequest represents a request for the best block information from a peer`
	`7`	`+// BestBlockRequest represents a best block request message`
`8`	`8`	`type BestBlockRequest struct {`
`9`	`9`	ID uint `gorm:"primaryKey"`
`10`	`10`	Network string `gorm:"index;not null"`
`11`	`11`	PeerID string `gorm:"index;not null"`
`12`		`- ReceivedAt time.Time`
	`12`	+ ReceivedAt time.Time `gorm:"index"`
`13`	`13`	`}`
Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,7 @@ type Block struct {`
`13`	`13`	DataHubURL string `gorm:"type:text" json:"DataHubURL"`
`14`	`14`	PeerID string `gorm:"index;not null" json:"PeerID"`
`15`	`15`	Header string `gorm:"type:text" json:"Header"`
`16`		- ReceivedAt time.Time `json:"ReceivedAt"`
	`16`	+ ReceivedAt time.Time `gorm:"index" json:"ReceivedAt"`
`17`	`17`	`}`
`18`	`18`
`19`	`19`	`type BlockHeader struct {`
Original file line number	Diff line number	Diff line change
`@@ -15,5 +15,5 @@ type Handshake struct {`
`15`	`15`	DataHubURL string `gorm:"type:text"`
`16`	`16`	UserAgent string `gorm:"type:text"`
`17`	`17`	`Services uint64`
`18`		`- ReceivedAt time.Time`
	`18`	+ ReceivedAt time.Time `gorm:"index"`
`19`	`19`	`}`
Original file line number	Diff line number	Diff line change
`@@ -9,5 +9,5 @@ type Message struct {`
`9`	`9`	Topic string `gorm:"index;not null"`
`10`	`10`	Data string `gorm:"type:text;not null"`
`11`	`11`	Peer string `gorm:"type:text"`
`12`		`- ReceivedAt time.Time`
	`12`	+ ReceivedAt time.Time `gorm:"index"`
`13`	`13`	`}`
Original file line number	Diff line number	Diff line change
`@@ -16,5 +16,5 @@ type MiningOn struct {`
`16`	`16`	Miner string `gorm:"index"`
`17`	`17`	`SizeInBytes uint64`
`18`	`18`	`TxCount uint64`
`19`		`- ReceivedAt time.Time`
	`19`	+ ReceivedAt time.Time `gorm:"index"`
`20`	`20`	`}`
Original file line number	Diff line number	Diff line change
`@@ -11,5 +11,5 @@ type RejectedTx struct {`
`11`	`11`	TxID string `gorm:"index;not null"`
`12`	`12`	Reason string `gorm:"type:text"`
`13`	`13`	PeerID string `gorm:"index;not null"`
`14`		`- ReceivedAt time.Time`
	`14`	+ ReceivedAt time.Time `gorm:"index"`
`15`	`15`	`}`
Original file line number	Diff line number	Diff line change
`@@ -11,5 +11,5 @@ type Subtree struct {`
`11`	`11`	Hash string `gorm:"index;not null"`
`12`	`12`	DataHubURL string `gorm:"type:text"`
`13`	`13`	PeerID string `gorm:"index;not null"`
`14`		`- ReceivedAt time.Time`
	`14`	+ ReceivedAt time.Time `gorm:"index"`
`15`	`15`	`}`