Stream legacy blocks straight to legacy without deserialization

Author: oskarszoon

services/legacy/peer_server.go

@@ -1387,9 +1387,12 @@ func (s *server) pushBlockMsg(sp *serverPeer, hash *chainhash.Hash, doneChan cha
 		_ = reader.Close()
 	}()
 
-	var msgBlock wire.MsgBlock
-	if err = msgBlock.Deserialize(reader); err != nil {
-		sp.server.logger.Errorf("Unable to deserialize requested block hash %v: %v", hash, err)
+	// Use RawBlockMessage to avoid deserialize/serialize overhead for large blocks.
+	// This reads raw bytes directly and writes them to the wire, bypassing the
+	// expensive process of creating Go structs for millions of transactions.
+	rawBlockMsg, err := NewRawBlockMessage(reader)
+	if err != nil {
+		sp.server.logger.Errorf("Unable to read requested block hash %v: %v", hash, err)
 
 		if doneChan != nil {
 			doneChan <- struct{}{}
@@ -1414,7 +1417,7 @@ func (s *server) pushBlockMsg(sp *serverPeer, hash *chainhash.Hash, doneChan cha
 		dc = doneChan
 	}
 
-	sp.QueueMessageWithEncoding(&msgBlock, dc, encoding)
+	sp.QueueMessageWithEncoding(rawBlockMsg, dc, encoding)
 
 	// When the peer requests the final block that was advertised in
 	// response to a getblocks message which requested more blocks than

services/legacy/raw_block_message.go

@@ -0,0 +1,59 @@
+package legacy
+
+import (
+	"io"
+
+	"github.com/bsv-blockchain/go-wire"
+	"github.com/bsv-blockchain/teranode/errors"
+)
+
+// RawBlockMessage implements wire.Message for streaming raw block bytes
+// without the overhead of deserializing into wire.MsgBlock structure.
+//
+// This is significantly more efficient for large blocks (e.g., 4GB+) because:
+// - No CPU time spent deserializing millions of transactions into Go structs
+// - No CPU time spent re-serializing those structs back to bytes
+// - Reduced memory overhead (no Go struct allocation per transaction)
+//
+// The raw bytes are read from the source and written directly to the wire,
+// bypassing the deserialize/serialize roundtrip that would otherwise require
+// allocating memory for each transaction struct.
+type RawBlockMessage struct {
+	data []byte
+}
+
+// NewRawBlockMessage creates a RawBlockMessage by reading all bytes from the reader.
+// The data must be in wire-format block encoding (header + txcount + transactions).
+func NewRawBlockMessage(reader io.Reader) (*RawBlockMessage, error) {
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, errors.NewProcessingError("failed to read block data", err)
+	}
+
+	return &RawBlockMessage{data: data}, nil
+}
+
+// Command returns the protocol command string for the message.
+// Implements wire.Message interface.
+func (m *RawBlockMessage) Command() string {
+	return wire.CmdBlock
+}
+
+// BsvEncode writes the raw block bytes to the writer.
+// Implements wire.Message interface.
+func (m *RawBlockMessage) BsvEncode(w io.Writer, _ uint32, _ wire.MessageEncoding) error {
+	_, err := w.Write(m.data)
+	return err
+}
+
+// Bsvdecode is not supported for RawBlockMessage.
+// Implements wire.Message interface.
+func (m *RawBlockMessage) Bsvdecode(_ io.Reader, _ uint32, _ wire.MessageEncoding) error {
+	return errors.NewProcessingError("RawBlockMessage does not support decoding")
+}
+
+// MaxPayloadLength returns the length of the raw block data.
+// Implements wire.Message interface.
+func (m *RawBlockMessage) MaxPayloadLength(_ uint32) uint64 {
+	return uint64(len(m.data))
+}

services/legacy/raw_block_message_test.go

@@ -0,0 +1,82 @@
+package legacy
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/bsv-blockchain/go-wire"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRawBlockMessage_Command(t *testing.T) {
+	msg := &RawBlockMessage{data: []byte{}}
+	require.Equal(t, wire.CmdBlock, msg.Command())
+}
+
+func TestRawBlockMessage_MaxPayloadLength(t *testing.T) {
+	data := make([]byte, 1000)
+	msg := &RawBlockMessage{data: data}
+	require.Equal(t, uint64(1000), msg.MaxPayloadLength(0))
+}
+
+func TestRawBlockMessage_BsvEncode(t *testing.T) {
+	testData := []byte{0x01, 0x02, 0x03, 0x04, 0x05}
+	msg := &RawBlockMessage{data: testData}
+
+	var buf bytes.Buffer
+	err := msg.BsvEncode(&buf, 0, wire.BaseEncoding)
+	require.NoError(t, err)
+	require.Equal(t, testData, buf.Bytes())
+}
+
+func TestRawBlockMessage_Bsvdecode(t *testing.T) {
+	msg := &RawBlockMessage{}
+	err := msg.Bsvdecode(nil, 0, wire.BaseEncoding)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "does not support decoding")
+}
+
+func TestNewRawBlockMessage(t *testing.T) {
+	testData := []byte{0x01, 0x02, 0x03, 0x04, 0x05}
+	reader := bytes.NewReader(testData)
+
+	msg, err := NewRawBlockMessage(reader)
+	require.NoError(t, err)
+	require.Equal(t, testData, msg.data)
+}
+
+func TestNewRawBlockMessage_ReadError(t *testing.T) {
+	// Create a reader that will return an error
+	reader := &errorReader{}
+
+	msg, err := NewRawBlockMessage(reader)
+	require.Error(t, err)
+	require.Nil(t, msg)
+}
+
+// errorReader is a test helper that always returns an error
+type errorReader struct{}
+
+func (r *errorReader) Read(_ []byte) (int, error) {
+	return 0, io.ErrUnexpectedEOF
+}
+
+func TestRawBlockMessage_LargeData(t *testing.T) {
+	// Test with a larger block-like structure (1MB)
+	data := make([]byte, 1024*1024)
+	for i := range data {
+		data[i] = byte(i % 256)
+	}
+
+	reader := bytes.NewReader(data)
+	msg, err := NewRawBlockMessage(reader)
+	require.NoError(t, err)
+	require.Equal(t, uint64(1024*1024), msg.MaxPayloadLength(0))
+
+	// Verify encode produces the same data
+	var buf bytes.Buffer
+	err = msg.BsvEncode(&buf, 0, wire.BaseEncoding)
+	require.NoError(t, err)
+	require.Equal(t, data, buf.Bytes())
+}