From 4aa8fee4967758a52ee3284addccecd48c8bb113 Mon Sep 17 00:00:00 2001 From: lnikon Date: Mon, 25 Aug 2025 01:09:55 +0400 Subject: [PATCH 01/17] Integration & load testing: - Framework core components in-progress - KVStore interface & implementation added - Config for the entire framework added --- Dockerfile | 1 + infra/assets/tkvpp_config_standalone.json | 6 +- infra/docker-compose.standalone.yaml | 14 +- src/config.cpp | 2 +- tests/go/kvtest/cmd/kvtest/main.go | 5 + tests/go/kvtest/cmd/root.go | 46 +++++ tests/go/kvtest/go.sum | 39 +++++ tests/go/kvtest/main.go | 13 +- tests/go/kvtest/pkg/adapters/tinykvpp.go | 122 ++++++++++++++ tests/go/kvtest/pkg/core/config.go | 100 +++++++++++ tests/go/kvtest/pkg/core/context.go | 18 ++ tests/go/kvtest/pkg/core/interface.go | 39 +++++ .../kvtest/proto/raft/v1/raft_service.proto | 69 ++++++++ .../{ => tinykvpp/v1}/tinykvpp_service.pb.go | 88 +++++----- .../proto/tinykvpp/v1/tinykvpp_service.proto | 45 +++++ .../tinykvpp/v1/tinykvpp_service_grpc.pb.go | 159 ++++++++++++++++++ 16 files changed, 709 insertions(+), 57 deletions(-) create mode 100644 tests/go/kvtest/cmd/kvtest/main.go create mode 100644 tests/go/kvtest/cmd/root.go create mode 100644 tests/go/kvtest/pkg/adapters/tinykvpp.go create mode 100644 tests/go/kvtest/pkg/core/config.go create mode 100644 tests/go/kvtest/pkg/core/context.go create mode 100644 tests/go/kvtest/pkg/core/interface.go create mode 100644 tests/go/kvtest/proto/raft/v1/raft_service.proto rename tests/go/kvtest/proto/{ => tinykvpp/v1}/tinykvpp_service.pb.go (75%) create mode 100644 tests/go/kvtest/proto/tinykvpp/v1/tinykvpp_service.proto create mode 100644 tests/go/kvtest/proto/tinykvpp/v1/tinykvpp_service_grpc.pb.go diff --git a/Dockerfile b/Dockerfile index bc2ab7c..35075e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ ARG BUILD_TYPE=release RUN apt-get update && \ apt-get -y install \ cmake \ + ninja-build \ python3 \ python3-pip \ python3-virtualenv && \ diff --git a/infra/assets/tkvpp_config_standalone.json b/infra/assets/tkvpp_config_standalone.json index 23dfa7d..01c6102 100644 --- a/infra/assets/tkvpp_config_standalone.json +++ b/infra/assets/tkvpp_config_standalone.json @@ -3,7 +3,7 @@ "loggingLevel": "debug" }, "database": { - "path": "./var/tkvpp", + "path": "/var/tkvpp", "walFilename": "wal", "manifestFilenamePrefix": "manifest_", "mode": "standalone" @@ -28,10 +28,10 @@ "server": { "transport": "grpc", "host": "0.0.0.0", - "port": 8081, + "port": 9891, "id": 1, "peers": [ - "0.0.0.0:8081" + "0.0.0.0:9891" ] } } diff --git a/infra/docker-compose.standalone.yaml b/infra/docker-compose.standalone.yaml index 619c3a7..dfe1f4c 100644 --- a/infra/docker-compose.standalone.yaml +++ b/infra/docker-compose.standalone.yaml @@ -1,7 +1,15 @@ --- -version: '3.8' services: - app: + tkvpp-node-standalone: image: tinykvpp-clang:latest - container_name: tkvpp-node-1 + security_opt: + - seccomp:unconfined + container_name: tkvpp-node-standalone command: ["/app/tkvpp", "-c", "/app/config.json"] + ports: + - "9891:9891" + volumes: + - ./assets/tkvpp_config_standalone.json:/app/config.json + - tkvpp-node-standalone:/var/tkvpp +volumes: + tkvpp-node-standalone: diff --git a/src/config.cpp b/src/config.cpp index 81f4f30..5ab3fcf 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -155,7 +155,7 @@ auto loadConfigJson(const std::string &configPath) -> json std::fstream configStream(configPath, std::fstream::in); if (!configStream.is_open()) { - throw std::runtime_error(fmt::format("Unable to open config file: %s", configPath)); + throw std::runtime_error(fmt::format("Unable to open config file: {}", configPath)); } return json::parse(configStream); } diff --git a/tests/go/kvtest/cmd/kvtest/main.go b/tests/go/kvtest/cmd/kvtest/main.go new file mode 100644 index 0000000..e5ad8c4 --- /dev/null +++ b/tests/go/kvtest/cmd/kvtest/main.go @@ -0,0 +1,5 @@ +package main + +import () + +func main() {} diff --git a/tests/go/kvtest/cmd/root.go b/tests/go/kvtest/cmd/root.go new file mode 100644 index 0000000..4e2dae0 --- /dev/null +++ b/tests/go/kvtest/cmd/root.go @@ -0,0 +1,46 @@ +/* +Copyright © 2025 NAME HERE +*/ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "kvtest", + Short: "A brief description of your application", + Long: `A longer description that spans multiple lines and likely contains +examples and usage of using your application. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + // Uncomment the following line if your bare application + // has an action associated with it: + // Run: func(cmd *cobra.Command, args []string) { }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + + // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.kvtest.yaml)") + + // Cobra also supports local flags, which will only run + // when this action is called directly. + rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/tests/go/kvtest/go.sum b/tests/go/kvtest/go.sum index e63baf5..bdf9b15 100644 --- a/tests/go/kvtest/go.sum +++ b/tests/go/kvtest/go.sum @@ -1,9 +1,45 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/go-chi/httprate v0.15.0 h1:j54xcWV9KGmPf/X4H32/aTH+wBlrvxL7P+SdnRqxh5g= github.com/go-chi/httprate v0.15.0/go.mod h1:rzGHhVrsBn3IMLYDOZQsSU4fJNWcjui4fWKJcCId1R4= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= +github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= +github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= @@ -18,3 +54,6 @@ google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tests/go/kvtest/main.go b/tests/go/kvtest/main.go index 4d11a74..f57034e 100644 --- a/tests/go/kvtest/main.go +++ b/tests/go/kvtest/main.go @@ -1,10 +1,11 @@ -package kvtest +/* +Copyright © 2025 NAME HERE -import ( - v1 "github.com/lnikon/tinykvpp/tests/go/kvtest/proto" - "google.golang.org/grpc" -) +*/ +package main + +import "github.com/lnikon/tinykvpp/tests/go/kvtest/cmd" func main() { - let req := v1.GetRequest{} + cmd.Execute() } diff --git a/tests/go/kvtest/pkg/adapters/tinykvpp.go b/tests/go/kvtest/pkg/adapters/tinykvpp.go new file mode 100644 index 0000000..f075412 --- /dev/null +++ b/tests/go/kvtest/pkg/adapters/tinykvpp.go @@ -0,0 +1,122 @@ +package adapters + +import ( + "context" + "fmt" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" + pb "github.com/lnikon/tinykvpp/tests/go/kvtest/proto/tinykvpp/v1" +) + +type TinyKVPPConfig struct { + Address string `yaml:"address"` + ConnectionTimeout time.Duration `yaml:"connection_timeout"` + RequestTimeout time.Duration `yaml:"request_timeout"` + MaxRetries int `yaml:"max_retries"` +} + +type TinyKVPPAdapter struct { + config TinyKVPPConfig + conn *grpc.ClientConn + client pb.TinyKVPPServiceClient +} + +var _ core.KVStoreInterface = (*TinyKVPPAdapter)(nil) + +func NewTinyKVPPAdapter(config core.Config) *TinyKVPPAdapter { + return &TinyKVPPAdapter{} +} + +func (a *TinyKVPPAdapter) Connect(config core.Config) error { + tinyConfig, ok := config.(TinyKVPPConfig) + if !ok { + return fmt.Errorf("invalid config type for TinyKVPP adapter") + } + + a.config = tinyConfig + + conn, err := grpc.NewClient( + a.config.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return fmt.Errorf("failed to connect to %s: %w", a.config.Address, err) + } + + a.conn = conn + a.client = pb.NewTinyKVPPServiceClient(conn) + + return a.HealthCheck(context.Background()) +} + +func (a *TinyKVPPAdapter) Close() error { + if a.conn != nil { + return a.conn.Close() + } + return nil +} + +func (a *TinyKVPPAdapter) Get(ctx context.Context, key []byte) ([]byte, error) { + if a.conn == nil { + return nil, fmt.Errorf("not connected to TinyKVPP") + } + + // Apply requet timeout + ctx, cancel := context.WithTimeout(ctx, a.config.RequestTimeout) + defer cancel() + + req := &pb.GetRequest{Key: string(key)} + resp, err := a.client.Get(ctx, req) + if err != nil { + return nil, fmt.Errorf("grpc get failed for key %s: %v", key, err) + } + + if !resp.Found { + return nil, &core.KVError{Op: "get", Err: fmt.Errorf("key not found: %s", string(key))} + } + + return []byte(resp.Value), nil +} + +func (a *TinyKVPPAdapter) Put(ctx context.Context, key, value []byte) error { + if a.conn == nil { + return fmt.Errorf("not connected to TinyKVPP") + } + + // Apply requet timeout + ctx, cancel := context.WithTimeout(ctx, a.config.RequestTimeout) + defer cancel() + + req := &pb.PutRequest{Key: string(key), Value: string(value)} + resp, err := a.client.Put(ctx, req) + if err != nil { + return fmt.Errorf("grpc put failed for key %s: %v\n", key, err) + } + + if len(resp.Error) > 0 { + return &core.KVError{Op: "put", Err: fmt.Errorf("put failed: %s", resp.Error)} + } + + if !resp.Success { + return &core.KVError{Op: "put", Err: fmt.Errorf("put operation unsuccessful")} + } + + return nil +} + +func (a *TinyKVPPAdapter) HealthCheck(ctx context.Context) error { + if a.conn == nil { + return fmt.Errorf("not connected") + } + + testKey := "__health_check__" + a.client.Get(ctx, &pb.GetRequest{Key: testKey}) + + return nil +} + +func (a *TinyKVPPAdapter) Name() string { + return "TinyKVPP" +} diff --git a/tests/go/kvtest/pkg/core/config.go b/tests/go/kvtest/pkg/core/config.go new file mode 100644 index 0000000..63e4a8f --- /dev/null +++ b/tests/go/kvtest/pkg/core/config.go @@ -0,0 +1,100 @@ +package core + +import ( + "time" +) + +type TestConfig struct { + // General settings + Timeout time.Duration `yaml:"timeout"` + LogLevel string `yaml:"log_level"` + + // KV Store configuration + Adapter AdapterConfig `yaml:"adapter"` + + // Test execution settings + Integration IntegrationConfig `yaml:"integration"` + Load LoadConfig `yaml:"load"` + + // Scenario configurations + Scenarios []ScenarioConfig `yaml:"scenarios"` +} + +// AdapterConfig specified which KV store adapter to use +type AdapterConfig struct { + Type string `yaml:"type"` + Address string `yaml:"address"` + ConnectionTimeout time.Duration `yaml:"connection_timeout"` + RequestTimeout time.Duration `yaml:"request_timeout"` + MaxRetries int `yaml:"max_retries"` +} + +// IntegrationConfig settings for integration tests +type IntegrationConfig struct { + Enabled bool `yaml:"enabled"` + Validations []string `yaml:"validations"` + Parallel bool `yaml:"parallel"` +} + +// LoadConfig settings for load tests +type LoadConfig struct { + Enabled bool `yaml:"enabled"` + Algorithm string `yaml:"algorithm"` + StartUsers int `yaml:"start_users"` + TargetUsers int `yaml:"target_users"` + Duration time.Duration `yaml:"durartion"` + RampUpTime time.Duration `yaml:"ramp_up_time"` + ThinkTime time.Duration `yaml:"think_time"` + Weights map[string]int `yaml:"weights"` +} + +// ScenarioConfig defines test scenario parameters +type ScenarioConfig struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + Enabled bool `yaml:"enabled"` + Parameters map[string]interface{} `yaml:"parameters"` +} + +// DefaultConfig() returns a default configuration +func DefaultConfig() *TestConfig { + return &TestConfig{ + Timeout: 30 * time.Second, + LogLevel: "info", + + Adapter: AdapterConfig{ + Type: "tinykvpp", + Address: "localhost:9891", + ConnectionTimeout: 5 * time.Second, + RequestTimeout: 1 * time.Second, + MaxRetries: 3, + }, + + Integration: IntegrationConfig{ + Enabled: true, + Validations: []string{"consistency", "isolation"}, + Parallel: false, + }, + + Load: LoadConfig{ + Enabled: true, + Algorithm: "constant", + StartUsers: 1, + TargetUsers: 10, + Duration: 60 * time.Second, + ThinkTime: 100 * time.Microsecond, + }, + + Scenarios: []ScenarioConfig{ + { + Name: "basic_crud", + Type: "crud", + Enabled: true, + Parameters: map[string]interface{}{ + "read_ratio": 0.7, + "write_ratio": 0.3, + }, + }, + }, + } +} diff --git a/tests/go/kvtest/pkg/core/context.go b/tests/go/kvtest/pkg/core/context.go new file mode 100644 index 0000000..2101ff8 --- /dev/null +++ b/tests/go/kvtest/pkg/core/context.go @@ -0,0 +1,18 @@ +package core + +import ( + "context" +) + +type TestContext struct { + KVStore KVStoreInterface + + ctx context.Context +} + +func NewTestContext(adapter KVStoreInterface) TestContext { + return TestContext{ + KVStore: adapter, + ctx: context.Background(), + } +} diff --git a/tests/go/kvtest/pkg/core/interface.go b/tests/go/kvtest/pkg/core/interface.go new file mode 100644 index 0000000..0cac6f1 --- /dev/null +++ b/tests/go/kvtest/pkg/core/interface.go @@ -0,0 +1,39 @@ +package core + +import ( + "context" +) + +// Configuration for KVStore adapters +type Config interface{} + +type KVStoreInterface interface { + // Connection management + Connect(config Config) error + Close() error + + // CRUD operations + Get(ctx context.Context, key []byte) ([]byte, error) + Put(ctx context.Context, key, value []byte) error + + // Testing utilities + HealthCheck(ctx context.Context) error + Name() string +} + +// Common errors +type KVError struct { + Op string + Err error +} + +func (e *KVError) Error() string { + return e.Op + ": " + e.Err.Error() +} + +// Predefined errors +var ( + ErrKeyNotFound = &KVError{Op: "get", Err: context.DeadlineExceeded} + ErrPutFailed = &KVError{Op: "put", Err: context.Canceled} + ErrDeleteFailed = &KVError{Op: "delete", Err: context.Canceled} +) diff --git a/tests/go/kvtest/proto/raft/v1/raft_service.proto b/tests/go/kvtest/proto/raft/v1/raft_service.proto new file mode 100644 index 0000000..c05b9f0 --- /dev/null +++ b/tests/go/kvtest/proto/raft/v1/raft_service.proto @@ -0,0 +1,69 @@ +syntax = "proto3"; + +package raft.v1; + +service RaftService { + rpc AppendEntries(AppendEntriesRequest) returns (AppendEntriesResponse); + rpc RequestVote(RequestVoteRequest) returns (RequestVoteResponse); + rpc Replicate(ReplicateRequest) returns (ReplicateResponse); +} + +enum Command { + COMMAND_UNSPECIFIED = 0; + COMMAND_PUT = 1; + COMMAND_GET = 2; +} + +enum NodeState { + NODE_STATE_UNSPECIFIED = 0; + NODE_STATE_LEADER = 1; + NODE_STATE_FOLLOWER = 2; + NODE_STATE_CANDIDATE = 3; +} + +message LogEntry { + uint32 term = 1; + uint32 index = 2; + bytes payload = 4; +} + +message AppendEntriesRequest { + uint32 term = 1; + string leader_id = 2; + uint32 prev_log_index = 3; + uint32 prev_log_term = 4; + repeated LogEntry entries = 5; + uint32 leader_commit = 6; + uint32 sender_id = 7; +} + +message AppendEntriesResponse { + uint32 term = 1; + bool success = 2; + uint32 match_index = 3; + uint32 responder_id = 4; +} + +message RequestVoteRequest { + uint32 term = 1; + uint32 candidate_id = 2; + uint32 last_log_index = 3; + uint32 last_log_term = 4; + uint32 sender_id = 7; +} + +message RequestVoteResponse { + uint32 term = 1; + uint32 vote_granted = 2; + uint32 responder_id = 3; +} + +message ReplicateRequest { + uint32 term = 1; + uint32 sender_id = 2; + repeated string payloads = 3; +} + +message ReplicateResponse { + string status = 1; +} diff --git a/tests/go/kvtest/proto/tinykvpp_service.pb.go b/tests/go/kvtest/proto/tinykvpp/v1/tinykvpp_service.pb.go similarity index 75% rename from tests/go/kvtest/proto/tinykvpp_service.pb.go rename to tests/go/kvtest/proto/tinykvpp/v1/tinykvpp_service.pb.go index 2a464f0..872d2f4 100644 --- a/tests/go/kvtest/proto/tinykvpp_service.pb.go +++ b/tests/go/kvtest/proto/tinykvpp/v1/tinykvpp_service.pb.go @@ -1,8 +1,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.7 +// protoc-gen-go v1.36.8 // protoc v3.19.6 -// source: tinykvpp_service.proto +// source: proto/tinykvpp/v1/tinykvpp_service.proto package v1 @@ -57,11 +57,11 @@ func (x DatabaseOperation_Type) String() string { } func (DatabaseOperation_Type) Descriptor() protoreflect.EnumDescriptor { - return file_tinykvpp_service_proto_enumTypes[0].Descriptor() + return file_proto_tinykvpp_v1_tinykvpp_service_proto_enumTypes[0].Descriptor() } func (DatabaseOperation_Type) Type() protoreflect.EnumType { - return &file_tinykvpp_service_proto_enumTypes[0] + return &file_proto_tinykvpp_v1_tinykvpp_service_proto_enumTypes[0] } func (x DatabaseOperation_Type) Number() protoreflect.EnumNumber { @@ -70,7 +70,7 @@ func (x DatabaseOperation_Type) Number() protoreflect.EnumNumber { // Deprecated: Use DatabaseOperation_Type.Descriptor instead. func (DatabaseOperation_Type) EnumDescriptor() ([]byte, []int) { - return file_tinykvpp_service_proto_rawDescGZIP(), []int{4, 0} + return file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDescGZIP(), []int{4, 0} } type PutRequest struct { @@ -83,7 +83,7 @@ type PutRequest struct { func (x *PutRequest) Reset() { *x = PutRequest{} - mi := &file_tinykvpp_service_proto_msgTypes[0] + mi := &file_proto_tinykvpp_v1_tinykvpp_service_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -95,7 +95,7 @@ func (x *PutRequest) String() string { func (*PutRequest) ProtoMessage() {} func (x *PutRequest) ProtoReflect() protoreflect.Message { - mi := &file_tinykvpp_service_proto_msgTypes[0] + mi := &file_proto_tinykvpp_v1_tinykvpp_service_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -108,7 +108,7 @@ func (x *PutRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PutRequest.ProtoReflect.Descriptor instead. func (*PutRequest) Descriptor() ([]byte, []int) { - return file_tinykvpp_service_proto_rawDescGZIP(), []int{0} + return file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDescGZIP(), []int{0} } func (x *PutRequest) GetKey() string { @@ -135,7 +135,7 @@ type PutResponse struct { func (x *PutResponse) Reset() { *x = PutResponse{} - mi := &file_tinykvpp_service_proto_msgTypes[1] + mi := &file_proto_tinykvpp_v1_tinykvpp_service_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -147,7 +147,7 @@ func (x *PutResponse) String() string { func (*PutResponse) ProtoMessage() {} func (x *PutResponse) ProtoReflect() protoreflect.Message { - mi := &file_tinykvpp_service_proto_msgTypes[1] + mi := &file_proto_tinykvpp_v1_tinykvpp_service_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -160,7 +160,7 @@ func (x *PutResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PutResponse.ProtoReflect.Descriptor instead. func (*PutResponse) Descriptor() ([]byte, []int) { - return file_tinykvpp_service_proto_rawDescGZIP(), []int{1} + return file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDescGZIP(), []int{1} } func (x *PutResponse) GetSuccess() bool { @@ -186,7 +186,7 @@ type GetRequest struct { func (x *GetRequest) Reset() { *x = GetRequest{} - mi := &file_tinykvpp_service_proto_msgTypes[2] + mi := &file_proto_tinykvpp_v1_tinykvpp_service_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -198,7 +198,7 @@ func (x *GetRequest) String() string { func (*GetRequest) ProtoMessage() {} func (x *GetRequest) ProtoReflect() protoreflect.Message { - mi := &file_tinykvpp_service_proto_msgTypes[2] + mi := &file_proto_tinykvpp_v1_tinykvpp_service_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -211,7 +211,7 @@ func (x *GetRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRequest.ProtoReflect.Descriptor instead. func (*GetRequest) Descriptor() ([]byte, []int) { - return file_tinykvpp_service_proto_rawDescGZIP(), []int{2} + return file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDescGZIP(), []int{2} } func (x *GetRequest) GetKey() string { @@ -231,7 +231,7 @@ type GetResponse struct { func (x *GetResponse) Reset() { *x = GetResponse{} - mi := &file_tinykvpp_service_proto_msgTypes[3] + mi := &file_proto_tinykvpp_v1_tinykvpp_service_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -243,7 +243,7 @@ func (x *GetResponse) String() string { func (*GetResponse) ProtoMessage() {} func (x *GetResponse) ProtoReflect() protoreflect.Message { - mi := &file_tinykvpp_service_proto_msgTypes[3] + mi := &file_proto_tinykvpp_v1_tinykvpp_service_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -256,7 +256,7 @@ func (x *GetResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetResponse.ProtoReflect.Descriptor instead. func (*GetResponse) Descriptor() ([]byte, []int) { - return file_tinykvpp_service_proto_rawDescGZIP(), []int{3} + return file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDescGZIP(), []int{3} } func (x *GetResponse) GetFound() bool { @@ -287,7 +287,7 @@ type DatabaseOperation struct { func (x *DatabaseOperation) Reset() { *x = DatabaseOperation{} - mi := &file_tinykvpp_service_proto_msgTypes[4] + mi := &file_proto_tinykvpp_v1_tinykvpp_service_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -299,7 +299,7 @@ func (x *DatabaseOperation) String() string { func (*DatabaseOperation) ProtoMessage() {} func (x *DatabaseOperation) ProtoReflect() protoreflect.Message { - mi := &file_tinykvpp_service_proto_msgTypes[4] + mi := &file_proto_tinykvpp_v1_tinykvpp_service_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -312,7 +312,7 @@ func (x *DatabaseOperation) ProtoReflect() protoreflect.Message { // Deprecated: Use DatabaseOperation.ProtoReflect.Descriptor instead. func (*DatabaseOperation) Descriptor() ([]byte, []int) { - return file_tinykvpp_service_proto_rawDescGZIP(), []int{4} + return file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDescGZIP(), []int{4} } func (x *DatabaseOperation) GetType() DatabaseOperation_Type { @@ -343,11 +343,11 @@ func (x *DatabaseOperation) GetRequestId() uint64 { return 0 } -var File_tinykvpp_service_proto protoreflect.FileDescriptor +var File_proto_tinykvpp_v1_tinykvpp_service_proto protoreflect.FileDescriptor -const file_tinykvpp_service_proto_rawDesc = "" + +const file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDesc = "" + "\n" + - "\x16tinykvpp_service.proto\x12\vtinykvpp.v1\"4\n" + + "(proto/tinykvpp/v1/tinykvpp_service.proto\x12\vtinykvpp.v1\"4\n" + "\n" + "PutRequest\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + @@ -378,20 +378,20 @@ const file_tinykvpp_service_proto_rawDesc = "" + "\x03Get\x12\x17.tinykvpp.v1.GetRequest\x1a\x18.tinykvpp.v1.GetResponseB/Z-github.com/lnikon/tinykvpp/protos/tinykvpp/v1b\x06proto3" var ( - file_tinykvpp_service_proto_rawDescOnce sync.Once - file_tinykvpp_service_proto_rawDescData []byte + file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDescOnce sync.Once + file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDescData []byte ) -func file_tinykvpp_service_proto_rawDescGZIP() []byte { - file_tinykvpp_service_proto_rawDescOnce.Do(func() { - file_tinykvpp_service_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_tinykvpp_service_proto_rawDesc), len(file_tinykvpp_service_proto_rawDesc))) +func file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDescGZIP() []byte { + file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDescOnce.Do(func() { + file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDesc), len(file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDesc))) }) - return file_tinykvpp_service_proto_rawDescData + return file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDescData } -var file_tinykvpp_service_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_tinykvpp_service_proto_msgTypes = make([]protoimpl.MessageInfo, 5) -var file_tinykvpp_service_proto_goTypes = []any{ +var file_proto_tinykvpp_v1_tinykvpp_service_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_proto_tinykvpp_v1_tinykvpp_service_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_proto_tinykvpp_v1_tinykvpp_service_proto_goTypes = []any{ (DatabaseOperation_Type)(0), // 0: tinykvpp.v1.DatabaseOperation.Type (*PutRequest)(nil), // 1: tinykvpp.v1.PutRequest (*PutResponse)(nil), // 2: tinykvpp.v1.PutResponse @@ -399,7 +399,7 @@ var file_tinykvpp_service_proto_goTypes = []any{ (*GetResponse)(nil), // 4: tinykvpp.v1.GetResponse (*DatabaseOperation)(nil), // 5: tinykvpp.v1.DatabaseOperation } -var file_tinykvpp_service_proto_depIdxs = []int32{ +var file_proto_tinykvpp_v1_tinykvpp_service_proto_depIdxs = []int32{ 0, // 0: tinykvpp.v1.DatabaseOperation.type:type_name -> tinykvpp.v1.DatabaseOperation.Type 1, // 1: tinykvpp.v1.TinyKVPPService.Put:input_type -> tinykvpp.v1.PutRequest 3, // 2: tinykvpp.v1.TinyKVPPService.Get:input_type -> tinykvpp.v1.GetRequest @@ -412,27 +412,27 @@ var file_tinykvpp_service_proto_depIdxs = []int32{ 0, // [0:1] is the sub-list for field type_name } -func init() { file_tinykvpp_service_proto_init() } -func file_tinykvpp_service_proto_init() { - if File_tinykvpp_service_proto != nil { +func init() { file_proto_tinykvpp_v1_tinykvpp_service_proto_init() } +func file_proto_tinykvpp_v1_tinykvpp_service_proto_init() { + if File_proto_tinykvpp_v1_tinykvpp_service_proto != nil { return } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: unsafe.Slice(unsafe.StringData(file_tinykvpp_service_proto_rawDesc), len(file_tinykvpp_service_proto_rawDesc)), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDesc), len(file_proto_tinykvpp_v1_tinykvpp_service_proto_rawDesc)), NumEnums: 1, NumMessages: 5, NumExtensions: 0, NumServices: 1, }, - GoTypes: file_tinykvpp_service_proto_goTypes, - DependencyIndexes: file_tinykvpp_service_proto_depIdxs, - EnumInfos: file_tinykvpp_service_proto_enumTypes, - MessageInfos: file_tinykvpp_service_proto_msgTypes, + GoTypes: file_proto_tinykvpp_v1_tinykvpp_service_proto_goTypes, + DependencyIndexes: file_proto_tinykvpp_v1_tinykvpp_service_proto_depIdxs, + EnumInfos: file_proto_tinykvpp_v1_tinykvpp_service_proto_enumTypes, + MessageInfos: file_proto_tinykvpp_v1_tinykvpp_service_proto_msgTypes, }.Build() - File_tinykvpp_service_proto = out.File - file_tinykvpp_service_proto_goTypes = nil - file_tinykvpp_service_proto_depIdxs = nil + File_proto_tinykvpp_v1_tinykvpp_service_proto = out.File + file_proto_tinykvpp_v1_tinykvpp_service_proto_goTypes = nil + file_proto_tinykvpp_v1_tinykvpp_service_proto_depIdxs = nil } diff --git a/tests/go/kvtest/proto/tinykvpp/v1/tinykvpp_service.proto b/tests/go/kvtest/proto/tinykvpp/v1/tinykvpp_service.proto new file mode 100644 index 0000000..fa2e35c --- /dev/null +++ b/tests/go/kvtest/proto/tinykvpp/v1/tinykvpp_service.proto @@ -0,0 +1,45 @@ +syntax = "proto3"; + +package tinykvpp.v1; + +option go_package = "github.com/lnikon/tinykvpp/protos/tinykvpp/v1"; + +service TinyKVPPService { + rpc Put(PutRequest) returns (PutResponse); + rpc Get(GetRequest) returns (GetResponse); +} + +message PutRequest { + string key = 1; + string value = 2; +} + +message PutResponse { + bool success = 1; + string error = 2; +} + +message GetRequest { + string key = 1; +} + +message GetResponse { + bool found = 1; + string value = 2; +} + +// For internal database use +message DatabaseOperation { + enum Type { + TYPE_UNSPECIFIED = 0; + TYPE_PUT = 1; + TYPE_DELETE = 2; + TYPE_BATCH = 3; + } + + Type type = 1; + string key = 2; + string value = 3; + // Reserve 4 for Batch operations + uint64 request_id = 5; +} diff --git a/tests/go/kvtest/proto/tinykvpp/v1/tinykvpp_service_grpc.pb.go b/tests/go/kvtest/proto/tinykvpp/v1/tinykvpp_service_grpc.pb.go new file mode 100644 index 0000000..895ee23 --- /dev/null +++ b/tests/go/kvtest/proto/tinykvpp/v1/tinykvpp_service_grpc.pb.go @@ -0,0 +1,159 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v3.19.6 +// source: proto/tinykvpp/v1/tinykvpp_service.proto + +package v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + TinyKVPPService_Put_FullMethodName = "/tinykvpp.v1.TinyKVPPService/Put" + TinyKVPPService_Get_FullMethodName = "/tinykvpp.v1.TinyKVPPService/Get" +) + +// TinyKVPPServiceClient is the client API for TinyKVPPService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type TinyKVPPServiceClient interface { + Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error) + Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetResponse, error) +} + +type tinyKVPPServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewTinyKVPPServiceClient(cc grpc.ClientConnInterface) TinyKVPPServiceClient { + return &tinyKVPPServiceClient{cc} +} + +func (c *tinyKVPPServiceClient) Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(PutResponse) + err := c.cc.Invoke(ctx, TinyKVPPService_Put_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *tinyKVPPServiceClient) Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetResponse) + err := c.cc.Invoke(ctx, TinyKVPPService_Get_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// TinyKVPPServiceServer is the server API for TinyKVPPService service. +// All implementations must embed UnimplementedTinyKVPPServiceServer +// for forward compatibility. +type TinyKVPPServiceServer interface { + Put(context.Context, *PutRequest) (*PutResponse, error) + Get(context.Context, *GetRequest) (*GetResponse, error) + mustEmbedUnimplementedTinyKVPPServiceServer() +} + +// UnimplementedTinyKVPPServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedTinyKVPPServiceServer struct{} + +func (UnimplementedTinyKVPPServiceServer) Put(context.Context, *PutRequest) (*PutResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Put not implemented") +} +func (UnimplementedTinyKVPPServiceServer) Get(context.Context, *GetRequest) (*GetResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Get not implemented") +} +func (UnimplementedTinyKVPPServiceServer) mustEmbedUnimplementedTinyKVPPServiceServer() {} +func (UnimplementedTinyKVPPServiceServer) testEmbeddedByValue() {} + +// UnsafeTinyKVPPServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to TinyKVPPServiceServer will +// result in compilation errors. +type UnsafeTinyKVPPServiceServer interface { + mustEmbedUnimplementedTinyKVPPServiceServer() +} + +func RegisterTinyKVPPServiceServer(s grpc.ServiceRegistrar, srv TinyKVPPServiceServer) { + // If the following call pancis, it indicates UnimplementedTinyKVPPServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&TinyKVPPService_ServiceDesc, srv) +} + +func _TinyKVPPService_Put_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PutRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TinyKVPPServiceServer).Put(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TinyKVPPService_Put_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TinyKVPPServiceServer).Put(ctx, req.(*PutRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _TinyKVPPService_Get_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TinyKVPPServiceServer).Get(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TinyKVPPService_Get_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TinyKVPPServiceServer).Get(ctx, req.(*GetRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// TinyKVPPService_ServiceDesc is the grpc.ServiceDesc for TinyKVPPService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var TinyKVPPService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "tinykvpp.v1.TinyKVPPService", + HandlerType: (*TinyKVPPServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Put", + Handler: _TinyKVPPService_Put_Handler, + }, + { + MethodName: "Get", + Handler: _TinyKVPPService_Get_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "proto/tinykvpp/v1/tinykvpp_service.proto", +} From 23b0ad402cd93c7c286397fc0b54dc3a5919fadf Mon Sep 17 00:00:00 2001 From: lnikon Date: Tue, 26 Aug 2025 01:12:36 +0400 Subject: [PATCH 02/17] Integration & load testing: - Add interface for scenarios - Implement CRUD scenario --- tests/go/kvtest/pkg/core/context.go | 89 ++++++++++++- tests/go/kvtest/pkg/scenarios/crud.go | 136 ++++++++++++++++++++ tests/go/kvtest/pkg/scenarios/interface.go | 38 ++++++ tests/go/kvtest/pkg/validators/interface.go | 3 + 4 files changed, 260 insertions(+), 6 deletions(-) create mode 100644 tests/go/kvtest/pkg/scenarios/crud.go create mode 100644 tests/go/kvtest/pkg/scenarios/interface.go create mode 100644 tests/go/kvtest/pkg/validators/interface.go diff --git a/tests/go/kvtest/pkg/core/context.go b/tests/go/kvtest/pkg/core/context.go index 2101ff8..6840028 100644 --- a/tests/go/kvtest/pkg/core/context.go +++ b/tests/go/kvtest/pkg/core/context.go @@ -2,17 +2,94 @@ package core import ( "context" + "fmt" + "math/rand" + "time" ) type TestContext struct { - KVStore KVStoreInterface + KV KVStoreInterface + // Metrics *metrics.MetricsCollector + Config *TestConfig + ctx context.Context + cancel context.CancelFunc + VirtualUserID int + TestPhase string - ctx context.Context + // Data tracking + generatedKeys []string + writtenData map[string][]byte } -func NewTestContext(adapter KVStoreInterface) TestContext { - return TestContext{ - KVStore: adapter, - ctx: context.Background(), +func NewTestContext(adapter KVStoreInterface, config *TestConfig) *TestContext { + ctx, cancel := context.WithCancel(context.Background()) + + return &TestContext{ + KV: adapter, + // Metrics: metrics.NewMetricsCollector(), + Config: config, + ctx: ctx, + cancel: cancel, + TestPhase: "init", + generatedKeys: make([]string, 0), + writtenData: make(map[string][]byte), + } +} + +func (tc *TestContext) Context() context.Context { + return tc.ctx +} + +func (tc *TestContext) Cancel() { + tc.cancel() +} + +func (tc *TestContext) GenerateKey() string { + key := fmt.Sprintf("test_%d_%d_%d", + tc.VirtualUserID, + time.Now().UnixNano(), + rand.Intn(1000)) + tc.generatedKeys = append(tc.generatedKeys, key) + return key +} + +func (tc *TestContext) GenerateValue(size int) []byte { + if size <= 0 { + size = 256 + } + + value := make([]byte, size) + for i := range value { + value[i] = byte('A' + (i % 26)) } + return value +} + +func (tc *TestContext) GetRandomExistingKey() string { + if len(tc.generatedKeys) == 0 { + return tc.GenerateKey() + } + + keys := make([]string, 0, len(tc.writtenData)) + for key := range tc.writtenData { + keys = append(keys, key) + } + + return keys[rand.Intn(len(keys))] +} + +func (tc *TestContext) RecordWrite(key string, value []byte) { + tc.writtenData[key] = value +} + +func (tc *TestContext) GetWrittenData() map[string][]byte { + return tc.writtenData +} + +func (tc *TestContext) Cleanup() error { + return fmt.Errorf("delete not implemented") + // TODO(lnikon): Uncomment once Delete() is implemented + // for _, key := range tc.generatedKeys { + // // tc.KV.Delete(tc.ctx, []byte(key)) + // } } diff --git a/tests/go/kvtest/pkg/scenarios/crud.go b/tests/go/kvtest/pkg/scenarios/crud.go new file mode 100644 index 0000000..e073f50 --- /dev/null +++ b/tests/go/kvtest/pkg/scenarios/crud.go @@ -0,0 +1,136 @@ +package scenarios + +import ( + "bytes" + "fmt" + "math/rand" + // "time" + + "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" +) + +// CRUDScenario implements basic CRUD operations +type CRUDScenario struct { + name string + readRatio float64 + writeRatio float64 + valueSize int +} + +// NewCRUDScenario creates a new CRUD scenario +func NewCRUDScenario(params map[string]interface{}) *CRUDScenario { + scenario := &CRUDScenario{ + name: "CRUD Operations", + readRatio: 0.7, + writeRatio: 0.3, + valueSize: 256, + } + + // Parse parameters + if val, ok := params["read_ratio"].(float64); ok { + scenario.readRatio = val + } + if val, ok := params["write_ratio"].(float64); ok { + scenario.writeRatio = val + } + if val, ok := params["value_size"].(int); ok { + scenario.valueSize = val + } + + return scenario +} + +func (s *CRUDScenario) Execute(ctx *core.TestContext) error { + // startTime := time.Now() + + operation := s.selectOperation() + + var err error + switch operation { + case "read": + err = s.executeRead(ctx) + case "write": + err = s.executeWrite(ctx) + case "delete": + case "exists": + err = fmt.Errorf("operation not implemented") + } + + // duration := time.Since(startTime) + + // TODO(lnikon): Implement metric collection + // ctx.Metrics.RecordOperation(operation, duration, err) + + return err +} + +func (s *CRUDScenario) Name() string { + return s.name +} + +func (s *CRUDScenario) Setup(ctx *core.TestContext) error { + // Pre-populate some data for read operations + for range 10 { + key := ctx.GenerateKey() + value := ctx.GenerateValue(s.valueSize) + + if err := ctx.KV.Put(ctx.Context(), []byte(key), value); err != nil { + return fmt.Errorf("setup failed to put key %s: %w", key, err) + } + + ctx.RecordWrite(key, value) + } + return nil +} + +func (s *CRUDScenario) Teardown(ctx *core.TestContext) error { + return ctx.Cleanup() +} + +func (s *CRUDScenario) selectOperation() string { + r := rand.Float64() + + if r < s.readRatio { + return "read" + } else if r < s.readRatio+s.writeRatio { + return "write" + } else if r < s.readRatio+s.writeRatio+0.1 { + // TODO(lnikon): Should be "exists" + // return "delete" + return "delete" + } else { + // TODO(lnikon): Should be "exists" + // return "exists" + return "exists" + } +} + +func (s *CRUDScenario) executeRead(ctx *core.TestContext) error { + key := ctx.GetRandomExistingKey() + + value, err := ctx.KV.Get(ctx.Context(), []byte(key)) + if err != nil { + return err + } + + writtenData := ctx.GetWrittenData() + if expectedValue, exists := writtenData[key]; exists { + if !bytes.Equal(value, expectedValue) { + return fmt.Errorf("data consistency error: expected %v, got %v", expectedValue, value) + } + } + + return nil +} + +func (s *CRUDScenario) executeWrite(ctx *core.TestContext) error { + key := ctx.GenerateKey() + value := ctx.GenerateValue(s.valueSize) + + err := ctx.KV.Put(ctx.Context(), []byte(key), value) + if err == nil { + ctx.RecordWrite(key, value) + } + + return err +} diff --git a/tests/go/kvtest/pkg/scenarios/interface.go b/tests/go/kvtest/pkg/scenarios/interface.go new file mode 100644 index 0000000..80859e9 --- /dev/null +++ b/tests/go/kvtest/pkg/scenarios/interface.go @@ -0,0 +1,38 @@ +package scenarios + +import ( + "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" +) + +// Scenario defines the interface all test scenarios must implement +type Scenario interface { + Execute(ctx *core.TestContext) error + + Name() string + + Setup(ctx *core.TestContext) error + + Teardown(ctx *core.TestContext) error +} + +// WeightedScenario used for load testing +type WeightedScenario struct { + Scenario Scenario + Weight int +} + +// ScenarioFactory creates scenarios based on configuration +type ScenarioFactory struct{} + +func NewScenarioFactory() *ScenarioFactory { + return &ScenarioFactory{} +} + +func (f *ScenarioFactory) Create(config core.ScenarioConfig) (Scenario, error) { + switch config.Type { + case "crud": + return NewCRUDScenario(config.Parameters), nil + default: + return NewCRUDScenario(config.Parameters), nil + } +} diff --git a/tests/go/kvtest/pkg/validators/interface.go b/tests/go/kvtest/pkg/validators/interface.go new file mode 100644 index 0000000..b275806 --- /dev/null +++ b/tests/go/kvtest/pkg/validators/interface.go @@ -0,0 +1,3 @@ +package validators + +import () From b90ae4fc312e99b0d0c812279a34735191c75114 Mon Sep 17 00:00:00 2001 From: lnikon Date: Wed, 27 Aug 2025 01:09:23 +0400 Subject: [PATCH 03/17] Integration & load testing: - Add integration test executor --- tests/go/kvtest/framework/core.go | 1 - tests/go/kvtest/pkg/core/integration.go | 94 +++++++++++++++++++ .../pkg/core/{interface.go => kvstore.go} | 0 tests/go/kvtest/pkg/core/scenario.go | 18 ++++ tests/go/kvtest/pkg/scenarios/factory.go | 21 +++++ tests/go/kvtest/pkg/scenarios/interface.go | 38 -------- 6 files changed, 133 insertions(+), 39 deletions(-) delete mode 100644 tests/go/kvtest/framework/core.go create mode 100644 tests/go/kvtest/pkg/core/integration.go rename tests/go/kvtest/pkg/core/{interface.go => kvstore.go} (100%) create mode 100644 tests/go/kvtest/pkg/core/scenario.go create mode 100644 tests/go/kvtest/pkg/scenarios/factory.go delete mode 100644 tests/go/kvtest/pkg/scenarios/interface.go diff --git a/tests/go/kvtest/framework/core.go b/tests/go/kvtest/framework/core.go deleted file mode 100644 index 83af475..0000000 --- a/tests/go/kvtest/framework/core.go +++ /dev/null @@ -1 +0,0 @@ -package framework diff --git a/tests/go/kvtest/pkg/core/integration.go b/tests/go/kvtest/pkg/core/integration.go new file mode 100644 index 0000000..63729e7 --- /dev/null +++ b/tests/go/kvtest/pkg/core/integration.go @@ -0,0 +1,94 @@ +package core + +import ( + "fmt" + "log" +) + +type IntegrationTestExecutor struct { + kvstore KVStoreInterface + scenarios []Scenario + config *TestConfig +} + +func NewIntegrationTestExecutor(kvstore KVStoreInterface, config *TestConfig) *IntegrationTestExecutor { + return &IntegrationTestExecutor{ + kvstore: kvstore, + config: config, + } +} + +func (e *IntegrationTestExecutor) AddScenario(scenario Scenario) { + e.scenarios = append(e.scenarios, scenario) +} + +func (e *IntegrationTestExecutor) Execute() (*IntegrationTestResult, error) { + result := &IntegrationTestResult{ + Results: make(map[string]*ScenarioResult), + } + + log.Printf("Starting integration tests with %d scenarios", len(e.scenarios)) + + for _, scenario := range e.scenarios { + log.Printf("Running scenario: %s", scenario.Name()) + + ctx := NewTestContext(e.kvstore, e.config) + ctx.TestPhase = "integration" + + scenarioResult := &ScenarioResult{ + Name: scenario.Name(), + } + + // Setup phase + if err := scenario.Setup(ctx); err != nil { + scenarioResult.Error = fmt.Errorf("setup Failed: %w", err) + result.Results[scenario.Name()] = scenarioResult + result.Failed++ + continue + } + + // Execute phase + if err := scenario.Execute(ctx); err != nil { + scenarioResult.Error = fmt.Errorf("setup failed: %w", err) + result.Results[scenario.Name()] = scenarioResult + result.Failed++ + continue + } else { + scenarioResult.Success = true + result.Results[scenario.Name()] = scenarioResult + result.Passed++ + } + + // Teardown phase + if err := scenario.Teardown(ctx); err != nil { + log.Printf("Warning: teardown failed for %s: %v", scenario.Name(), err) + } + + // Stop metrics collection + // TODO(lnikon): Implement MetricsSummary + // ctx.Metrics.Stop() + // scenarioResult.Metrics = ctx.Metrics.MetricsSummary + } + + result.Total = len(e.scenarios) + return result, nil +} + +type IntegrationTestResult struct { + Total int + Passed int + Failed int + Results map[string]*ScenarioResult +} + +type ScenarioResult struct { + Name string + Success bool + Error error + // TODO(lnikon): Implement MetricsSummary + // Metrics *MetricsSummary +} + +func (r *IntegrationTestResult) AllPassed() bool { + return r.Failed == 0 +} diff --git a/tests/go/kvtest/pkg/core/interface.go b/tests/go/kvtest/pkg/core/kvstore.go similarity index 100% rename from tests/go/kvtest/pkg/core/interface.go rename to tests/go/kvtest/pkg/core/kvstore.go diff --git a/tests/go/kvtest/pkg/core/scenario.go b/tests/go/kvtest/pkg/core/scenario.go new file mode 100644 index 0000000..dc1310d --- /dev/null +++ b/tests/go/kvtest/pkg/core/scenario.go @@ -0,0 +1,18 @@ +package core + +// Scenario defines the interface all test scenarios must implement +type Scenario interface { + Execute(ctx *TestContext) error + + Name() string + + Setup(ctx *TestContext) error + + Teardown(ctx *TestContext) error +} + +// WeightedScenario used for load testing +type WeightedScenario struct { + Scenario Scenario + Weight int +} diff --git a/tests/go/kvtest/pkg/scenarios/factory.go b/tests/go/kvtest/pkg/scenarios/factory.go new file mode 100644 index 0000000..5c2dc82 --- /dev/null +++ b/tests/go/kvtest/pkg/scenarios/factory.go @@ -0,0 +1,21 @@ +package scenarios + +import ( + "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" +) + +// ScenarioFactory creates scenarios based on configuration +type ScenarioFactory struct{} + +func NewScenarioFactory() *ScenarioFactory { + return &ScenarioFactory{} +} + +func (f *ScenarioFactory) Create(config core.ScenarioConfig) (core.Scenario, error) { + switch config.Type { + case "crud": + return NewCRUDScenario(config.Parameters), nil + default: + return NewCRUDScenario(config.Parameters), nil + } +} diff --git a/tests/go/kvtest/pkg/scenarios/interface.go b/tests/go/kvtest/pkg/scenarios/interface.go deleted file mode 100644 index 80859e9..0000000 --- a/tests/go/kvtest/pkg/scenarios/interface.go +++ /dev/null @@ -1,38 +0,0 @@ -package scenarios - -import ( - "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" -) - -// Scenario defines the interface all test scenarios must implement -type Scenario interface { - Execute(ctx *core.TestContext) error - - Name() string - - Setup(ctx *core.TestContext) error - - Teardown(ctx *core.TestContext) error -} - -// WeightedScenario used for load testing -type WeightedScenario struct { - Scenario Scenario - Weight int -} - -// ScenarioFactory creates scenarios based on configuration -type ScenarioFactory struct{} - -func NewScenarioFactory() *ScenarioFactory { - return &ScenarioFactory{} -} - -func (f *ScenarioFactory) Create(config core.ScenarioConfig) (Scenario, error) { - switch config.Type { - case "crud": - return NewCRUDScenario(config.Parameters), nil - default: - return NewCRUDScenario(config.Parameters), nil - } -} From ac6b302cf9101460c2214286033c21fffd149d84 Mon Sep 17 00:00:00 2001 From: lnikon Date: Wed, 27 Aug 2025 22:01:19 +0400 Subject: [PATCH 04/17] Integration & load testing: - Add test orchestrator --- tests/go/kvtest/pkg/core/config.go | 2 +- tests/go/kvtest/pkg/core/kvstore.go | 4 +- tests/go/kvtest/pkg/executors/orchestrator.go | 69 +++++++++++++++++++ tests/go/kvtest/pkg/scenarios/factory.go | 4 +- 4 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 tests/go/kvtest/pkg/executors/orchestrator.go diff --git a/tests/go/kvtest/pkg/core/config.go b/tests/go/kvtest/pkg/core/config.go index 63e4a8f..5ab1bb5 100644 --- a/tests/go/kvtest/pkg/core/config.go +++ b/tests/go/kvtest/pkg/core/config.go @@ -56,7 +56,7 @@ type ScenarioConfig struct { Parameters map[string]interface{} `yaml:"parameters"` } -// DefaultConfig() returns a default configuration +// DefaultConfig returns a default configuration func DefaultConfig() *TestConfig { return &TestConfig{ Timeout: 30 * time.Second, diff --git a/tests/go/kvtest/pkg/core/kvstore.go b/tests/go/kvtest/pkg/core/kvstore.go index 0cac6f1..ff752e4 100644 --- a/tests/go/kvtest/pkg/core/kvstore.go +++ b/tests/go/kvtest/pkg/core/kvstore.go @@ -4,7 +4,7 @@ import ( "context" ) -// Configuration for KVStore adapters +// Config for KVStore adapters type Config interface{} type KVStoreInterface interface { @@ -21,7 +21,7 @@ type KVStoreInterface interface { Name() string } -// Common errors +// KVError represents kvtest specific error type KVError struct { Op string Err error diff --git a/tests/go/kvtest/pkg/executors/orchestrator.go b/tests/go/kvtest/pkg/executors/orchestrator.go new file mode 100644 index 0000000..aabba8e --- /dev/null +++ b/tests/go/kvtest/pkg/executors/orchestrator.go @@ -0,0 +1,69 @@ +package executors + +import ( + "fmt" + "log" + + "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" + "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/scenarios" +) + +type TestOrchestrator struct { + kvstore core.KVStoreInterface + config *core.TestConfig + factory *scenarios.ScenarioFactory +} + +func NewTestOrchestrator(kvstore core.KVStoreInterface, config *core.TestConfig) *TestOrchestrator { + return &TestOrchestrator{ + kvstore: kvstore, + config: config, + factory: scenarios.NewScenarioFactory(), + } +} + +func (o *TestOrchestrator) RunComplete() (*CompleteTestResult, error) { + result := &CompleteTestResult{} + + if o.config.Integration.Enabled { + log.Println("=== Phase 1: Integration Testing ===") + + integrationResult, err := o.runIntegrationTests() + if err != nil { + return nil, fmt.Errorf("integration tests failed: %w", err) + } + + result.Integration = integrationResult + + if !integrationResult.AllPassed() { + log.Printf("integration tests failed (%d/%d passed)", + result.Integration.Passed, result.Integration.Total) + return result, nil + } + + log.Printf("integration tests completed successfully (%d/%d passed)", + result.Integration.Passed, result.Integration.Total) + } + + return result, nil +} + +func (o *TestOrchestrator) runIntegrationTests() (*core.IntegrationTestResult, error) { + executor := core.NewIntegrationTestExecutor(o.kvstore, o.config) + + for _, scenariConfig := range o.config.Scenarios { + if scenariConfig.Enabled { + scenario, err := o.factory.Create(scenariConfig) + if err != nil { + return nil, fmt.Errorf("failed to create scenario %s: %w", scenariConfig.Name, err) + } + executor.AddScenario(scenario) + } + } + + return executor.Execute() +} + +type CompleteTestResult struct { + Integration *core.IntegrationTestResult +} diff --git a/tests/go/kvtest/pkg/scenarios/factory.go b/tests/go/kvtest/pkg/scenarios/factory.go index 5c2dc82..fc5110e 100644 --- a/tests/go/kvtest/pkg/scenarios/factory.go +++ b/tests/go/kvtest/pkg/scenarios/factory.go @@ -1,8 +1,6 @@ package scenarios -import ( - "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" -) +import "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" // ScenarioFactory creates scenarios based on configuration type ScenarioFactory struct{} From a5ab9325ac815e90998a25e5e1f47a486f3ce945 Mon Sep 17 00:00:00 2001 From: lnikon Date: Sat, 13 Sep 2025 20:40:53 +0400 Subject: [PATCH 05/17] Integration & load testing: - Implement simple CRUD testing via kvtest - Fix memtables not being flushed --- .gitignore | 4 + lib/db/db.cpp | 8 +- lib/structures/lsmtree/levels/level.cpp | 28 +-- lib/structures/lsmtree/levels/levels.cpp | 1 + lib/structures/lsmtree/lsmtree.cpp | 15 +- lib/structures/lsmtree/lsmtree.h | 6 +- src/config.h | 4 + src/main.cpp | 8 +- tests/go/kvtest/LICENSE | 0 tests/go/kvtest/cmd/kvtest/kvtest.yml | 23 +++ tests/go/kvtest/cmd/kvtest/main.go | 210 ++++++++++++++++++++++- tests/go/kvtest/pkg/adapters/tinykvpp.go | 4 +- tests/go/kvtest/pkg/config/loader.go | 84 +++++++++ tests/go/kvtest/pkg/core/integration.go | 2 +- tests/go/kvtest/pkg/scenarios/crud.go | 4 + 15 files changed, 366 insertions(+), 35 deletions(-) create mode 100644 tests/go/kvtest/LICENSE create mode 100644 tests/go/kvtest/cmd/kvtest/kvtest.yml create mode 100644 tests/go/kvtest/pkg/config/loader.go diff --git a/.gitignore b/.gitignore index a3adc5f..82a98fc 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ *.exe *.out *.app +./tests/go/kvtest/cmd/kvtest/kvtest # Project-specific *.bak @@ -46,3 +47,6 @@ build var tmp current + +# Compiled Protobufs +*.pb.* diff --git a/lib/db/db.cpp b/lib/db/db.cpp index 649f9a5..8e609f8 100644 --- a/lib/db/db.cpp +++ b/lib/db/db.cpp @@ -40,7 +40,9 @@ auto serializeOperation(const db::client_request_t &request) -> std::string } default: { - spdlog::critical("Unknown operation type: {}", magic_enum::enum_name(request.type)); + spdlog::critical( + "(serializeOperation): Unknown operation type: {}", magic_enum::enum_name(request.type) + ); break; } } @@ -398,7 +400,9 @@ auto db_t::onRaftCommit(const raft::v1::LogEntry &entry) -> bool } default: { - spdlog::critical("Unknown operation type: {}", magic_enum::enum_name(op.type())); + spdlog::critical( + "(onRaftCommit): Unknown operation type: {}", magic_enum::enum_name(op.type()) + ); return false; } } diff --git a/lib/structures/lsmtree/levels/level.cpp b/lib/structures/lsmtree/levels/level.cpp index bb72f1e..9a6e3c3 100644 --- a/lib/structures/lsmtree/levels/level.cpp +++ b/lib/structures/lsmtree/levels/level.cpp @@ -119,11 +119,17 @@ auto level_t::compact() const noexcept -> segments::regular_segment::shared_ptr_ const auto bytesUsedForLevel{bytes_used()}; // If level size hasn't reached the size limit then skip the compaction - if ((index() == 0 && - bytesUsedForLevel < m_pConfig->LSMTreeConfig.LevelZeroCompactionThreshold) || - (index() != 0 && - bytesUsedForLevel < - m_pConfig->LSMTreeConfig.LevelNonZeroCompactionThreshold * std::pow(10, index()))) + const bool isZeroLevel{index() == 0}; + const auto zeroLevelThreshold{m_pConfig->LSMTreeConfig.LevelZeroCompactionThreshold}; + if (isZeroLevel && bytesUsedForLevel < zeroLevelThreshold) + { + return nullptr; + } + + const auto nonZeroLevelThreshold{ + m_pConfig->LSMTreeConfig.LevelNonZeroCompactionThreshold * std::pow(10, index()) + }; + if (!isZeroLevel && bytesUsedForLevel < nonZeroLevelThreshold) { return nullptr; } @@ -135,13 +141,13 @@ auto level_t::compact() const noexcept -> segments::regular_segment::shared_ptr_ std::vector>, - IteratorCompare> + IteratorCompare> minHeap; for (const auto &segment : m_storage) { - // TODO(lnikon): reset memtable inside regular_segment_t at the of the - // flush() and recover it here e.g. segment->recover_memtable(); + // TODO(lnikon): reset memtable inside regular_segment_t at the of the flush() and recover + // it here e.g. segment->recover_memtable(); const auto ¤tMemtable = segment->memtable().value(); minHeap.emplace(currentMemtable.begin(), currentMemtable.end()); } @@ -154,8 +160,7 @@ auto level_t::compact() const noexcept -> segments::regular_segment::shared_ptr_ minHeap.pop(); // Add the smallest element to the merged sequence. - // If two elements have the same key, then choose the one with the - // greatest timestamp + // If two elements have the same key, then choose the one with the greatest timestamp if (mergedMemtable.empty() || lastKey != current.first->m_key) { mergedMemtable.emplace(*current.first); @@ -171,8 +176,7 @@ auto level_t::compact() const noexcept -> segments::regular_segment::shared_ptr_ } // Create a new segment from the compacted segment. - // The postfix "_compacted" signals that the segment is an intermediate - // result + // The postfix "_compacted" signals that the segment is an intermediate result auto name{fmt::format("{}_{}_compacted", helpers::segment_name(), index())}; return segments::factories::lsmtree_segment_factory( name, helpers::segment_path(m_pConfig->datadir_path(), name), mergedMemtable diff --git a/lib/structures/lsmtree/levels/levels.cpp b/lib/structures/lsmtree/levels/levels.cpp index 4b8585a..de9c4ab 100644 --- a/lib/structures/lsmtree/levels/levels.cpp +++ b/lib/structures/lsmtree/levels/levels.cpp @@ -183,6 +183,7 @@ auto levels_t::size() const noexcept -> levels_t::levels_storage_t::size_type [[nodiscard]] auto levels_t::flush_to_level0(memtable::memtable_t memtable) const noexcept -> segments::regular_segment::shared_ptr_t { + absl::MutexLock lock{&m_mutex}; assert(m_levels[0]); diff --git a/lib/structures/lsmtree/lsmtree.cpp b/lib/structures/lsmtree/lsmtree.cpp index e9a5060..9d81c82 100644 --- a/lib/structures/lsmtree/lsmtree.cpp +++ b/lib/structures/lsmtree/lsmtree.cpp @@ -127,9 +127,9 @@ void lsmtree_t::memtable_flush_task(std::stop_token stoken) noexcept // TODO: Is it possible to do the flushing async? while (true) { + // Flush the remaining memtables on stop request and return if (stoken.stop_requested()) { - // Flush the remaining memtables on stop request spdlog::debug( "Flushing remaining memtables on stop " "request. queue.size={}", @@ -144,7 +144,9 @@ void lsmtree_t::memtable_flush_task(std::stop_token stoken) noexcept // TODO: Assert will crash the program, maybe we // should return an error code? - assert(m_pLevels->flush_to_level0(std::move(memtable))); + absl::WriterMutexLock lock{&m_mutex}; + bool ok{m_pLevels->flush_to_level0(std::move(memtable))}; + assert(ok); } return; } @@ -162,7 +164,8 @@ void lsmtree_t::memtable_flush_task(std::stop_token stoken) noexcept // TODO: Assert will crash the program, maybe we should // return an error code? absl::WriterMutexLock lock{&m_mutex}; - assert(m_pLevels->flush_to_level0(std::move(memtable.value()))); + bool ok{m_pLevels->flush_to_level0(std::move(memtable.value()))}; + assert(ok); } } } @@ -218,7 +221,7 @@ auto lsmtree_builder_t::build( ); } -auto lsmtree_builder_t::build_memtable_from_wal(wal_t pWal) const noexcept +auto lsmtree_builder_t::build_memtable_from_wal(wal_t pWal) noexcept -> std::optional { memtable::memtable_t table; @@ -248,7 +251,7 @@ auto lsmtree_builder_t::build_memtable_from_wal(wal_t pWal) const noexcept } default: { - spdlog::error("Unkown WAL operation {}", std::to_underlying(op.type())); + spdlog::error("Unknown WAL operation {}", std::to_underlying(op.type())); break; } }; @@ -259,7 +262,7 @@ auto lsmtree_builder_t::build_memtable_from_wal(wal_t pWal) const noexcept auto lsmtree_builder_t::build_levels_from_manifest( config::shared_ptr_t pConfig, db::manifest::shared_ptr_t pManifest -) const noexcept -> std::optional> +) noexcept -> std::optional> { assert(pManifest); diff --git a/lib/structures/lsmtree/lsmtree.h b/lib/structures/lsmtree/lsmtree.h index 86851ed..935ee75 100644 --- a/lib/structures/lsmtree/lsmtree.h +++ b/lib/structures/lsmtree/lsmtree.h @@ -88,12 +88,12 @@ struct lsmtree_builder_t final -> std::shared_ptr; private: - [[nodiscard]] auto build_memtable_from_wal(wal_t pWal) const noexcept + [[nodiscard]] static auto build_memtable_from_wal(wal_t pWal) noexcept -> std::optional; - [[nodiscard]] auto build_levels_from_manifest( + [[nodiscard]] static auto build_levels_from_manifest( config::shared_ptr_t pConfig, db::manifest::shared_ptr_t pManifest - ) const noexcept -> std::optional>; + ) noexcept -> std::optional>; }; } // namespace structures::lsmtree diff --git a/src/config.h b/src/config.h index bd2bfd5..e76d20f 100644 --- a/src/config.h +++ b/src/config.h @@ -10,15 +10,19 @@ using nlohmann::json_schema::json_validator; using json = nlohmann::json; auto loadConfigJson(const std::string &configPath) -> json; + void validateConfigJson(const json &configJson); void configureLogging(const std::string &loggingLevel); auto loadDatabaseConfig(const json &configJson) -> config::shared_ptr_t; + void loadWALConfig(const json &walConfig, config::shared_ptr_t dbConfig); + void loadLSMTreeConfig( const json &lsmtreeConfig, config::shared_ptr_t dbConfig, const std::string &configPath ); + auto loadServerConfig(const json &configJson, config::shared_ptr_t dbConfig); auto initializeDatabaseConfig(const json &configJson, const std::string &configPath) diff --git a/src/main.cpp b/src/main.cpp index d27a3c2..4ce0814 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -53,7 +53,7 @@ namespace { if (pConfig->ServerConfig.id == 0) { - spdlog::error("maybe_create_consensus_module: ID of the node should be positve integer"); + spdlog::error("maybe_create_consensus_module: ID of the node should be a positive integer"); return std::nullopt; } @@ -75,13 +75,7 @@ namespace replicas.emplace_back( consensus::node_config_t{.m_id = replicaId, .m_ip = replicaIp}, std::move(stub) ); - spdlog::info( - "maybe_create_consensus_module: Emplacing replica into replicas " - "vector. replicaId={} replicaIp={}", - replicaId, - replicaIp - ); } ++replicaId; } diff --git a/tests/go/kvtest/LICENSE b/tests/go/kvtest/LICENSE new file mode 100644 index 0000000..e69de29 diff --git a/tests/go/kvtest/cmd/kvtest/kvtest.yml b/tests/go/kvtest/cmd/kvtest/kvtest.yml new file mode 100644 index 0000000..5f681b2 --- /dev/null +++ b/tests/go/kvtest/cmd/kvtest/kvtest.yml @@ -0,0 +1,23 @@ +timeout: 30s +log_level: info + +adapter: + type: tinykvpp + address: localhost:9891 + connection_timeout: 5s + request_timeout: 1s + max_retries: 3 + +integration: + enabled: true + validations: + - consistency + +scenarios: + - name: basic_crud + type: crud + enabled: true + parameters: + read_ratio: 0.7 + write_ratio: 0.3 + value_size: 5 diff --git a/tests/go/kvtest/cmd/kvtest/main.go b/tests/go/kvtest/cmd/kvtest/main.go index e5ad8c4..8bd4fdd 100644 --- a/tests/go/kvtest/cmd/kvtest/main.go +++ b/tests/go/kvtest/cmd/kvtest/main.go @@ -1,5 +1,211 @@ package main -import () +import ( + "fmt" + "log" + "os" -func main() {} + "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/adapters" + "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/config" + "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" + "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/scenarios" + "github.com/spf13/cobra" +) + +var ( + configFile string + adapterType string + address string + verbose bool +) + +func main() { + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) + } +} + +var rootCmd = &cobra.Command{ + Use: "kvtest", + Short: "A simple testing framework for key-value stores", + Long: `KVTest is a simple testing framework for key-value stores that supports +integration testing with basic CRUD operations.`, + Version: "0.0.1", +} + +var runCmd = &cobra.Command{ + Use: "run", + Short: "Run integration tests", + RunE: runTests, +} + +var validateCmd = &cobra.Command{ + Use: "validate", + Short: "Validate configuration", + RunE: validateConfig, +} + +var generateConfigCmd = &cobra.Command{ + Use: "generate [output-file]", + Short: "Generate sample configration", + Args: cobra.MaximumNArgs(1), + RunE: generateConfig, +} + +func init() { + // Global flags + rootCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "Configuration file path") + rootCmd.PersistentFlags().StringVarP(&adapterType, "adapter", "a", "tinykvpp", "KV store adapter type") + rootCmd.PersistentFlags().StringVar(&address, "address", "localhost:9891", "KV store address") + rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Enable verbose logging") + + // Add subcommands + rootCmd.AddCommand(runCmd) + rootCmd.AddCommand(validateCmd) + rootCmd.AddCommand(generateConfigCmd) +} + +func runTests(cmd *cobra.Command, args []string) error { + // Load configuration + cfg, err := loadConfiguration() + if err != nil { + return fmt.Errorf("failed to load configuration: %w", err) + } + + // Override with command line flags if provided + if adapterType != "tinykvpp" { + cfg.Adapter.Type = adapterType + } + if address != "localhost:9891" { + cfg.Adapter.Address = address + } + + // Validate configuration + if err := config.Validate(cfg); err != nil { + return fmt.Errorf("configuration validation failed: %w", err) + } + + // Create KV store adapter + kvstore, err := createAdapter(cfg) + if err != nil { + return fmt.Errorf("failed to create adapter: %w", err) + } + defer kvstore.Close() + + // Connect to the KV store + if err := kvstore.Connect(createAdapterConfig(cfg)); err != nil { + return fmt.Errorf("failed to connect to KV store: %w", err) + } + + // Create scenario factory + factory := scenarios.NewScenarioFactory() + + // Create integration test executor + executor := core.NewIntegrationTestExecutor(kvstore, cfg) + + // Add scenarios from configuration + for _, scenarioCfg := range cfg.Scenarios { + if !scenarioCfg.Enabled { + continue + } + + scenario, err := factory.Create(scenarioCfg) + if err != nil { + return fmt.Errorf("failed to create scenario %s: %w", scenarioCfg.Name, err) + } + + executor.AddScenario(scenario) + } + + // Execute tests + log.Printf("Starting integration testing...") + result, err := executor.Execute() + if err != nil { + return fmt.Errorf("test execution failed: %w", err) + } + + // Print results + printResults(result) + + if !result.AllPassed() { + os.Exit(1) + } + + return nil +} + +func validateConfig(cmd *cobra.Command, args []string) error { return nil } + +func generateConfig(cmd *cobra.Command, args []string) error { return nil } + +func loadConfiguration() (*core.TestConfig, error) { + if configFile != "" { + return config.LoadFromFile(configFile) + } + + // Try to load from default locations + defaultFiles := []string{"kvtest.yaml", "kvtest.yml", ".kvtest.yaml"} + + for _, file := range defaultFiles { + if _, err := os.Stat(file); err == nil { + return config.LoadFromFile(file) + } + } + + // Return default configuration if no file found + return core.DefaultConfig(), nil +} + +func createAdapter(cfg *core.TestConfig) (core.KVStoreInterface, error) { + switch cfg.Adapter.Type { + case "tinykvpp": + return adapters.NewTinyKVPPAdapter(), nil + default: + return nil, fmt.Errorf("unsupported adapter type: %s", cfg.Adapter.Type) + } +} + +func createAdapterConfig(cfg *core.TestConfig) core.Config { + switch cfg.Adapter.Type { + case "tinykvpp": + return adapters.TinyKVPPConfig{ + Address: cfg.Adapter.Address, + ConnectionTimeout: cfg.Adapter.ConnectionTimeout, + RequestTimeout: cfg.Adapter.RequestTimeout, + MaxRetries: cfg.Adapter.MaxRetries, + } + default: + return nil + } +} + +func printResults(result *core.IntegrationTestResult) { + fmt.Printf("\n=== Integration Test Results ===\n") + fmt.Printf("Total scenarios: %d\n", result.Total) + fmt.Printf("Passed: %d\n", result.Passed) + fmt.Printf("Failed: %d\n", result.Failed) + fmt.Printf("Success rate: %.1f%%\n", float64(result.Passed)/float64(result.Total)*100) + + if len(result.Results) > 0 { + fmt.Printf("\n=== Scenario Details ===\n") + for name, scenarioResult := range result.Results { + status := "PASS" + if !scenarioResult.Success { + status = "FAIL" + } + + fmt.Printf("[%s] %s", status, name) + if scenarioResult.Error != nil { + fmt.Printf(" - %v", scenarioResult.Error) + } + fmt.Println() + } + } + + fmt.Printf("\n=== Summary ===\n") + if result.AllPassed() { + fmt.Println("✅ All tests passed!") + } else { + fmt.Printf("❌ %d test(s) failed\n", result.Failed) + } +} diff --git a/tests/go/kvtest/pkg/adapters/tinykvpp.go b/tests/go/kvtest/pkg/adapters/tinykvpp.go index f075412..e71528e 100644 --- a/tests/go/kvtest/pkg/adapters/tinykvpp.go +++ b/tests/go/kvtest/pkg/adapters/tinykvpp.go @@ -27,7 +27,7 @@ type TinyKVPPAdapter struct { var _ core.KVStoreInterface = (*TinyKVPPAdapter)(nil) -func NewTinyKVPPAdapter(config core.Config) *TinyKVPPAdapter { +func NewTinyKVPPAdapter() *TinyKVPPAdapter { return &TinyKVPPAdapter{} } @@ -92,7 +92,7 @@ func (a *TinyKVPPAdapter) Put(ctx context.Context, key, value []byte) error { req := &pb.PutRequest{Key: string(key), Value: string(value)} resp, err := a.client.Put(ctx, req) if err != nil { - return fmt.Errorf("grpc put failed for key %s: %v\n", key, err) + return fmt.Errorf("grpc put failed for key %s: %v", key, err) } if len(resp.Error) > 0 { diff --git a/tests/go/kvtest/pkg/config/loader.go b/tests/go/kvtest/pkg/config/loader.go new file mode 100644 index 0000000..3e70c62 --- /dev/null +++ b/tests/go/kvtest/pkg/config/loader.go @@ -0,0 +1,84 @@ +package config + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v3" + + "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" +) + +// LoadFromFile loads configuration from a YAML file +func LoadFromFile(filename string) (*core.TestConfig, error) { + data, err := os.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + config := core.DefaultConfig() + if err := yaml.Unmarshal(data, config); err != nil { + return nil, fmt.Errorf("failed to parse config file: %w", err) + } + + return config, nil +} + +// SaveToFile saves configuration to a YAML file +func SaveToFile(config *core.TestConfig, filename string) error { + data, err := yaml.Marshal(config) + if err != nil { + return fmt.Errorf("failed to marshal config: %w", err) + } + + if err := os.WriteFile(filename, data, 0644); err != nil { + return fmt.Errorf("failed to write config file: %w", err) + } + + return nil +} + +func Validate(config *core.TestConfig) error { + // Validate adapter configuration + if config.Adapter.Type == "" { + return fmt.Errorf("adapter.type is required") + } + + if config.Adapter.Address == "" { + return fmt.Errorf("adapter.address is required") + } + + if config.Adapter.ConnectionTimeout <= 0 { + return fmt.Errorf("adapter.connection_timeout must be positive") + } + + if config.Adapter.RequestTimeout <= 0 { + return fmt.Errorf("adapter.request_timeout must be positive") + } + + // Validate scenarios + if len(config.Scenarios) == 0 { + return fmt.Errorf("at least one scenario must be present") + } + + enabledScenarios := 0 + for i, scenario := range config.Scenarios { + if scenario.Name == "" { + return fmt.Errorf("scenario[%d].name is required", i) + } + + if scenario.Type == "" { + return fmt.Errorf("scenario[%d].type is required", i) + } + + if scenario.Enabled { + enabledScenarios++ + } + } + + if enabledScenarios == 0 { + return fmt.Errorf("at least one scenario must be enabled") + } + + return nil +} diff --git a/tests/go/kvtest/pkg/core/integration.go b/tests/go/kvtest/pkg/core/integration.go index 63729e7..9ea7644 100644 --- a/tests/go/kvtest/pkg/core/integration.go +++ b/tests/go/kvtest/pkg/core/integration.go @@ -49,7 +49,7 @@ func (e *IntegrationTestExecutor) Execute() (*IntegrationTestResult, error) { // Execute phase if err := scenario.Execute(ctx); err != nil { - scenarioResult.Error = fmt.Errorf("setup failed: %w", err) + scenarioResult.Error = fmt.Errorf("execute failed: %w", err) result.Results[scenario.Name()] = scenarioResult result.Failed++ continue diff --git a/tests/go/kvtest/pkg/scenarios/crud.go b/tests/go/kvtest/pkg/scenarios/crud.go index e073f50..fb9da12 100644 --- a/tests/go/kvtest/pkg/scenarios/crud.go +++ b/tests/go/kvtest/pkg/scenarios/crud.go @@ -3,7 +3,9 @@ package scenarios import ( "bytes" "fmt" + "log" "math/rand" + // "time" "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" @@ -108,6 +110,8 @@ func (s *CRUDScenario) selectOperation() string { func (s *CRUDScenario) executeRead(ctx *core.TestContext) error { key := ctx.GetRandomExistingKey() + log.Printf("requesting existing key: %s", key) + value, err := ctx.KV.Get(ctx.Context(), []byte(key)) if err != nil { return err From 5ca69a2f9b9fe7ec3238a84dd0bf635570ad98d2 Mon Sep 17 00:00:00 2001 From: lnikon Date: Sat, 13 Sep 2025 22:20:00 +0400 Subject: [PATCH 06/17] Integration & load testing: - Leftover cleanup --- tests/go/kvtest/cmd/root.go | 46 ------------------------------------- tests/go/kvtest/main.go | 11 --------- 2 files changed, 57 deletions(-) delete mode 100644 tests/go/kvtest/cmd/root.go delete mode 100644 tests/go/kvtest/main.go diff --git a/tests/go/kvtest/cmd/root.go b/tests/go/kvtest/cmd/root.go deleted file mode 100644 index 4e2dae0..0000000 --- a/tests/go/kvtest/cmd/root.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright © 2025 NAME HERE -*/ -package cmd - -import ( - "os" - - "github.com/spf13/cobra" -) - -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: "kvtest", - Short: "A brief description of your application", - Long: `A longer description that spans multiple lines and likely contains -examples and usage of using your application. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - // Uncomment the following line if your bare application - // has an action associated with it: - // Run: func(cmd *cobra.Command, args []string) { }, -} - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - err := rootCmd.Execute() - if err != nil { - os.Exit(1) - } -} - -func init() { - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - - // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.kvtest.yaml)") - - // Cobra also supports local flags, which will only run - // when this action is called directly. - rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") -} diff --git a/tests/go/kvtest/main.go b/tests/go/kvtest/main.go deleted file mode 100644 index f57034e..0000000 --- a/tests/go/kvtest/main.go +++ /dev/null @@ -1,11 +0,0 @@ -/* -Copyright © 2025 NAME HERE - -*/ -package main - -import "github.com/lnikon/tinykvpp/tests/go/kvtest/cmd" - -func main() { - cmd.Execute() -} From 053e60ed8d9b3df4eff320dbc3d00cfa6dd945b3 Mon Sep 17 00:00:00 2001 From: lnikon Date: Sat, 13 Sep 2025 22:20:56 +0400 Subject: [PATCH 07/17] Integration & load testing: - Ignore binaries --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 82a98fc..08fbf23 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ *.out *.app ./tests/go/kvtest/cmd/kvtest/kvtest +./tests/go/kvtest/kvtest # Project-specific *.bak From e30c4c5214bd61375e6ecfb021a03de167a842d6 Mon Sep 17 00:00:00 2001 From: lnikon Date: Sat, 20 Sep 2025 13:54:29 +0400 Subject: [PATCH 08/17] Integration & load testing: - ReadYourWrites scenario in-progress --- infra/assets/tkvpp_config_standalone.json | 2 +- run-clang-tidy.py | 544 ++++++++++++---------- src/main.cpp | 1 - tests/go/kvtest/pkg/scenarios/crud.go | 2 + tests/go/kvtest/pkg/scenarios/factory.go | 2 + tests/go/kvtest/pkg/scenarios/wr.go | 59 +++ 6 files changed, 366 insertions(+), 244 deletions(-) create mode 100644 tests/go/kvtest/pkg/scenarios/wr.go diff --git a/infra/assets/tkvpp_config_standalone.json b/infra/assets/tkvpp_config_standalone.json index 01c6102..2693b3b 100644 --- a/infra/assets/tkvpp_config_standalone.json +++ b/infra/assets/tkvpp_config_standalone.json @@ -14,7 +14,7 @@ "storageType": "persistent" }, "lsm": { - "flushThreshold": 1024, + "flushThreshold": 128, "maximumLevels": 12, "levelZeroCompaction": { "compactionStrategy": "levelled", diff --git a/run-clang-tidy.py b/run-clang-tidy.py index a1c9e5c..449ba3c 100755 --- a/run-clang-tidy.py +++ b/run-clang-tidy.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 # -#===- run-clang-tidy.py - Parallel clang-tidy runner ---------*- python -*--===# +# ===- run-clang-tidy.py - Parallel clang-tidy runner ---------*- python -*--===# # # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # -#===------------------------------------------------------------------------===# +# ===------------------------------------------------------------------------===# # FIXME: Integrate with clang-tidy-diff.py """ @@ -49,278 +49,338 @@ import traceback try: - import yaml + import yaml except ImportError: - yaml = None + yaml = None -is_py2 = sys.version[0] == '2' +is_py2 = sys.version[0] == "2" if is_py2: import Queue as queue else: import queue as queue + def find_compilation_database(path): - """Adjusts the directory until a compilation database is found.""" - result = './' - while not os.path.isfile(os.path.join(result, path)): - if os.path.realpath(result) == '/': - print('Error: could not find compilation database.') - sys.exit(1) - result += '../' - return os.path.realpath(result) + """Adjusts the directory until a compilation database is found.""" + result = "./" + while not os.path.isfile(os.path.join(result, path)): + if os.path.realpath(result) == "/": + print("Error: could not find compilation database.") + sys.exit(1) + result += "../" + return os.path.realpath(result) def make_absolute(f, directory): - if os.path.isabs(f): - return f - return os.path.normpath(os.path.join(directory, f)) - - -def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path, - header_filter, extra_arg, extra_arg_before, quiet, - config): - """Gets a command line for clang-tidy.""" - start = [clang_tidy_binary] - if header_filter is not None: - start.append('-header-filter=' + header_filter) - if checks: - start.append('-checks=' + checks) - if tmpdir is not None: - start.append('-export-fixes') - # Get a temporary file. We immediately close the handle so clang-tidy can - # overwrite it. - (handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir) - os.close(handle) - start.append(name) - for arg in extra_arg: - start.append('-extra-arg=%s' % arg) - for arg in extra_arg_before: - start.append('-extra-arg-before=%s' % arg) - start.append('-p=' + build_path) - if quiet: - start.append('-quiet') - if config: - start.append('-config=' + config) - start.append(f) - return start + if os.path.isabs(f): + return f + return os.path.normpath(os.path.join(directory, f)) + + +def get_tidy_invocation( + f, + clang_tidy_binary, + checks, + tmpdir, + build_path, + header_filter, + extra_arg, + extra_arg_before, + quiet, + config, +): + """Gets a command line for clang-tidy.""" + start = [clang_tidy_binary] + if header_filter is not None: + start.append("-header-filter=" + header_filter) + if checks: + start.append("-checks=" + checks) + if tmpdir is not None: + start.append("-export-fixes") + # Get a temporary file. We immediately close the handle so clang-tidy can + # overwrite it. + (handle, name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir) + os.close(handle) + start.append(name) + for arg in extra_arg: + start.append("-extra-arg=%s" % arg) + for arg in extra_arg_before: + start.append("-extra-arg-before=%s" % arg) + start.append("-p=" + build_path) + if quiet: + start.append("-quiet") + if config: + start.append("-config=" + config) + start.append(f) + return start def merge_replacement_files(tmpdir, mergefile): - """Merge all replacement files in a directory into a single file""" - # The fixes suggested by clang-tidy >= 4.0.0 are given under - # the top level key 'Diagnostics' in the output yaml files - mergekey="Diagnostics" - merged=[] - for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')): - content = yaml.safe_load(open(replacefile, 'r')) - if not content: - continue # Skip empty files. - merged.extend(content.get(mergekey, [])) - - if merged: - # MainSourceFile: The key is required by the definition inside - # include/clang/Tooling/ReplacementsYaml.h, but the value - # is actually never used inside clang-apply-replacements, - # so we set it to '' here. - output = { 'MainSourceFile': '', mergekey: merged } - with open(mergefile, 'w') as out: - yaml.safe_dump(output, out) - else: - # Empty the file: - open(mergefile, 'w').close() + """Merge all replacement files in a directory into a single file""" + # The fixes suggested by clang-tidy >= 4.0.0 are given under + # the top level key 'Diagnostics' in the output yaml files + mergekey = "Diagnostics" + merged = [] + for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")): + content = yaml.safe_load(open(replacefile, "r")) + if not content: + continue # Skip empty files. + merged.extend(content.get(mergekey, [])) + + if merged: + # MainSourceFile: The key is required by the definition inside + # include/clang/Tooling/ReplacementsYaml.h, but the value + # is actually never used inside clang-apply-replacements, + # so we set it to '' here. + output = {"MainSourceFile": "", mergekey: merged} + with open(mergefile, "w") as out: + yaml.safe_dump(output, out) + else: + # Empty the file: + open(mergefile, "w").close() def check_clang_apply_replacements_binary(args): - """Checks if invoking supplied clang-apply-replacements binary works.""" - try: - subprocess.check_call([args.clang_apply_replacements_binary, '--version']) - except: - print('Unable to run clang-apply-replacements. Is clang-apply-replacements ' - 'binary correctly specified?', file=sys.stderr) - traceback.print_exc() - sys.exit(1) + """Checks if invoking supplied clang-apply-replacements binary works.""" + try: + subprocess.check_call([args.clang_apply_replacements_binary, "--version"]) + except: + print( + "Unable to run clang-apply-replacements. Is clang-apply-replacements " + "binary correctly specified?", + file=sys.stderr, + ) + traceback.print_exc() + sys.exit(1) def apply_fixes(args, tmpdir): - """Calls clang-apply-fixes on a given directory.""" - invocation = [args.clang_apply_replacements_binary] - if args.format: - invocation.append('-format') - if args.style: - invocation.append('-style=' + args.style) - invocation.append(tmpdir) - subprocess.call(invocation) + """Calls clang-apply-fixes on a given directory.""" + invocation = [args.clang_apply_replacements_binary] + if args.format: + invocation.append("-format") + if args.style: + invocation.append("-style=" + args.style) + invocation.append(tmpdir) + subprocess.call(invocation) def run_tidy(args, tmpdir, build_path, queue, lock, failed_files): - """Takes filenames out of queue and runs clang-tidy on them.""" - while True: - name = queue.get() - invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks, - tmpdir, build_path, args.header_filter, - args.extra_arg, args.extra_arg_before, - args.quiet, args.config) - - proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, err = proc.communicate() - if proc.returncode != 0: - failed_files.append(name) - with lock: - sys.stdout.write(' '.join(invocation) + '\n' + output.decode('utf-8')) - if len(err) > 0: - sys.stdout.flush() - sys.stderr.write(err.decode('utf-8')) - queue.task_done() + """Takes filenames out of queue and runs clang-tidy on them.""" + while True: + name = queue.get() + invocation = get_tidy_invocation( + name, + args.clang_tidy_binary, + args.checks, + tmpdir, + build_path, + args.header_filter, + args.extra_arg, + args.extra_arg_before, + args.quiet, + args.config, + ) + + proc = subprocess.Popen( + invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + output, err = proc.communicate() + if proc.returncode != 0: + failed_files.append(name) + with lock: + sys.stdout.write(" ".join(invocation) + "\n" + output.decode("utf-8")) + if len(err) > 0: + sys.stdout.flush() + sys.stderr.write(err.decode("utf-8")) + queue.task_done() def main(): - parser = argparse.ArgumentParser(description='Runs clang-tidy over all files ' - 'in a compilation database. Requires ' - 'clang-tidy and clang-apply-replacements in ' - '$PATH.') - parser.add_argument('-clang-tidy-binary', metavar='PATH', - default='clang-tidy', - help='path to clang-tidy binary') - parser.add_argument('-clang-apply-replacements-binary', metavar='PATH', - default='clang-apply-replacements', - help='path to clang-apply-replacements binary') - parser.add_argument('-checks', default=None, - help='checks filter, when not specified, use clang-tidy ' - 'default') - parser.add_argument('-config', default=None, - help='Specifies a configuration in YAML/JSON format: ' - ' -config="{Checks: \'*\', ' - ' CheckOptions: [{key: x, ' - ' value: y}]}" ' - 'When the value is empty, clang-tidy will ' - 'attempt to find a file named .clang-tidy for ' - 'each source file in its parent directories.') - parser.add_argument('-header-filter', default=None, - help='regular expression matching the names of the ' - 'headers to output diagnostics from. Diagnostics from ' - 'the main file of each translation unit are always ' - 'displayed.') - if yaml: - parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes', - help='Create a yaml file to store suggested fixes in, ' - 'which can be applied with clang-apply-replacements.') - parser.add_argument('-j', type=int, default=0, - help='number of tidy instances to be run in parallel.') - parser.add_argument('files', nargs='*', default=['.*'], - help='files to be processed (regex on path)') - parser.add_argument('-fix', action='store_true', help='apply fix-its') - parser.add_argument('-format', action='store_true', help='Reformat code ' - 'after applying fixes') - parser.add_argument('-style', default='file', help='The style of reformat ' - 'code after applying fixes') - parser.add_argument('-p', dest='build_path', - help='Path used to read a compile command database.') - parser.add_argument('-extra-arg', dest='extra_arg', - action='append', default=[], - help='Additional argument to append to the compiler ' - 'command line.') - parser.add_argument('-extra-arg-before', dest='extra_arg_before', - action='append', default=[], - help='Additional argument to prepend to the compiler ' - 'command line.') - parser.add_argument('-quiet', action='store_true', - help='Run clang-tidy in quiet mode') - args = parser.parse_args() - - db_path = 'compile_commands.json' - - if args.build_path is not None: - build_path = args.build_path - else: - # Find our database - build_path = find_compilation_database(db_path) - - try: - invocation = [args.clang_tidy_binary, '-list-checks'] - invocation.append('-p=' + build_path) - if args.checks: - invocation.append('-checks=' + args.checks) - invocation.append('-') - if args.quiet: - # Even with -quiet we still want to check if we can call clang-tidy. - with open(os.devnull, 'w') as dev_null: - subprocess.check_call(invocation, stdout=dev_null) + parser = argparse.ArgumentParser( + description="Runs clang-tidy over all files " + "in a compilation database. Requires " + "clang-tidy and clang-apply-replacements in " + "$PATH." + ) + parser.add_argument( + "-clang-tidy-binary", + metavar="PATH", + default="clang-tidy", + help="path to clang-tidy binary", + ) + parser.add_argument( + "-clang-apply-replacements-binary", + metavar="PATH", + default="clang-apply-replacements", + help="path to clang-apply-replacements binary", + ) + parser.add_argument( + "-checks", + default=None, + help="checks filter, when not specified, use clang-tidy default", + ) + parser.add_argument( + "-config", + default=None, + help="Specifies a configuration in YAML/JSON format: " + " -config=\"{Checks: '*', " + " CheckOptions: [{key: x, " + ' value: y}]}" ' + "When the value is empty, clang-tidy will " + "attempt to find a file named .clang-tidy for " + "each source file in its parent directories.", + ) + parser.add_argument( + "-header-filter", + default=None, + help="regular expression matching the names of the " + "headers to output diagnostics from. Diagnostics from " + "the main file of each translation unit are always " + "displayed.", + ) + if yaml: + parser.add_argument( + "-export-fixes", + metavar="filename", + dest="export_fixes", + help="Create a yaml file to store suggested fixes in, " + "which can be applied with clang-apply-replacements.", + ) + parser.add_argument( + "-j", + type=int, + default=0, + help="number of tidy instances to be run in parallel.", + ) + parser.add_argument( + "files", nargs="*", default=[".*"], help="files to be processed (regex on path)" + ) + parser.add_argument("-fix", action="store_true", help="apply fix-its") + parser.add_argument( + "-format", action="store_true", help="Reformat code after applying fixes" + ) + parser.add_argument( + "-style", default="file", help="The style of reformat code after applying fixes" + ) + parser.add_argument( + "-p", dest="build_path", help="Path used to read a compile command database." + ) + parser.add_argument( + "-extra-arg", + dest="extra_arg", + action="append", + default=[], + help="Additional argument to append to the compiler command line.", + ) + parser.add_argument( + "-extra-arg-before", + dest="extra_arg_before", + action="append", + default=[], + help="Additional argument to prepend to the compiler command line.", + ) + parser.add_argument( + "-quiet", action="store_true", help="Run clang-tidy in quiet mode" + ) + args = parser.parse_args() + + db_path = "compile_commands.json" + + if args.build_path is not None: + build_path = args.build_path else: - subprocess.check_call(invocation) - except: - print("Unable to run clang-tidy.", file=sys.stderr) - sys.exit(1) - - # Load the database and extract all files. - database = json.load(open(os.path.join(build_path, db_path))) - files = [make_absolute(entry['file'], entry['directory']) - for entry in database] - - max_task = args.j - if max_task == 0: - max_task = multiprocessing.cpu_count() - - tmpdir = None - if args.fix or (yaml and args.export_fixes): - check_clang_apply_replacements_binary(args) - tmpdir = tempfile.mkdtemp() - - # Build up a big regexy filter from all command line arguments. - file_name_re = re.compile('|'.join(args.files)) - - return_code = 0 - try: - # Spin up a bunch of tidy-launching threads. - task_queue = queue.Queue(max_task) - # List of files with a non-zero return code. - failed_files = [] - lock = threading.Lock() - for _ in range(max_task): - t = threading.Thread(target=run_tidy, - args=(args, tmpdir, build_path, task_queue, lock, failed_files)) - t.daemon = True - t.start() - - # Fill the queue with files. - for name in files: - if file_name_re.search(name): - task_queue.put(name) - - # Wait for all threads to be done. - task_queue.join() - if len(failed_files): - return_code = 1 - - except KeyboardInterrupt: - # This is a sad hack. Unfortunately subprocess goes - # bonkers with ctrl-c and we start forking merrily. - print('\nCtrl-C detected, goodbye.') - if tmpdir: - shutil.rmtree(tmpdir) - os.kill(0, 9) + # Find our database + build_path = find_compilation_database(db_path) - if yaml and args.export_fixes: - print('Writing fixes to ' + args.export_fixes + ' ...') try: - merge_replacement_files(tmpdir, args.export_fixes) + invocation = [args.clang_tidy_binary, "-list-checks"] + invocation.append("-p=" + build_path) + if args.checks: + invocation.append("-checks=" + args.checks) + invocation.append("-") + if args.quiet: + # Even with -quiet we still want to check if we can call clang-tidy. + with open(os.devnull, "w") as dev_null: + subprocess.check_call(invocation, stdout=dev_null) + else: + subprocess.check_call(invocation) except: - print('Error exporting fixes.\n', file=sys.stderr) - traceback.print_exc() - return_code=1 + print("Unable to run clang-tidy.", file=sys.stderr) + sys.exit(1) + + # Load the database and extract all files. + database = json.load(open(os.path.join(build_path, db_path))) + files = [make_absolute(entry["file"], entry["directory"]) for entry in database] - if args.fix: - print('Applying fixes ...') + max_task = args.j + if max_task == 0: + max_task = multiprocessing.cpu_count() + + tmpdir = None + if args.fix or (yaml and args.export_fixes): + check_clang_apply_replacements_binary(args) + tmpdir = tempfile.mkdtemp() + + # Build up a big regexy filter from all command line arguments. + file_name_re = re.compile("|".join(args.files)) + + return_code = 0 try: - apply_fixes(args, tmpdir) - except: - print('Error applying fixes.\n', file=sys.stderr) - traceback.print_exc() - return_code=1 + # Spin up a bunch of tidy-launching threads. + task_queue = queue.Queue(max_task) + # List of files with a non-zero return code. + failed_files = [] + lock = threading.Lock() + for _ in range(max_task): + t = threading.Thread( + target=run_tidy, + args=(args, tmpdir, build_path, task_queue, lock, failed_files), + ) + t.daemon = True + t.start() + + # Fill the queue with files. + for name in files: + if file_name_re.search(name): + task_queue.put(name) + + # Wait for all threads to be done. + task_queue.join() + if len(failed_files): + return_code = 1 + + except KeyboardInterrupt: + # This is a sad hack. Unfortunately subprocess goes + # bonkers with ctrl-c and we start forking merrily. + print("\nCtrl-C detected, goodbye.") + if tmpdir: + shutil.rmtree(tmpdir) + os.kill(0, 9) + + if yaml and args.export_fixes: + print("Writing fixes to " + args.export_fixes + " ...") + try: + merge_replacement_files(tmpdir, args.export_fixes) + except: + print("Error exporting fixes.\n", file=sys.stderr) + traceback.print_exc() + return_code = 1 + + if args.fix: + print("Applying fixes ...") + try: + apply_fixes(args, tmpdir) + except: + print("Error applying fixes.\n", file=sys.stderr) + traceback.print_exc() + return_code = 1 + + if tmpdir: + shutil.rmtree(tmpdir) + sys.exit(return_code) - if tmpdir: - shutil.rmtree(tmpdir) - sys.exit(return_code) -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() diff --git a/src/main.cpp b/src/main.cpp index 4ce0814..602850e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -75,7 +75,6 @@ namespace replicas.emplace_back( consensus::node_config_t{.m_id = replicaId, .m_ip = replicaIp}, std::move(stub) ); - spdlog::info( } ++replicaId; } diff --git a/tests/go/kvtest/pkg/scenarios/crud.go b/tests/go/kvtest/pkg/scenarios/crud.go index fb9da12..054cda0 100644 --- a/tests/go/kvtest/pkg/scenarios/crud.go +++ b/tests/go/kvtest/pkg/scenarios/crud.go @@ -42,10 +42,12 @@ func NewCRUDScenario(params map[string]interface{}) *CRUDScenario { return scenario } +// Execute method runs the test scenario func (s *CRUDScenario) Execute(ctx *core.TestContext) error { // startTime := time.Now() operation := s.selectOperation() + log.Printf("executing %v", operation) var err error switch operation { diff --git a/tests/go/kvtest/pkg/scenarios/factory.go b/tests/go/kvtest/pkg/scenarios/factory.go index fc5110e..6ab1146 100644 --- a/tests/go/kvtest/pkg/scenarios/factory.go +++ b/tests/go/kvtest/pkg/scenarios/factory.go @@ -13,6 +13,8 @@ func (f *ScenarioFactory) Create(config core.ScenarioConfig) (core.Scenario, err switch config.Type { case "crud": return NewCRUDScenario(config.Parameters), nil + case "write_read": + return NewWriteReadScenario(config.Parameters), nil default: return NewCRUDScenario(config.Parameters), nil } diff --git a/tests/go/kvtest/pkg/scenarios/wr.go b/tests/go/kvtest/pkg/scenarios/wr.go new file mode 100644 index 0000000..d6fd8b1 --- /dev/null +++ b/tests/go/kvtest/pkg/scenarios/wr.go @@ -0,0 +1,59 @@ +package scenarios + +import ( + "bytes" + "fmt" + + "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" +) + +type WriteReadScenario struct { + name string + valueSize int +} + +func NewWriteReadScenario(params map[string]interface{}) *WriteReadScenario { + scenario := &WriteReadScenario{ + name: "ReadYourWrites", + valueSize: 256, + } + + if val, ok := params["value_size"].(int); ok { + scenario.valueSize = val + } + + return scenario +} + +func (wr *WriteReadScenario) Execute(ctx *core.TestContext) error { + key := ctx.GenerateKey() + expectedValue := ctx.GenerateValue(wr.valueSize) + + err := ctx.KV.Put(ctx.Context(), []byte(key), []byte(expectedValue)) + if err != nil { + return err + } + + value, err := ctx.KV.Get(ctx.Context(), []byte(key)) + if err != nil { + return nil + } + + if !bytes.Equal(expectedValue, value) { + return fmt.Errorf("data consistency error: expected %v, got %v", expectedValue, value) + } + + return nil +} + +func (wr *WriteReadScenario) Name() string { + return wr.name +} + +func (wr *WriteReadScenario) Setup(ctx *core.TestContext) error { + return nil +} + +func (wr *WriteReadScenario) Teardown(ctx *core.TestContext) error { + return nil +} From 140c65aec8984c26a6154cfe05b92768b54ebe87 Mon Sep 17 00:00:00 2001 From: lnikon Date: Mon, 22 Sep 2025 00:59:28 +0400 Subject: [PATCH 09/17] Integration & load testing: - ReadYourWrites scenario in-progress --- conan/profiles/debug-clang | 4 +- lib/raft/raft.cpp | 2 +- tests/go/kvtest/cmd/kvtest/kvtest.yml | 9 +++- tests/go/kvtest/cmd/kvtest/main.go | 8 +-- tests/go/kvtest/go.sum | 12 +++++ tests/go/kvtest/pkg/adapters/tinykvpp.go | 4 +- tests/go/kvtest/pkg/config/loader.go | 2 +- tests/go/kvtest/pkg/executors/orchestrator.go | 4 +- tests/go/kvtest/pkg/scenarios/crud.go | 2 +- tests/go/kvtest/pkg/scenarios/factory.go | 2 +- tests/go/kvtest/pkg/scenarios/wr.go | 51 ++++++++++++------- 11 files changed, 68 insertions(+), 32 deletions(-) diff --git a/conan/profiles/debug-clang b/conan/profiles/debug-clang index 0f9e24a..06db1b0 100644 --- a/conan/profiles/debug-clang +++ b/conan/profiles/debug-clang @@ -1,6 +1,6 @@ [buildenv] -CC=/usr/bin/clang-20 -CXX=/usr/bin/clang++-20 +CC=/usr/bin/clang +CXX=/usr/bin/clang++ [settings] arch=x86_64 diff --git a/lib/raft/raft.cpp b/lib/raft/raft.cpp index c311d1e..1cac86d 100644 --- a/lib/raft/raft.cpp +++ b/lib/raft/raft.cpp @@ -27,7 +27,7 @@ const std::string_view gRaftFilename = "RAFT_PERSISTENCE"; auto constructFilename(std::string_view filename, std::uint32_t peerId) -> std::string { - return fmt::format("./var/tkvpp/{}_NODE_{}", filename, peerId); + return fmt::format("/var/tkvpp/{}_NODE_{}", filename, peerId); } auto generate_random_timeout(const int minTimeout, const int maxTimeout) -> int diff --git a/tests/go/kvtest/cmd/kvtest/kvtest.yml b/tests/go/kvtest/cmd/kvtest/kvtest.yml index 5f681b2..4b4d428 100644 --- a/tests/go/kvtest/cmd/kvtest/kvtest.yml +++ b/tests/go/kvtest/cmd/kvtest/kvtest.yml @@ -16,8 +16,15 @@ integration: scenarios: - name: basic_crud type: crud - enabled: true + enabled: false parameters: read_ratio: 0.7 write_ratio: 0.3 value_size: 5 + - name: read_your_writes + type: write_read + enabled: true + parameters: + value_size: 256 + num_operations: 16 + delay: "1ms" diff --git a/tests/go/kvtest/cmd/kvtest/main.go b/tests/go/kvtest/cmd/kvtest/main.go index 8bd4fdd..17d4cc1 100644 --- a/tests/go/kvtest/cmd/kvtest/main.go +++ b/tests/go/kvtest/cmd/kvtest/main.go @@ -5,10 +5,10 @@ import ( "log" "os" - "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/adapters" - "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/config" - "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" - "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/scenarios" + "github.com/lnikon/kvtest/pkg/adapters" + "github.com/lnikon/kvtest/pkg/config" + "github.com/lnikon/kvtest/pkg/core" + "github.com/lnikon/kvtest/pkg/scenarios" "github.com/spf13/cobra" ) diff --git a/tests/go/kvtest/go.sum b/tests/go/kvtest/go.sum index bdf9b15..fc22faf 100644 --- a/tests/go/kvtest/go.sum +++ b/tests/go/kvtest/go.sum @@ -25,9 +25,13 @@ github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -42,16 +46,24 @@ go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= +google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= +google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tests/go/kvtest/pkg/adapters/tinykvpp.go b/tests/go/kvtest/pkg/adapters/tinykvpp.go index e71528e..1a5447b 100644 --- a/tests/go/kvtest/pkg/adapters/tinykvpp.go +++ b/tests/go/kvtest/pkg/adapters/tinykvpp.go @@ -8,8 +8,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" - pb "github.com/lnikon/tinykvpp/tests/go/kvtest/proto/tinykvpp/v1" + "github.com/lnikon/kvtest/pkg/core" + pb "github.com/lnikon/kvtest/proto/tinykvpp/v1" ) type TinyKVPPConfig struct { diff --git a/tests/go/kvtest/pkg/config/loader.go b/tests/go/kvtest/pkg/config/loader.go index 3e70c62..da69757 100644 --- a/tests/go/kvtest/pkg/config/loader.go +++ b/tests/go/kvtest/pkg/config/loader.go @@ -6,7 +6,7 @@ import ( "gopkg.in/yaml.v3" - "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" + "github.com/lnikon/kvtest/pkg/core" ) // LoadFromFile loads configuration from a YAML file diff --git a/tests/go/kvtest/pkg/executors/orchestrator.go b/tests/go/kvtest/pkg/executors/orchestrator.go index aabba8e..8b718fb 100644 --- a/tests/go/kvtest/pkg/executors/orchestrator.go +++ b/tests/go/kvtest/pkg/executors/orchestrator.go @@ -4,8 +4,8 @@ import ( "fmt" "log" - "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" - "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/scenarios" + "github.com/lnikon/kvtest/pkg/core" + "github.com/lnikon/kvtest/pkg/scenarios" ) type TestOrchestrator struct { diff --git a/tests/go/kvtest/pkg/scenarios/crud.go b/tests/go/kvtest/pkg/scenarios/crud.go index 054cda0..8b94b9d 100644 --- a/tests/go/kvtest/pkg/scenarios/crud.go +++ b/tests/go/kvtest/pkg/scenarios/crud.go @@ -8,7 +8,7 @@ import ( // "time" - "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" + "github.com/lnikon/kvtest/pkg/core" ) // CRUDScenario implements basic CRUD operations diff --git a/tests/go/kvtest/pkg/scenarios/factory.go b/tests/go/kvtest/pkg/scenarios/factory.go index 6ab1146..2572806 100644 --- a/tests/go/kvtest/pkg/scenarios/factory.go +++ b/tests/go/kvtest/pkg/scenarios/factory.go @@ -1,6 +1,6 @@ package scenarios -import "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" +import "github.com/lnikon/kvtest/pkg/core" // ScenarioFactory creates scenarios based on configuration type ScenarioFactory struct{} diff --git a/tests/go/kvtest/pkg/scenarios/wr.go b/tests/go/kvtest/pkg/scenarios/wr.go index d6fd8b1..65c34e4 100644 --- a/tests/go/kvtest/pkg/scenarios/wr.go +++ b/tests/go/kvtest/pkg/scenarios/wr.go @@ -3,44 +3,61 @@ package scenarios import ( "bytes" "fmt" + "time" - "github.com/lnikon/tinykvpp/tests/go/kvtest/pkg/core" + "github.com/lnikon/kvtest/pkg/core" ) type WriteReadScenario struct { - name string - valueSize int + name string + valueSize int + numOperations int + delay time.Duration } func NewWriteReadScenario(params map[string]interface{}) *WriteReadScenario { scenario := &WriteReadScenario{ - name: "ReadYourWrites", - valueSize: 256, + name: "ReadYourWrites", + valueSize: 256, + numOperations: 1024, + delay: 100 * time.Millisecond, } if val, ok := params["value_size"].(int); ok { scenario.valueSize = val } + if val, ok := params["num_operations"].(int); ok { + scenario.numOperations = val + } + + if val, ok := params["delay"].(time.Duration); ok { + scenario.delay = val + } + return scenario } func (wr *WriteReadScenario) Execute(ctx *core.TestContext) error { - key := ctx.GenerateKey() - expectedValue := ctx.GenerateValue(wr.valueSize) + for i := 0; i < wr.numOperations; i++ { + key := ctx.GenerateKey() + expectedValue := ctx.GenerateValue(wr.valueSize) - err := ctx.KV.Put(ctx.Context(), []byte(key), []byte(expectedValue)) - if err != nil { - return err - } + err := ctx.KV.Put(ctx.Context(), []byte(key), []byte(expectedValue)) + if err != nil { + return err + } - value, err := ctx.KV.Get(ctx.Context(), []byte(key)) - if err != nil { - return nil - } + value, err := ctx.KV.Get(ctx.Context(), []byte(key)) + if err != nil { + return nil + } + + if !bytes.Equal(expectedValue, value) { + return fmt.Errorf("data consistency error: expected %v, got %v", expectedValue, value) + } - if !bytes.Equal(expectedValue, value) { - return fmt.Errorf("data consistency error: expected %v, got %v", expectedValue, value) + time.Sleep(wr.delay) } return nil From b3127f8cf32062f005aa8b208387a47e6dd2c995 Mon Sep 17 00:00:00 2001 From: lnikon Date: Fri, 3 Oct 2025 18:04:27 +0400 Subject: [PATCH 10/17] Integration & load testing: - Implement basic memtable stress test - General refactoring --- flake.lock | 61 --- flake.nix | 54 --- infra/assets/tkvpp_config_standalone.json | 6 +- lib/structures/lsmtree/levels/level.cpp | 3 +- lib/structures/lsmtree/levels/levels.cpp | 36 +- lib/structures/lsmtree/lsmtree.cpp | 2 - lib/structures/lsmtree/segments/helpers.h | 4 +- main_run_replicas.sh | 5 - raft_run_replicas.sh | 5 - run-clang-tidy.py | 386 ------------------ tests/go/kvtest/cmd/kvtest/kvtest.yml | 10 +- tests/go/kvtest/go.sum | 73 ++-- tests/go/kvtest/pkg/scenarios/factory.go | 13 +- .../scenarios/{wr.go => readyourwrites.go} | 22 +- .../pkg/scenarios/storage/memtable_stress.go | 185 +++++++++ 15 files changed, 267 insertions(+), 598 deletions(-) delete mode 100644 flake.lock delete mode 100644 flake.nix delete mode 100755 main_run_replicas.sh delete mode 100755 raft_run_replicas.sh delete mode 100755 run-clang-tidy.py rename tests/go/kvtest/pkg/scenarios/{wr.go => readyourwrites.go} (64%) create mode 100644 tests/go/kvtest/pkg/scenarios/storage/memtable_stress.go diff --git a/flake.lock b/flake.lock deleted file mode 100644 index 33fdb4a..0000000 --- a/flake.lock +++ /dev/null @@ -1,61 +0,0 @@ -{ - "nodes": { - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1712163089, - "narHash": "sha256-Um+8kTIrC19vD4/lUCN9/cU9kcOsD1O1m+axJqQPyMM=", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "fd281bd6b7d3e32ddfa399853946f782553163b5", - "type": "github" - }, - "original": { - "owner": "nixos", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index 036833e..0000000 --- a/flake.nix +++ /dev/null @@ -1,54 +0,0 @@ -{ - description = "tinykvpp's flake"; - - inputs = { - nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable"; - flake-utils.url = "github:numtide/flake-utils"; - }; - - outputs = { self, nixpkgs, flake-utils, ... }: flake-utils.lib.eachDefaultSystem (system: - let - pkgs = import nixpkgs { - inherit system; - }; - system = "x86_64-linux"; - tinykvpp = (with pkgs; stdenv.mkDerivation { - name = "tinykvpp"; - src = fetchgit { - url = "https://github.com/lnikon/tinykvpp"; - sha256 = "76EuOwIbxshtkaf8z50hmmUyFj0TTdF+HJbnNlvQDrY="; - fetchSubmodules = true; - }; - nativeBuildInputs = [ - cmake - clang - boost - grpc - protobuf - spdlog - fmt - re2 - abseil-cpp - cmake - vim - catch2_3 - systemd - openssl - ]; - buildPhase = "make -j $NIX_BUILD_CORES"; - } - ); - in - rec { - defaultApp = flake-utils.lib.mkApp { - drv = defaultPackage; - }; - defaultPackage = tinykvpp; - devShell = pkgs.mkShell { - buildInputs = [ - tinykvpp - ]; - }; - } - ); -} diff --git a/infra/assets/tkvpp_config_standalone.json b/infra/assets/tkvpp_config_standalone.json index 2693b3b..f82596a 100644 --- a/infra/assets/tkvpp_config_standalone.json +++ b/infra/assets/tkvpp_config_standalone.json @@ -14,15 +14,15 @@ "storageType": "persistent" }, "lsm": { - "flushThreshold": 128, + "flushThreshold": 64000000, "maximumLevels": 12, "levelZeroCompaction": { "compactionStrategy": "levelled", - "compactionThreshold": 1024 + "compactionThreshold": 256000000 }, "levelNonZeroCompaction": { "compactionStrategy": "tiered", - "compactionThreshold": 1024 + "compactionThreshold": 256000000 } }, "server": { diff --git a/lib/structures/lsmtree/levels/level.cpp b/lib/structures/lsmtree/levels/level.cpp index 9a6e3c3..5018a8e 100644 --- a/lib/structures/lsmtree/levels/level.cpp +++ b/lib/structures/lsmtree/levels/level.cpp @@ -235,14 +235,13 @@ void level_t::merge(const segments::regular_segment::shared_ptr_t &pSegment) noe // TODO(lnikon): Make this parameter configurable. Use measurement // units(mb). - const std::size_t segmentSize{1024}; memtable::memtable_t newMemtable; segments::storage::segment_storage_t newSegments; for (const auto ¤tRecord : mergedMemtable) { newMemtable.emplace(currentRecord); - if (newMemtable.size() >= segmentSize) + if (newMemtable.size() >= m_pConfig->LSMTreeConfig.SegmentSize) { auto name{fmt::format("{}_{}", helpers::segment_name(), index())}; newSegments.emplace( diff --git a/lib/structures/lsmtree/levels/levels.cpp b/lib/structures/lsmtree/levels/levels.cpp index de9c4ab..c6c9c46 100644 --- a/lib/structures/lsmtree/levels/levels.cpp +++ b/lib/structures/lsmtree/levels/levels.cpp @@ -98,14 +98,17 @@ auto levels_t::compact() -> segments::regular_segment::shared_ptr_t } )); - // Update manifest with new segment - ASSERT(m_pManifest->add( - db::manifest::manifest_t::segment_record_t{ - .op = segment_operation_k::add_segment_k, - .name = compactedCurrentLevelSegment->get_name(), - .level = currentLevel->index() - } - )); + { + // Update manifest with new segment + auto ok{m_pManifest->add( + db::manifest::manifest_t::segment_record_t{ + .op = segment_operation_k::add_segment_k, + .name = compactedCurrentLevelSegment->get_name(), + .level = currentLevel->index() + } + )}; + ASSERT(ok); + } // If computation succeeded, then flush the compacted segment into disk compactedCurrentLevelSegment->flush(); @@ -121,13 +124,16 @@ auto levels_t::compact() -> segments::regular_segment::shared_ptr_t // Purge the segment representing the compacted level and update the // manifest - ASSERT(m_pManifest->add( - db::manifest::manifest_t::segment_record_t{ - .op = segment_operation_k::remove_segment_k, - .name = compactedCurrentLevelSegment->get_name(), - .level = currentLevel->index() - } - )); + { + bool ok{m_pManifest->add( + db::manifest::manifest_t::segment_record_t{ + .op = segment_operation_k::remove_segment_k, + .name = compactedCurrentLevelSegment->get_name(), + .level = currentLevel->index() + } + )}; + ASSERT(ok); + } compactedCurrentLevelSegment->remove_from_disk(); // After merging current level into the next level purge the current diff --git a/lib/structures/lsmtree/lsmtree.cpp b/lib/structures/lsmtree/lsmtree.cpp index 9d81c82..12ff417 100644 --- a/lib/structures/lsmtree/lsmtree.cpp +++ b/lib/structures/lsmtree/lsmtree.cpp @@ -12,8 +12,6 @@ #include "structures/lsmtree/lsmtree_types.h" #include "structures/lsmtree/lsmtree.h" #include "db/manifest/manifest.h" -#include "wal/wal.h" -#include "tinykvpp/v1/tinykvpp_service.grpc.pb.h" #include "tinykvpp/v1/tinykvpp_service.pb.h" namespace structures::lsmtree diff --git a/lib/structures/lsmtree/segments/helpers.h b/lib/structures/lsmtree/segments/helpers.h index a91f765..596dfd2 100644 --- a/lib/structures/lsmtree/segments/helpers.h +++ b/lib/structures/lsmtree/segments/helpers.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include "config/config.h" +#include "structures/lsmtree/segments/types.h" #include diff --git a/main_run_replicas.sh b/main_run_replicas.sh deleted file mode 100755 index 81837ea..0000000 --- a/main_run_replicas.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -./build/Main -c ./assets/tkvpp_config_1.json &> log_1.txt & -./build/Main -c ./assets/tkvpp_config_2.json &> log_2.txt & -./build/Main -c ./assets/tkvpp_config_3.json &> log_3.txt & diff --git a/raft_run_replicas.sh b/raft_run_replicas.sh deleted file mode 100755 index 5bf7f8c..0000000 --- a/raft_run_replicas.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -./build/RaftMain --id 1 --nodes 0.0.0.0:8081,0.0.0.0:8082,0.0.0.0:8083 &> log_1.txt & -./build/RaftMain --id 2 --nodes 0.0.0.0:8081,0.0.0.0:8082,0.0.0.0:8083 &> log_2.txt & -./build/RaftMain --id 3 --nodes 0.0.0.0:8081,0.0.0.0:8082,0.0.0.0:8083 &> log_3.txt & diff --git a/run-clang-tidy.py b/run-clang-tidy.py deleted file mode 100755 index 449ba3c..0000000 --- a/run-clang-tidy.py +++ /dev/null @@ -1,386 +0,0 @@ -#!/usr/bin/env python3 -# -# ===- run-clang-tidy.py - Parallel clang-tidy runner ---------*- python -*--===# -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -# ===------------------------------------------------------------------------===# -# FIXME: Integrate with clang-tidy-diff.py - -""" -Parallel clang-tidy runner -========================== - -Runs clang-tidy over all files in a compilation database. Requires clang-tidy -and clang-apply-replacements in $PATH. - -Example invocations. -- Run clang-tidy on all files in the current working directory with a default - set of checks and show warnings in the cpp files and all project headers. - run-clang-tidy.py $PWD - -- Fix all header guards. - run-clang-tidy.py -fix -checks=-*,llvm-header-guard - -- Fix all header guards included from clang-tidy and header guards - for clang-tidy headers. - run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \ - -header-filter=extra/clang-tidy - -Compilation database setup: -http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html -""" - -from __future__ import print_function - -import argparse -import glob -import json -import multiprocessing -import os -import re -import shutil -import subprocess -import sys -import tempfile -import threading -import traceback - -try: - import yaml -except ImportError: - yaml = None - -is_py2 = sys.version[0] == "2" - -if is_py2: - import Queue as queue -else: - import queue as queue - - -def find_compilation_database(path): - """Adjusts the directory until a compilation database is found.""" - result = "./" - while not os.path.isfile(os.path.join(result, path)): - if os.path.realpath(result) == "/": - print("Error: could not find compilation database.") - sys.exit(1) - result += "../" - return os.path.realpath(result) - - -def make_absolute(f, directory): - if os.path.isabs(f): - return f - return os.path.normpath(os.path.join(directory, f)) - - -def get_tidy_invocation( - f, - clang_tidy_binary, - checks, - tmpdir, - build_path, - header_filter, - extra_arg, - extra_arg_before, - quiet, - config, -): - """Gets a command line for clang-tidy.""" - start = [clang_tidy_binary] - if header_filter is not None: - start.append("-header-filter=" + header_filter) - if checks: - start.append("-checks=" + checks) - if tmpdir is not None: - start.append("-export-fixes") - # Get a temporary file. We immediately close the handle so clang-tidy can - # overwrite it. - (handle, name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir) - os.close(handle) - start.append(name) - for arg in extra_arg: - start.append("-extra-arg=%s" % arg) - for arg in extra_arg_before: - start.append("-extra-arg-before=%s" % arg) - start.append("-p=" + build_path) - if quiet: - start.append("-quiet") - if config: - start.append("-config=" + config) - start.append(f) - return start - - -def merge_replacement_files(tmpdir, mergefile): - """Merge all replacement files in a directory into a single file""" - # The fixes suggested by clang-tidy >= 4.0.0 are given under - # the top level key 'Diagnostics' in the output yaml files - mergekey = "Diagnostics" - merged = [] - for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")): - content = yaml.safe_load(open(replacefile, "r")) - if not content: - continue # Skip empty files. - merged.extend(content.get(mergekey, [])) - - if merged: - # MainSourceFile: The key is required by the definition inside - # include/clang/Tooling/ReplacementsYaml.h, but the value - # is actually never used inside clang-apply-replacements, - # so we set it to '' here. - output = {"MainSourceFile": "", mergekey: merged} - with open(mergefile, "w") as out: - yaml.safe_dump(output, out) - else: - # Empty the file: - open(mergefile, "w").close() - - -def check_clang_apply_replacements_binary(args): - """Checks if invoking supplied clang-apply-replacements binary works.""" - try: - subprocess.check_call([args.clang_apply_replacements_binary, "--version"]) - except: - print( - "Unable to run clang-apply-replacements. Is clang-apply-replacements " - "binary correctly specified?", - file=sys.stderr, - ) - traceback.print_exc() - sys.exit(1) - - -def apply_fixes(args, tmpdir): - """Calls clang-apply-fixes on a given directory.""" - invocation = [args.clang_apply_replacements_binary] - if args.format: - invocation.append("-format") - if args.style: - invocation.append("-style=" + args.style) - invocation.append(tmpdir) - subprocess.call(invocation) - - -def run_tidy(args, tmpdir, build_path, queue, lock, failed_files): - """Takes filenames out of queue and runs clang-tidy on them.""" - while True: - name = queue.get() - invocation = get_tidy_invocation( - name, - args.clang_tidy_binary, - args.checks, - tmpdir, - build_path, - args.header_filter, - args.extra_arg, - args.extra_arg_before, - args.quiet, - args.config, - ) - - proc = subprocess.Popen( - invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - output, err = proc.communicate() - if proc.returncode != 0: - failed_files.append(name) - with lock: - sys.stdout.write(" ".join(invocation) + "\n" + output.decode("utf-8")) - if len(err) > 0: - sys.stdout.flush() - sys.stderr.write(err.decode("utf-8")) - queue.task_done() - - -def main(): - parser = argparse.ArgumentParser( - description="Runs clang-tidy over all files " - "in a compilation database. Requires " - "clang-tidy and clang-apply-replacements in " - "$PATH." - ) - parser.add_argument( - "-clang-tidy-binary", - metavar="PATH", - default="clang-tidy", - help="path to clang-tidy binary", - ) - parser.add_argument( - "-clang-apply-replacements-binary", - metavar="PATH", - default="clang-apply-replacements", - help="path to clang-apply-replacements binary", - ) - parser.add_argument( - "-checks", - default=None, - help="checks filter, when not specified, use clang-tidy default", - ) - parser.add_argument( - "-config", - default=None, - help="Specifies a configuration in YAML/JSON format: " - " -config=\"{Checks: '*', " - " CheckOptions: [{key: x, " - ' value: y}]}" ' - "When the value is empty, clang-tidy will " - "attempt to find a file named .clang-tidy for " - "each source file in its parent directories.", - ) - parser.add_argument( - "-header-filter", - default=None, - help="regular expression matching the names of the " - "headers to output diagnostics from. Diagnostics from " - "the main file of each translation unit are always " - "displayed.", - ) - if yaml: - parser.add_argument( - "-export-fixes", - metavar="filename", - dest="export_fixes", - help="Create a yaml file to store suggested fixes in, " - "which can be applied with clang-apply-replacements.", - ) - parser.add_argument( - "-j", - type=int, - default=0, - help="number of tidy instances to be run in parallel.", - ) - parser.add_argument( - "files", nargs="*", default=[".*"], help="files to be processed (regex on path)" - ) - parser.add_argument("-fix", action="store_true", help="apply fix-its") - parser.add_argument( - "-format", action="store_true", help="Reformat code after applying fixes" - ) - parser.add_argument( - "-style", default="file", help="The style of reformat code after applying fixes" - ) - parser.add_argument( - "-p", dest="build_path", help="Path used to read a compile command database." - ) - parser.add_argument( - "-extra-arg", - dest="extra_arg", - action="append", - default=[], - help="Additional argument to append to the compiler command line.", - ) - parser.add_argument( - "-extra-arg-before", - dest="extra_arg_before", - action="append", - default=[], - help="Additional argument to prepend to the compiler command line.", - ) - parser.add_argument( - "-quiet", action="store_true", help="Run clang-tidy in quiet mode" - ) - args = parser.parse_args() - - db_path = "compile_commands.json" - - if args.build_path is not None: - build_path = args.build_path - else: - # Find our database - build_path = find_compilation_database(db_path) - - try: - invocation = [args.clang_tidy_binary, "-list-checks"] - invocation.append("-p=" + build_path) - if args.checks: - invocation.append("-checks=" + args.checks) - invocation.append("-") - if args.quiet: - # Even with -quiet we still want to check if we can call clang-tidy. - with open(os.devnull, "w") as dev_null: - subprocess.check_call(invocation, stdout=dev_null) - else: - subprocess.check_call(invocation) - except: - print("Unable to run clang-tidy.", file=sys.stderr) - sys.exit(1) - - # Load the database and extract all files. - database = json.load(open(os.path.join(build_path, db_path))) - files = [make_absolute(entry["file"], entry["directory"]) for entry in database] - - max_task = args.j - if max_task == 0: - max_task = multiprocessing.cpu_count() - - tmpdir = None - if args.fix or (yaml and args.export_fixes): - check_clang_apply_replacements_binary(args) - tmpdir = tempfile.mkdtemp() - - # Build up a big regexy filter from all command line arguments. - file_name_re = re.compile("|".join(args.files)) - - return_code = 0 - try: - # Spin up a bunch of tidy-launching threads. - task_queue = queue.Queue(max_task) - # List of files with a non-zero return code. - failed_files = [] - lock = threading.Lock() - for _ in range(max_task): - t = threading.Thread( - target=run_tidy, - args=(args, tmpdir, build_path, task_queue, lock, failed_files), - ) - t.daemon = True - t.start() - - # Fill the queue with files. - for name in files: - if file_name_re.search(name): - task_queue.put(name) - - # Wait for all threads to be done. - task_queue.join() - if len(failed_files): - return_code = 1 - - except KeyboardInterrupt: - # This is a sad hack. Unfortunately subprocess goes - # bonkers with ctrl-c and we start forking merrily. - print("\nCtrl-C detected, goodbye.") - if tmpdir: - shutil.rmtree(tmpdir) - os.kill(0, 9) - - if yaml and args.export_fixes: - print("Writing fixes to " + args.export_fixes + " ...") - try: - merge_replacement_files(tmpdir, args.export_fixes) - except: - print("Error exporting fixes.\n", file=sys.stderr) - traceback.print_exc() - return_code = 1 - - if args.fix: - print("Applying fixes ...") - try: - apply_fixes(args, tmpdir) - except: - print("Error applying fixes.\n", file=sys.stderr) - traceback.print_exc() - return_code = 1 - - if tmpdir: - shutil.rmtree(tmpdir) - sys.exit(return_code) - - -if __name__ == "__main__": - main() diff --git a/tests/go/kvtest/cmd/kvtest/kvtest.yml b/tests/go/kvtest/cmd/kvtest/kvtest.yml index 4b4d428..03c0172 100644 --- a/tests/go/kvtest/cmd/kvtest/kvtest.yml +++ b/tests/go/kvtest/cmd/kvtest/kvtest.yml @@ -14,6 +14,14 @@ integration: - consistency scenarios: + - name: memtable_stress + type: memtable_stress + enabled: true + parameters: + record_size: 16000 + record_count: 8096 + concurrent_writers: 1 + flush_threshold: 6400000 - name: basic_crud type: crud enabled: false @@ -23,7 +31,7 @@ scenarios: value_size: 5 - name: read_your_writes type: write_read - enabled: true + enabled: false parameters: value_size: 256 num_operations: 16 diff --git a/tests/go/kvtest/go.sum b/tests/go/kvtest/go.sum index fc22faf..a0eeebb 100644 --- a/tests/go/kvtest/go.sum +++ b/tests/go/kvtest/go.sum @@ -1,71 +1,48 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/go-chi/httprate v0.15.0 h1:j54xcWV9KGmPf/X4H32/aTH+wBlrvxL7P+SdnRqxh5g= -github.com/go-chi/httprate v0.15.0/go.mod h1:rzGHhVrsBn3IMLYDOZQsSU4fJNWcjui4fWKJcCId1R4= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= -github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= -github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= -github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= -github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= -github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= -github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= -github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= -github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= -google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tests/go/kvtest/pkg/scenarios/factory.go b/tests/go/kvtest/pkg/scenarios/factory.go index 2572806..46505e5 100644 --- a/tests/go/kvtest/pkg/scenarios/factory.go +++ b/tests/go/kvtest/pkg/scenarios/factory.go @@ -1,6 +1,11 @@ package scenarios -import "github.com/lnikon/kvtest/pkg/core" +import ( + "fmt" + + "github.com/lnikon/kvtest/pkg/core" + "github.com/lnikon/kvtest/pkg/scenarios/storage" +) // ScenarioFactory creates scenarios based on configuration type ScenarioFactory struct{} @@ -14,8 +19,10 @@ func (f *ScenarioFactory) Create(config core.ScenarioConfig) (core.Scenario, err case "crud": return NewCRUDScenario(config.Parameters), nil case "write_read": - return NewWriteReadScenario(config.Parameters), nil + return NewReadYourWrites(config.Parameters), nil + case "memtable_stress": + return storage.NewMemtableStressScenario(config.Parameters), nil default: - return NewCRUDScenario(config.Parameters), nil + return nil, fmt.Errorf("unkown scenario: %s", config.Type) } } diff --git a/tests/go/kvtest/pkg/scenarios/wr.go b/tests/go/kvtest/pkg/scenarios/readyourwrites.go similarity index 64% rename from tests/go/kvtest/pkg/scenarios/wr.go rename to tests/go/kvtest/pkg/scenarios/readyourwrites.go index 65c34e4..d0226e1 100644 --- a/tests/go/kvtest/pkg/scenarios/wr.go +++ b/tests/go/kvtest/pkg/scenarios/readyourwrites.go @@ -8,15 +8,15 @@ import ( "github.com/lnikon/kvtest/pkg/core" ) -type WriteReadScenario struct { +type ReadYourWritesScenario struct { name string valueSize int numOperations int delay time.Duration } -func NewWriteReadScenario(params map[string]interface{}) *WriteReadScenario { - scenario := &WriteReadScenario{ +func NewReadYourWrites(params map[string]interface{}) *ReadYourWritesScenario { + scenario := &ReadYourWritesScenario{ name: "ReadYourWrites", valueSize: 256, numOperations: 1024, @@ -38,10 +38,10 @@ func NewWriteReadScenario(params map[string]interface{}) *WriteReadScenario { return scenario } -func (wr *WriteReadScenario) Execute(ctx *core.TestContext) error { - for i := 0; i < wr.numOperations; i++ { +func (s *ReadYourWritesScenario) Execute(ctx *core.TestContext) error { + for i := 0; i < s.numOperations; i++ { key := ctx.GenerateKey() - expectedValue := ctx.GenerateValue(wr.valueSize) + expectedValue := ctx.GenerateValue(s.valueSize) err := ctx.KV.Put(ctx.Context(), []byte(key), []byte(expectedValue)) if err != nil { @@ -57,20 +57,20 @@ func (wr *WriteReadScenario) Execute(ctx *core.TestContext) error { return fmt.Errorf("data consistency error: expected %v, got %v", expectedValue, value) } - time.Sleep(wr.delay) + time.Sleep(s.delay) } return nil } -func (wr *WriteReadScenario) Name() string { - return wr.name +func (s *ReadYourWritesScenario) Name() string { + return s.name } -func (wr *WriteReadScenario) Setup(ctx *core.TestContext) error { +func (s *ReadYourWritesScenario) Setup(ctx *core.TestContext) error { return nil } -func (wr *WriteReadScenario) Teardown(ctx *core.TestContext) error { +func (s *ReadYourWritesScenario) Teardown(ctx *core.TestContext) error { return nil } diff --git a/tests/go/kvtest/pkg/scenarios/storage/memtable_stress.go b/tests/go/kvtest/pkg/scenarios/storage/memtable_stress.go new file mode 100644 index 0000000..403c176 --- /dev/null +++ b/tests/go/kvtest/pkg/scenarios/storage/memtable_stress.go @@ -0,0 +1,185 @@ +package storage + +import ( + "fmt" + "log" + "slices" + "sync/atomic" + "time" + + "github.com/lnikon/kvtest/pkg/core" +) + +// MemtableStressScenario tests memtable behavior under various load conditions +type MemtableStressScenario struct { + name string + recordSize int + recordCount int + concurrentWriters int + flushThreshold int + + // Internal state for tracking + writeLatencies []time.Duration + flushDetected bool + consistencyErrors atomic.Int64 +} + +// NewMemtableStressScenario creates a new memtable stress test scenario +func NewMemtableStressScenario(params map[string]interface{}) *MemtableStressScenario { + scenario := &MemtableStressScenario{ + name: "MemtableStress", + recordSize: 1024, + recordCount: 10000, + concurrentWriters: 4, + flushThreshold: 8 * 1024 * 1024, + writeLatencies: make([]time.Duration, 0, 10000), + } + + // Apply parameters from config + if val, ok := params["record_size"].(int); ok { + scenario.recordSize = val + } + if val, ok := params["record_count"].(int); ok { + scenario.recordCount = val + } + if val, ok := params["concurrent_writes"].(int); ok { + scenario.concurrentWriters = val + } + if val, ok := params["flush_threshold"].(int); ok { + scenario.flushThreshold = val + } + + return scenario +} + +func (s *MemtableStressScenario) Execute(ctx *core.TestContext) error { + log.Printf("[%s] === Test 1: Sequential Writes (Flush Detection) === ", s.name) + if err := s.testSequentialWrites(ctx); err != nil { + return fmt.Errorf("sequantial writes failec: %w", err) + } + log.Printf("[%s] Sequential writes test passed", s.name) + + return nil +} + +func (s *MemtableStressScenario) Name() string { + return s.name +} + +func (s *MemtableStressScenario) Setup(ctx *core.TestContext) error { + + log.Printf("[%s] Setting up memtable stress test", s.name) + log.Printf("[%s] Configuration:", s.name) + log.Printf(" - Record size: %d bytes", s.recordSize) + log.Printf(" - Record count: %d", s.recordCount) + log.Printf(" - Concurrent writers: %d", s.concurrentWriters) + log.Printf(" - Expected flush threshold: %d bytes", s.flushThreshold) + + // Optional: Clean up any existing test data + // This would require a Delete or Clear operation on your KV store + // For now, we'll use unique key prefixes to avoid conflicts + + return nil + +} + +func (s *MemtableStressScenario) Teardown(ctx *core.TestContext) error { + log.Printf("[%s] Tearing down memtable stress test", s.name) + return nil +} + +func (s *MemtableStressScenario) testSequentialWrites(ctx *core.TestContext) error { + startTime := time.Now() + lastReportTime := startTime + + var recordsSinceReport int + + // Track latency percentiles to detect flush spikes + var latencyWindow []time.Duration + const windowSize = 100 + + for i := 0; i < s.recordCount; i++ { + key := fmt.Sprintf("seq_key_%08d", i) + value := ctx.GenerateValue(s.recordSize) + + // Measure individual write latency + writeStart := time.Now() + err := ctx.KV.Put(ctx.Context(), []byte(key), value) + writeLatency := time.Since(writeStart) + + if err != nil { + return fmt.Errorf("put failed at record %d: %w", i, err) + } + + // Track latencies + s.writeLatencies = append(s.writeLatencies, writeLatency) + latencyWindow = append(latencyWindow, writeLatency) + if len(latencyWindow) > windowSize { + latencyWindow = latencyWindow[1:] + } + + recordsSinceReport++ + + // Record potential flush by latency spike + if writeLatency > 50*time.Millisecond && !s.flushDetected { + log.Printf("[%s] Potentional flush detected at record %d (latency: %v)", + s.name, i, writeLatency) + s.flushDetected = true + } + + if i > 0 && i%1000 == 0 { + elapsed := time.Since(lastReportTime) + throughput := float64(recordsSinceReport) / elapsed.Seconds() + avgLatency := s.calculateAverage(latencyWindow) + p99Latency := s.calculateP99(latencyWindow) + + log.Printf("[%s] Progress: %d/%d records | Throughput: %.0f | Avg latency: %v | P99: %v", + s.name, i, s.recordCount, throughput, avgLatency, p99Latency) + + lastReportTime = time.Now() + recordsSinceReport = 0 + } + } + + totalDuration := time.Since(startTime) + overallThroughput := float64(s.recordCount) / totalDuration.Seconds() + + log.Printf("[%s] Sequential write test completed:", s.name) + log.Printf(" - Total time: %v", totalDuration) + log.Printf(" - Overall throughput: %.0f ops/s", overallThroughput) + log.Printf(" - Flush detected: %v", s.flushDetected) + + return nil +} + +func (s *MemtableStressScenario) calculateAverage(latencies []time.Duration) time.Duration { + if len(latencies) == 0 { + return 0 + } + var sum time.Duration + for _, lat := range latencies { + sum += lat + } + return sum / time.Duration(len(latencies)) +} + +func (s *MemtableStressScenario) calculateP99(latencies []time.Duration) time.Duration { + return s.calculatePercentile(latencies, 0.99) +} + +func (s *MemtableStressScenario) calculatePercentile(latencies []time.Duration, percentile float64) time.Duration { + if len(latencies) == 0 { + return 0 + } + + sorted := make([]time.Duration, len(latencies)) + copy(sorted, latencies) + slices.Sort(sorted) + + index := int(float64(len(sorted)) * percentile) + if index >= len(sorted) { + index = len(sorted) - 1 + } + + return sorted[index] +} From 63dc3b32aa47c140334eb8b1203861ccfb52391f Mon Sep 17 00:00:00 2001 From: lnikon Date: Sun, 19 Oct 2025 17:10:22 +0400 Subject: [PATCH 11/17] Integration & load testing: - Fix Raft unit tests - Add mise support --- CMakeLists.txt | 25 +- assets/database_config_schema.json | 175 ----- assets/tkvpp_config_1.json | 39 -- assets/tkvpp_config_2.json | 39 -- assets/tkvpp_config_3.json | 39 -- .../tkvpp_config_replicated_single_node.json | 37 -- assets/tkvpp_config_standalone.json | 39 -- .../tkvpp_config_replicated_single_node.json | 37 -- infra/assets/tkvpp_config_standalone.json | 6 +- lib/raft/raft.cpp | 139 ++-- lib/raft/raft.h | 11 +- lib/raft/raft_test.cpp | 605 +++++++++--------- lib/structures/lsmtree/levels/level.cpp | 4 +- src/main.cpp | 11 +- tests/go/kvtest/cmd/kvtest/kvtest.yml | 2 +- 15 files changed, 423 insertions(+), 785 deletions(-) delete mode 100644 assets/database_config_schema.json delete mode 100644 assets/tkvpp_config_1.json delete mode 100644 assets/tkvpp_config_2.json delete mode 100644 assets/tkvpp_config_3.json delete mode 100644 assets/tkvpp_config_replicated_single_node.json delete mode 100644 assets/tkvpp_config_standalone.json delete mode 100644 infra/assets/tkvpp_config_replicated_single_node.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 9637587..110b004 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,8 @@ set(CMAKE_CXX_EXTENSIONS Off) set(TINYKVPP_COMMON_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -pedantic-errors") set(TINYKVPP_COMMON_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic-errors") + "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic-errors" +) message("Using compiler: ${CMAKE_CXX_COMPILER_ID}") @@ -18,11 +19,11 @@ message("Using compiler: ${CMAKE_CXX_COMPILER_ID}") # Support thread annotations by Clang if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set(CMAKE_C_FLAGS "${TINYKVPP_COMMON_C_FLAGS} -Wthread-safety") - set(CMAKE_CXX_FLAGS "${TINYKVPP_COMMON_CXX_FLAGS} -Wthread-safety") + set(CMAKE_C_FLAGS "${TINYKVPP_COMMON_C_FLAGS} -Wthread-safety") + set(CMAKE_CXX_FLAGS "${TINYKVPP_COMMON_CXX_FLAGS} -Wthread-safety") else() - set(CMAKE_C_FLAGS "${TINYKVPP_COMMON_C_FLAGS}") - set(CMAKE_CXX_FLAGS "${TINYKVPP_COMMON_CXX_FLAGS}") + set(CMAKE_C_FLAGS "${TINYKVPP_COMMON_C_FLAGS}") + set(CMAKE_CXX_FLAGS "${TINYKVPP_COMMON_CXX_FLAGS}") endif() set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}") @@ -58,11 +59,13 @@ add_subdirectory(examples) # Custom targets to build docker images and run tests add_custom_target( - GCCReleaseDockerImage - COMMAND sh scripts/build_gcc_release_docker_image.sh - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + GCCReleaseDockerImage + COMMAND sh scripts/build_gcc_release_docker_image.sh + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) add_custom_target( - ClangReleaseDockerImage - COMMAND sh scripts/build_clang_release_docker_image.sh - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + ClangReleaseDockerImage + COMMAND sh scripts/build_clang_release_docker_image.sh + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) diff --git a/assets/database_config_schema.json b/assets/database_config_schema.json deleted file mode 100644 index 334a172..0000000 --- a/assets/database_config_schema.json +++ /dev/null @@ -1,175 +0,0 @@ -{ - "$id": "https://json-schema.hyperjump.io/schema", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$title": "Schema for frankie's JSON config", - "type": "object", - "properties": { - "logging": { - "type": "object", - "properties": { - "loggingLevel": { - "$ref": "#/$defs/loggingLevel" - } - }, - "required": [ - "loggingLevel" - ] - }, - "database": { - "type": "object", - "description": "Core database configuration settings", - "properties": { - "path": { - "type": "string", - "description": "Database storage directory path" - }, - "manifestFilenamePrefix": { - "type": "string", - "description": "Prefix for manifest files" - } - }, - "required": [ - "path", - "manifestFilenamePrefix" - ] - }, - "wal": { - "type": "object", - "description": "WAL configuration", - "properties": { - "enable": { - "type": "boolean", - "description": "Enable/disable the WAL" - }, - "storageType": { - "$ref": "#/$defs/walStorageType" - }, - "filename": { - "type": "string", - "description": "Write-Ahead Log filename" - } - }, - "required": [ - "enable", - "storageType", - "filename" - ] - }, - "lsm": { - "type": "object", - "properties": { - "flushThreshold": { - "type": "integer", - "description": "The threshold of bytes at which the memtable should be flushed", - "minimum": 1 - }, - "levelZeroCompaction": { - "$ref": "#/$defs/compaction" - }, - "levelNonZeroCompaction": { - "$ref": "#/$defs/compaction" - } - }, - "required": [ - "flushThreshold", - "maximumLevels", - "levelZeroCompaction", - "levelNonZeroCompaction" - ] - }, - "server": { - "type": "object", - "description": "Server configuration settings", - "properties": { - "transport": { - "$ref": "#/$defs/serverTransport" - }, - "host": { - "type": "string", - "description": "Server host address" - }, - "port": { - "type": "integer", - "description": "Server port number", - "minimum": 1024, - "maximum": 65535 - }, - "id": { - "type": "integer", - "description": "ID of the node", - "minimum": 1 - }, - "peers": { - "type": "array", - "description": "Array of IPv4 addresses of peers", - "items": { - "type": "string" - } - } - }, - "required": [ - "transport", - "host", - "port", - "id", - "peers" - ] - } - }, - "required": [ - "database", - "wal", - "lsm", - "server" - ], - "$defs": { - "serverTransport": { - "type": "string", - "enum": [ - "grpc", - "tcp" - ] - }, - "loggingLevel": { - "type": "string", - "enum": [ - "info", - "debug", - "trace", - "off" - ] - }, - "compactionStrategy": { - "type": "string", - "enum": [ - "levelled", - "tiered" - ] - }, - "walStorageType": { - "type": "string", - "enum": [ - "inMemory", - "persistent", - "replicated" - ] - }, - "compaction": { - "type": "object", - "properties": { - "compactionStrategy": { - "$ref": "#/$defs/compactionStrategy" - }, - "compactionThreshold": { - "type": "integer", - "description": "Number of files that trigger compaction", - "minimum": 1 - } - }, - "required": [ - "compactionStrategy", - "compactionThreshold" - ] - } - } -} diff --git a/assets/tkvpp_config_1.json b/assets/tkvpp_config_1.json deleted file mode 100644 index 67e003f..0000000 --- a/assets/tkvpp_config_1.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "logging": { - "loggingLevel": "info" - }, - "database": { - "path": "./var/tkvpp", - "walFilename": "wal", - "manifestFilenamePrefix": "manifest_", - "mode": "replicated" - }, - "wal": { - "enable": true, - "filename": "wal", - "storageType": "persistent" - }, - "lsm": { - "flushThreshold": 1024, - "maximumLevels": 12, - "levelZeroCompaction": { - "compactionStrategy": "levelled", - "compactionThreshold": 1024 - }, - "levelNonZeroCompaction": { - "compactionStrategy": "tiered", - "compactionThreshold": 1024 - } - }, - "server": { - "transport": "grpc", - "host": "0.0.0.0", - "port": 8081, - "id": 1, - "peers": [ - "0.0.0.0:8081", - "0.0.0.0:8082", - "0.0.0.0:8083" - ] - } -} diff --git a/assets/tkvpp_config_2.json b/assets/tkvpp_config_2.json deleted file mode 100644 index 41238c5..0000000 --- a/assets/tkvpp_config_2.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "logging": { - "loggingLevel": "info" - }, - "database": { - "path": "./var/tkvpp", - "walFilename": "wal", - "manifestFilenamePrefix": "manifest_", - "mode": "replicated" - }, - "wal": { - "enable": true, - "filename": "wal", - "storageType": "persistent" - }, - "lsm": { - "flushThreshold": 1024, - "maximumLevels": 12, - "levelZeroCompaction": { - "compactionStrategy": "levelled", - "compactionThreshold": 1024 - }, - "levelNonZeroCompaction": { - "compactionStrategy": "tiered", - "compactionThreshold": 1024 - } - }, - "server": { - "transport": "grpc", - "host": "0.0.0.0", - "port": 8082, - "id": 2, - "peers": [ - "0.0.0.0:8081", - "0.0.0.0:8082", - "0.0.0.0:8083" - ] - } -} diff --git a/assets/tkvpp_config_3.json b/assets/tkvpp_config_3.json deleted file mode 100644 index 636d0d8..0000000 --- a/assets/tkvpp_config_3.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "logging": { - "loggingLevel": "info" - }, - "database": { - "path": "./var/tkvpp", - "walFilename": "wal", - "manifestFilenamePrefix": "manifest_", - "mode": "replicated" - }, - "wal": { - "enable": true, - "filename": "wal", - "storageType": "persistent" - }, - "lsm": { - "flushThreshold": 1024, - "maximumLevels": 12, - "levelZeroCompaction": { - "compactionStrategy": "levelled", - "compactionThreshold": 1024 - }, - "levelNonZeroCompaction": { - "compactionStrategy": "tiered", - "compactionThreshold": 1024 - } - }, - "server": { - "transport": "grpc", - "host": "0.0.0.0", - "port": 8083, - "id": 3, - "peers": [ - "0.0.0.0:8081", - "0.0.0.0:8082", - "0.0.0.0:8083" - ] - } -} diff --git a/assets/tkvpp_config_replicated_single_node.json b/assets/tkvpp_config_replicated_single_node.json deleted file mode 100644 index f7a8bae..0000000 --- a/assets/tkvpp_config_replicated_single_node.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "logging": { - "loggingLevel": "debug" - }, - "database": { - "path": "./var/tkvpp", - "walFilename": "wal", - "manifestFilenamePrefix": "manifest_", - "mode": "replicated" - }, - "wal": { - "enable": true, - "filename": "wal", - "storageType": "persistent" - }, - "lsm": { - "flushThreshold": 1024, - "maximumLevels": 12, - "levelZeroCompaction": { - "compactionStrategy": "levelled", - "compactionThreshold": 1024 - }, - "levelNonZeroCompaction": { - "compactionStrategy": "tiered", - "compactionThreshold": 1024 - } - }, - "server": { - "transport": "grpc", - "host": "0.0.0.0", - "port": 8081, - "id": 1, - "peers": [ - "0.0.0.0:8081" - ] - } -} diff --git a/assets/tkvpp_config_standalone.json b/assets/tkvpp_config_standalone.json deleted file mode 100644 index 3d0ad19..0000000 --- a/assets/tkvpp_config_standalone.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "logging": { - "loggingLevel": "debug" - }, - "database": { - "path": "./var/tkvpp", - "walFilename": "wal", - "manifestFilenamePrefix": "manifest_", - "mode": "standalone" - }, - "wal": { - "enable": true, - "filename": "wal", - "storageType": "persistent" - }, - "lsm": { - "flushThreshold": 1024, - "maximumLevels": 12, - "levelZeroCompaction": { - "compactionStrategy": "levelled", - "compactionThreshold": 1024 - }, - "levelNonZeroCompaction": { - "compactionStrategy": "tiered", - "compactionThreshold": 1024 - } - }, - "server": { - "transport": "grpc", - "host": "0.0.0.0", - "port": 8081, - "id": 1, - "peers": [ - "0.0.0.0:8081", - "0.0.0.0:8082", - "0.0.0.0:8083" - ] - } -} diff --git a/infra/assets/tkvpp_config_replicated_single_node.json b/infra/assets/tkvpp_config_replicated_single_node.json deleted file mode 100644 index f7a8bae..0000000 --- a/infra/assets/tkvpp_config_replicated_single_node.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "logging": { - "loggingLevel": "debug" - }, - "database": { - "path": "./var/tkvpp", - "walFilename": "wal", - "manifestFilenamePrefix": "manifest_", - "mode": "replicated" - }, - "wal": { - "enable": true, - "filename": "wal", - "storageType": "persistent" - }, - "lsm": { - "flushThreshold": 1024, - "maximumLevels": 12, - "levelZeroCompaction": { - "compactionStrategy": "levelled", - "compactionThreshold": 1024 - }, - "levelNonZeroCompaction": { - "compactionStrategy": "tiered", - "compactionThreshold": 1024 - } - }, - "server": { - "transport": "grpc", - "host": "0.0.0.0", - "port": 8081, - "id": 1, - "peers": [ - "0.0.0.0:8081" - ] - } -} diff --git a/infra/assets/tkvpp_config_standalone.json b/infra/assets/tkvpp_config_standalone.json index f82596a..785af3a 100644 --- a/infra/assets/tkvpp_config_standalone.json +++ b/infra/assets/tkvpp_config_standalone.json @@ -27,11 +27,9 @@ }, "server": { "transport": "grpc", - "host": "0.0.0.0", + "host": "127.0.0.1", "port": 9891, "id": 1, - "peers": [ - "0.0.0.0:9891" - ] + "peers": [] } } diff --git a/lib/raft/raft.cpp b/lib/raft/raft.cpp index 1cac86d..e34c1fc 100644 --- a/lib/raft/raft.cpp +++ b/lib/raft/raft.cpp @@ -18,6 +18,7 @@ #include "raft/raft.h" #include "concurrency/thread_pool.h" +#include "config/config.h" #include "wal/wal.h" namespace @@ -25,9 +26,14 @@ namespace const std::string_view gRaftFilename = "RAFT_PERSISTENCE"; -auto constructFilename(std::string_view filename, std::uint32_t peerId) -> std::string +auto constructFilename( + config::shared_ptr_t pConfig, std::string_view filename, std::uint32_t peerId +) -> std::filesystem::path { - return fmt::format("/var/tkvpp/{}_NODE_{}", filename, peerId); + ASSERT(pConfig); + return fmt::format( + "{}/{}_NODE_{}", pConfig->DatabaseConfig.DatabasePath.c_str(), filename, peerId + ); } auto generate_random_timeout(const int minTimeout, const int maxTimeout) -> int @@ -155,11 +161,15 @@ auto raft_node_grpc_client_t::ip() const -> ip_t // consensus_module_t // ------------------------------- consensus_module_t::consensus_module_t( - node_config_t nodeConfig, std::vector replicas, wal_ptr_t pWal + config::shared_ptr_t pDatabaseConfig, + node_config_t nodeConfig, + std::vector replicas, + wal_ptr_t pWal ) noexcept : m_rpcThreadPool{std::make_unique(2, "raft_rpc")}, m_internalThreadPool{std::make_unique(2, "raft_internal")}, - m_config{std::move(nodeConfig)}, + m_pDatabaseConfig{std::move(pDatabaseConfig)}, + m_nodeConfig{std::move(nodeConfig)}, m_currentTerm{0}, m_votedFor{0}, m_log{std::move(pWal)}, @@ -169,8 +179,8 @@ consensus_module_t::consensus_module_t( m_leaderHeartbeatReceived{false}, m_voteCount{0} { - ASSERT(m_config.m_id != 0); - ASSERT(m_config.m_id <= replicas.size() + 1); + ASSERT(m_nodeConfig.m_id != 0); + ASSERT(m_nodeConfig.m_id <= replicas.size() + 1); for (auto &&nodeClient : replicas) { @@ -270,7 +280,7 @@ auto consensus_module_t::AppendEntries( spdlog::debug( "Node={} recevied AppendEntries RPC from leader={} during term={}", - m_config.m_id, + m_nodeConfig.m_id, pRequest->sender_id(), pRequest->term() ); @@ -287,9 +297,9 @@ auto consensus_module_t::AppendEntries( } } - if (leaderIp.empty() && pRequest->sender_id() == m_config.m_id) + if (leaderIp.empty() && pRequest->sender_id() == m_nodeConfig.m_id) { - leaderIp = m_config.m_ip; + leaderIp = m_nodeConfig.m_ip; } if (!leaderIp.empty()) @@ -306,13 +316,13 @@ auto consensus_module_t::AppendEntries( { spdlog::debug( "Node={} receviedTerm={} is smaller than currentTerm={}", - m_config.m_id, + m_nodeConfig.m_id, pRequest->term(), m_currentTerm ); pResponse->set_term(m_currentTerm); pResponse->set_success(false); - pResponse->set_responder_id(m_config.m_id); + pResponse->set_responder_id(m_nodeConfig.m_id); return grpc::Status::OK; } @@ -320,8 +330,8 @@ auto consensus_module_t::AppendEntries( { spdlog::debug( "Node={} receviedTerm={} is higher than currentTerm={}. " - "Reverting to follower", - m_config.m_id, + "Reverting to follower.", + m_nodeConfig.m_id, pRequest->term(), m_currentTerm ); @@ -336,14 +346,14 @@ auto consensus_module_t::AppendEntries( { spdlog::error( "Node={} received prevlogindex={} which does not exist. logSize={}", - m_config.m_id, + m_nodeConfig.m_id, pRequest->prev_log_index(), m_log->size() ); pResponse->set_term(m_currentTerm); pResponse->set_success(false); - pResponse->set_responder_id(m_config.m_id); + pResponse->set_responder_id(m_nodeConfig.m_id); return grpc::Status::OK; } @@ -352,7 +362,7 @@ auto consensus_module_t::AppendEntries( spdlog::error( "Node={} received prevlogindex={} with mismatched prevlogterm={}. Current log " "term={}", - m_config.m_id, + m_nodeConfig.m_id, pRequest->prev_log_index(), pRequest->prev_log_term(), logEntry->term() @@ -360,7 +370,7 @@ auto consensus_module_t::AppendEntries( pResponse->set_term(m_currentTerm); pResponse->set_success(false); - pResponse->set_responder_id(m_config.m_id); + pResponse->set_responder_id(m_nodeConfig.m_id); return grpc::Status::OK; } } @@ -397,10 +407,12 @@ auto consensus_module_t::AppendEntries( std::min(pRequest->leader_commit(), (uint32_t)m_log->size()), std::nullopt )) { - spdlog::error("Node={} is unable to persist commitIndex", m_config.m_id, m_commitIndex); + spdlog::error( + "Node={} is unable to persist commitIndex", m_nodeConfig.m_id, m_commitIndex + ); pResponse->set_term(m_currentTerm); pResponse->set_success(false); - pResponse->set_responder_id(m_config.m_id); + pResponse->set_responder_id(m_nodeConfig.m_id); return grpc::Status::OK; } } @@ -409,7 +421,7 @@ auto consensus_module_t::AppendEntries( { spdlog::info( "Follower Node={} PREPARING applying m_lastApplied+1={}, m_commitIndex={}", - m_config.m_id, + m_nodeConfig.m_id, m_lastApplied + 1, m_commitIndex ); @@ -420,20 +432,22 @@ auto consensus_module_t::AppendEntries( pResponse->set_term(m_currentTerm); pResponse->set_success(true); - pResponse->set_responder_id(m_config.m_id); + pResponse->set_responder_id(m_nodeConfig.m_id); pResponse->set_match_index(m_log->size()); // Update @m_votedFor if (!updatePersistentState(std::nullopt, pRequest->sender_id())) { - spdlog::error("Node={} is unable to persist votedFor", m_config.m_id, m_votedFor); + spdlog::error("Node={} is unable to persist votedFor", m_nodeConfig.m_id, m_votedFor); } { m_leaderHeartbeatReceived.store(true); } - spdlog::debug("Node={} is resetting election timeout at term={}", m_config.m_id, m_currentTerm); + spdlog::debug( + "Node={} is resetting election timeout at term={}", m_nodeConfig.m_id, m_currentTerm + ); return grpc::Status::OK; } @@ -453,7 +467,7 @@ auto consensus_module_t::RequestVote( spdlog::debug( "Node={} received RequestVote RPC from candidate={} during " "term={} peerTerm={}", - m_config.m_id, + m_nodeConfig.m_id, pRequest->candidate_id(), m_currentTerm, pRequest->term() @@ -461,7 +475,7 @@ auto consensus_module_t::RequestVote( pResponse->set_term(m_currentTerm); pResponse->set_vote_granted(0); - pResponse->set_responder_id(m_config.m_id); + pResponse->set_responder_id(m_nodeConfig.m_id); // Become follower if candidates has higher term if (pRequest->term() > m_currentTerm) @@ -489,10 +503,12 @@ auto consensus_module_t::RequestVote( { if (!updatePersistentState(std::nullopt, pRequest->candidate_id())) { - spdlog::error("Node={} is unable to persist votedFor", m_config.m_id, m_votedFor); + spdlog::error( + "Node={} is unable to persist votedFor", m_nodeConfig.m_id, m_votedFor + ); } - spdlog::info("Node={} votedFor={}", m_config.m_id, pRequest->candidate_id()); + spdlog::info("Node={} votedFor={}", m_nodeConfig.m_id, pRequest->candidate_id()); pResponse->set_term(m_currentTerm); pResponse->set_vote_granted(1); @@ -652,14 +668,14 @@ auto consensus_module_t::getState() const -> raft::v1::NodeState void consensus_module_t::becomeFollower(uint32_t newTerm) { m_currentTerm = newTerm; - m_state = raft::v1::NodeState::NODE_STATE_LEADER; - spdlog::info("Node={} reverted to follower state in term={}", m_config.m_id, m_currentTerm); + m_state = raft::v1::NodeState::NODE_STATE_FOLLOWER; + spdlog::info("Node={} reverted to follower state in term={}", m_nodeConfig.m_id, m_currentTerm); cleanupHeartbeatThread(); if (!updatePersistentState(std::nullopt, 0)) { - spdlog::error("Node={} is unable to persist votedFor={}", m_config.m_id, m_votedFor); + spdlog::error("Node={} is unable to persist votedFor={}", m_nodeConfig.m_id, m_votedFor); } if (m_onLeaderChangeCbk) @@ -673,7 +689,7 @@ void consensus_module_t::becomeLeader() { if (m_state == raft::v1::NodeState::NODE_STATE_LEADER) { - spdlog::warn("Node={} is already a leader", m_config.m_id); + spdlog::warn("Node={} is already a leader", m_nodeConfig.m_id); return; } @@ -682,7 +698,7 @@ void consensus_module_t::becomeLeader() { absl::WriterMutexLock locker{&m_leaderMutex}; - m_currentLeaderHint = m_config.m_ip; + m_currentLeaderHint = m_nodeConfig.m_ip; } // TODO(lnikon): Should I reinit m_nextIndex and m_matchIndex here again? @@ -690,7 +706,7 @@ void consensus_module_t::becomeLeader() cleanupHeartbeatThread(); m_heartbeatThread = std::jthread([this](std::stop_token token) { runHeartbeatThread(token); }); - spdlog::info("Node={} become a leader at term={}", m_config.m_id, m_currentTerm); + spdlog::info("Node={} become a leader at term={}", m_nodeConfig.m_id, m_currentTerm); if (m_onLeaderChangeCbk) { @@ -760,7 +776,7 @@ auto consensus_module_t::waitForHeartbeat(std::stop_token token) -> bool spdlog::debug( "Timer thread at node={} will block for {}ms for the leader " "to send a heartbeat", - m_config.m_id, + m_nodeConfig.m_id, timeoutMs ); @@ -772,7 +788,7 @@ auto consensus_module_t::waitForHeartbeat(std::stop_token token) -> bool if (token.stop_requested()) { - spdlog::info("Node={} election thread terminating due to stop request", m_config.m_id); + spdlog::info("Node={} election thread terminating due to stop request", m_nodeConfig.m_id); return true; } @@ -781,7 +797,7 @@ auto consensus_module_t::waitForHeartbeat(std::stop_token token) -> bool // leader election if (signaled && m_leaderHeartbeatReceived.load()) { - spdlog::debug("Node={} received heartbeat from", m_config.m_id); + spdlog::debug("Node={} received heartbeat from", m_nodeConfig.m_id); m_leaderHeartbeatReceived.store(false); return true; } @@ -838,7 +854,7 @@ void consensus_module_t::sendAppendEntriesAsync( } request.set_leader_commit(m_commitIndex); - request.set_sender_id(m_config.m_id); + request.set_sender_id(m_nodeConfig.m_id); // Add log entries starting from nextIndex for (const auto &logEntrie : logEntries) @@ -909,19 +925,19 @@ auto consensus_module_t::onSendAppendEntriesRPC( if (m_log->size() >= majorityIndex && majorityIndex > m_commitIndex) { - spdlog::info("Node={} found majorityIndex={}", m_config.m_id, majorityIndex); + spdlog::info("Node={} found majorityIndex={}", m_nodeConfig.m_id, majorityIndex); if (m_log->read(majorityIndex - 1)->term() == m_currentTerm) { spdlog::info( "Node={} is updating commitIndex to majorityIndex={}", - m_config.m_id, + m_nodeConfig.m_id, majorityIndex ); if (!updatePersistentState(majorityIndex, std::nullopt)) { spdlog::error( - "Node={} unable to persist commitIndex={}", m_config.m_id, m_commitIndex + "Node={} unable to persist commitIndex={}", m_nodeConfig.m_id, m_commitIndex ); // TODO(lnikon): Returning false here is dangerous, because // is it non-distinguishable from nextIndex related issues @@ -939,7 +955,7 @@ auto consensus_module_t::onSendAppendEntriesRPC( spdlog::warn( "Node={} received AppendEntriesResponse from peer={} " "at term={} success={} match_index={}", - m_config.m_id, + m_nodeConfig.m_id, client.id(), response.term(), response.success(), @@ -988,27 +1004,30 @@ void consensus_module_t::startElection() newTerm = ++m_currentTerm; m_state = raft::v1::NodeState::NODE_STATE_CANDIDATE; - spdlog::info("Node={} starts election. New term={}", m_config.m_id, m_currentTerm); + spdlog::info("Node={} starts election. New term={}", m_nodeConfig.m_id, m_currentTerm); // Node in a canditate state should vote for itself. m_voteCount++; - if (!updatePersistentState(std::nullopt, m_config.m_id)) + if (!updatePersistentState(std::nullopt, m_nodeConfig.m_id)) { - spdlog::error("Node={} is unable to persist votedFor={}", m_config.m_id, m_votedFor); + spdlog::error( + "Node={} is unable to persist votedFor={}", m_nodeConfig.m_id, m_votedFor + ); } // Early majority check for a single node cluster if (hasMajority(m_voteCount.load())) { spdlog::info( - "Node={} achieved majority (single node cluster), becoming leader", m_config.m_id + "Node={} achieved majority (single node cluster), becoming leader", + m_nodeConfig.m_id ); becomeLeader(); return; } request.set_term(m_currentTerm); - request.set_candidate_id(m_config.m_id); + request.set_candidate_id(m_nodeConfig.m_id); request.set_last_log_term(getLastLogTerm()); request.set_last_log_index(getLastLogIndex()); } @@ -1027,7 +1046,7 @@ void consensus_module_t::sendRequestVoteRPCs( spdlog::debug( "Node={} is creating RequestVoteRPC thread for the " "peer={} during term={}", - m_config.m_id, + m_nodeConfig.m_id, id, newTerm ); @@ -1038,7 +1057,7 @@ void consensus_module_t::sendRequestVoteRPCs( for (auto &thread : requesterThreads) { - spdlog::debug("Node={} is joining RequestVoteRPC thread", m_config.m_id); + spdlog::debug("Node={} is joining RequestVoteRPC thread", m_nodeConfig.m_id); if (thread.joinable()) { thread.join(); @@ -1134,7 +1153,7 @@ auto consensus_module_t::findMajorityIndexMatch() const -> uint32_t for (const auto &[peer, matchIdx] : m_matchIndex) { - spdlog::debug("Node={} peer={} matchIdx={}", m_config.m_id, peer, matchIdx); + spdlog::debug("Node={} peer={} matchIdx={}", m_nodeConfig.m_id, peer, matchIdx); matchIndexes.emplace_back(matchIdx); } @@ -1161,7 +1180,7 @@ auto consensus_module_t::waitForMajorityReplication(uint32_t logIndex) const -> }; spdlog::info( - "Node={} is waiting for majority to agree on logIndex={}", m_config.m_id, logIndex + "Node={} is waiting for majority to agree on logIndex={}", m_nodeConfig.m_id, logIndex ); const auto timeout{absl::Seconds(generate_raft_timeout())}; @@ -1201,14 +1220,16 @@ auto consensus_module_t::updatePersistentState( auto consensus_module_t::flushPersistentState() -> bool { - auto path = std::filesystem::path(constructFilename(gRaftFilename, m_config.m_id)); + auto path = std::filesystem::path( + constructFilename(m_pDatabaseConfig, gRaftFilename, m_nodeConfig.m_id) + ); std::ofstream fsa(path, std::fstream::out | std::fstream::trunc); if (!fsa.is_open()) { spdlog::error( "Node={} is unable to open {} to flush " "commitIndex={} and votedFor={}", - m_config.m_id, + m_nodeConfig.m_id, path.c_str(), m_commitIndex, m_votedFor @@ -1223,7 +1244,7 @@ auto consensus_module_t::flushPersistentState() -> bool spdlog::error( "Node={} is unable to write into {} the " "currentTerm={}, commitIndex={} and votedFor={}", - m_config.m_id, + m_nodeConfig.m_id, path.c_str(), m_currentTerm, m_commitIndex, @@ -1238,10 +1259,10 @@ auto consensus_module_t::flushPersistentState() -> bool auto consensus_module_t::restorePersistentState() -> bool { { - auto path = std::filesystem::path(constructFilename(gRaftFilename, m_config.m_id)); + auto path{constructFilename(m_pDatabaseConfig, gRaftFilename, m_nodeConfig.m_id)}; if (!std::filesystem::exists(path)) { - spdlog::info("Node={} is running the first time", m_config.m_id); + spdlog::info("Node={} is running the first time", m_nodeConfig.m_id); return true; } @@ -1251,7 +1272,7 @@ auto consensus_module_t::restorePersistentState() -> bool spdlog::error( "Node={} is unable to open {} to restore commitIndex " "and votedFor", - m_config.m_id, + m_nodeConfig.m_id, path.c_str() ); return false; @@ -1260,7 +1281,7 @@ auto consensus_module_t::restorePersistentState() -> bool ifs >> m_currentTerm >> m_commitIndex >> m_votedFor; spdlog::info( "Node={} restored currentTerm={}, commitIndex={} and votedFor={}", - m_config.m_id, + m_nodeConfig.m_id, m_commitIndex, m_commitIndex, m_votedFor @@ -1377,10 +1398,10 @@ void consensus_module_t::cleanupHeartbeatThread() { if (m_heartbeatThread.joinable()) { - spdlog::info("Node={} clearing existing heartbeat threads", m_config.m_id); + spdlog::info("Node={} clearing existing heartbeat threads", m_nodeConfig.m_id); m_heartbeatThread.request_stop(); m_heartbeatThread.join(); - spdlog::info("Node={} existing heartbeat threads cleared", m_config.m_id); + spdlog::info("Node={} existing heartbeat threads cleared", m_nodeConfig.m_id); } } diff --git a/lib/raft/raft.h b/lib/raft/raft.h index 801014d..8b62bc3 100644 --- a/lib/raft/raft.h +++ b/lib/raft/raft.h @@ -21,6 +21,7 @@ #include #include +#include "config/config.h" #include "raft/v1/raft_service.grpc.pb.h" #include "raft/v1/raft_service.pb.h" #include "concurrency/thread_pool.h" @@ -129,7 +130,10 @@ class consensus_module_t final : public raft::v1::RaftService::Service consensus_module_t() = delete; consensus_module_t( - node_config_t nodeConfig, std::vector replicas, wal_ptr_t pWal + config::shared_ptr_t pConfig, + node_config_t nodeConfig, + std::vector replicas, + wal_ptr_t pWal ) noexcept; consensus_module_t(const consensus_module_t &) = delete; @@ -263,8 +267,11 @@ class consensus_module_t final : public raft::v1::RaftService::Service mutable absl::Mutex m_leaderMutex; std::string m_currentLeaderHint ABSL_GUARDED_BY(m_leaderMutex); + // Database-wide config. + config::shared_ptr_t m_pDatabaseConfig; + // Stores ID and IP of the current node. Received from outside. - node_config_t m_config; + node_config_t m_nodeConfig; // Executed on follower node after successful log replication to update the state machine. on_commit_cbk_t m_onCommitCbk; diff --git a/lib/raft/raft_test.cpp b/lib/raft/raft_test.cpp index c884bfb..a5a42dd 100644 --- a/lib/raft/raft_test.cpp +++ b/lib/raft/raft_test.cpp @@ -1,18 +1,20 @@ #include #include #include +#include #include #include #include #include #include +#include "config/config.h" #include "raft.h" #include "wal/wal.h" -#include "Raft_mock.grpc.pb.h" // Generated mock stubs +#include "raft/v1/raft_service_mock.grpc.pb.h" // Generated mock stubs using namespace testing; -using namespace raft; +using namespace consensus; // Test fixtures class RaftNodeGrpcClientTest : public ::testing::Test @@ -22,42 +24,52 @@ class RaftNodeGrpcClientTest : public ::testing::Test { config.m_id = 1; config.m_ip = "127.0.0.1:5000"; - mockStub = std::make_unique(); + mockStub = std::make_unique(); mockStubPtr = mockStub.get(); } - node_config_t config; - std::unique_ptr mockStub; - MockRaftServiceStub *mockStubPtr; + node_config_t config; + std::unique_ptr mockStub; + raft::v1::MockRaftServiceStub *mockStubPtr; }; class ConsensusModuleTest : public ::testing::Test { protected: + config::shared_ptr_t pDbConfig{nullptr}; + consensus::node_config_t nodeConfig; + consensus::consensus_module_t::wal_ptr_t walPtr; + std::filesystem::path walPath; + void SetUp() override { nodeConfig.m_id = 1; nodeConfig.m_ip = "127.0.0.1:5001"; + pDbConfig = config::make_shared(); + pDbConfig->DatabaseConfig.DatabasePath = "./var/tkvpp/"; + // Create directories for test files - std::filesystem::create_directories("./var/tkvpp/"); + std::filesystem::create_directories(pDbConfig->DatabaseConfig.DatabasePath); // Create real WAL instance walPath = std::filesystem::temp_directory_path() / ("test_wal_" + std::to_string(nodeConfig.m_id) + ".log"); - auto walOpt = wal::wal_builder_t{}.set_file_path(walPath).build( - wal::log_storage_type_k::in_memory_k - ); - + auto walOpt = + wal::wal_builder_t{}.set_file_path(walPath).build( + wal::log_storage_type_k::in_memory_k + ); ASSERT_TRUE(walOpt.has_value()); - walPtr = std::make_shared>(std::move(walOpt.value())); + walPtr = std::make_shared>( + std::move(walOpt.value()) + ); } void TearDown() override { // Clean up any persistent state files - std::filesystem::remove_all("./var/tkvpp/"); + std::filesystem::remove_all(pDbConfig->DatabaseConfig.DatabasePath); if (std::filesystem::exists(walPath)) { std::filesystem::remove(walPath); @@ -67,39 +79,39 @@ class ConsensusModuleTest : public ::testing::Test auto createConsensusModule(std::vector replicas = {}) -> std::unique_ptr { - return std::make_unique(nodeConfig, std::move(replicas), walPtr); + return std::make_unique( + pDbConfig, nodeConfig, std::move(replicas), walPtr + ); } auto createConsensusModuleWithPersistentWAL(std::vector replicas = {}) -> std::unique_ptr { - auto persistentWalOpt = wal::wal_builder_t{}.set_file_path(walPath).build( - wal::log_storage_type_k::file_based_persistent_k - ); + auto persistentWalOpt = + wal::wal_builder_t{}.set_file_path(walPath).build( + wal::log_storage_type_k::file_based_persistent_k + ); if (!persistentWalOpt.has_value()) { return nullptr; } - auto persistentWalPtr = - std::make_shared>(std::move(persistentWalOpt.value())); + auto persistentWalPtr = std::make_shared>( + std::move(persistentWalOpt.value()) + ); return std::make_unique( - nodeConfig, std::move(replicas), persistentWalPtr + pDbConfig, nodeConfig, std::move(replicas), persistentWalPtr ); } - auto createMockClient(id_t id, const std::string &ip) -> raft_node_grpc_client_t + auto createMockClient(id_t id, const std::string &ip) -> consensus::raft_node_grpc_client_t { node_config_t clientConfig{.m_id = id, .m_ip = ip}; - auto mockStub = std::make_unique(); + auto mockStub = std::make_unique(); return {clientConfig, std::move(mockStub)}; } - - node_config_t nodeConfig; - std::shared_ptr> walPtr; - std::filesystem::path walPath; }; // ============================================================================ @@ -119,7 +131,7 @@ TEST_F(RaftNodeGrpcClientTest, AppendEntriesSuccess) EXPECT_CALL(*mockStubPtr, AppendEntries(_, _, _)) .WillOnce(DoAll( WithArg<2>( - [](AppendEntriesResponse *response) + [](raft::v1::AppendEntriesResponse *response) { response->set_success(true); response->set_term(1); @@ -128,9 +140,9 @@ TEST_F(RaftNodeGrpcClientTest, AppendEntriesSuccess) Return(grpc::Status::OK) )); - raft_node_grpc_client_t client(config, std::move(mockStub)); - AppendEntriesRequest request; - AppendEntriesResponse response; + raft_node_grpc_client_t client(config, std::move(mockStub)); + raft::v1::AppendEntriesRequest request; + raft::v1::AppendEntriesResponse response; bool result = client.appendEntries(request, &response); @@ -144,9 +156,9 @@ TEST_F(RaftNodeGrpcClientTest, AppendEntriesFailure) EXPECT_CALL(*mockStubPtr, AppendEntries(_, _, _)) .WillOnce(Return(grpc::Status(grpc::StatusCode::UNAVAILABLE, "Service unavailable"))); - raft_node_grpc_client_t client(config, std::move(mockStub)); - AppendEntriesRequest request; - AppendEntriesResponse response; + raft_node_grpc_client_t client(config, std::move(mockStub)); + raft::v1::AppendEntriesRequest request; + raft::v1::AppendEntriesResponse response; bool result = client.appendEntries(request, &response); @@ -158,43 +170,43 @@ TEST_F(RaftNodeGrpcClientTest, RequestVoteSuccess) EXPECT_CALL(*mockStubPtr, RequestVote(_, _, _)) .WillOnce(DoAll( WithArg<2>( - [](RequestVoteResponse *response) + [](raft::v1::RequestVoteResponse *response) { - response->set_votegranted(1); + response->set_vote_granted(1); response->set_term(2); } ), Return(grpc::Status::OK) )); - raft_node_grpc_client_t client(config, std::move(mockStub)); - RequestVoteRequest request; - RequestVoteResponse response; + raft_node_grpc_client_t client(config, std::move(mockStub)); + raft::v1::RequestVoteRequest request; + raft::v1::RequestVoteResponse response; bool result = client.requestVote(request, &response); EXPECT_TRUE(result); - EXPECT_EQ(response.votegranted(), 1); + EXPECT_EQ(response.vote_granted(), 1); EXPECT_EQ(response.term(), 2); } -TEST_F(RaftNodeGrpcClientTest, ReplicateSuccess) -{ - EXPECT_CALL(*mockStubPtr, Replicate(_, _, _)) - .WillOnce(DoAll( - WithArg<2>([](ReplicateEntriesResponse *response) { response->set_status("OK"); }), - Return(grpc::Status::OK) - )); - - raft_node_grpc_client_t client(config, std::move(mockStub)); - ReplicateEntriesRequest request; - ReplicateEntriesResponse response; - - bool result = client.replicate(request, &response); - - EXPECT_TRUE(result); - EXPECT_EQ(response.status(), "OK"); -} +// TEST_F(RaftNodeGrpcClientTest, ReplicateSuccess) +// { +// EXPECT_CALL(*mockStubPtr, Replicate(_, _, _)) +// .WillOnce(DoAll( +// WithArg<2>([](ReplicateEntriesResponse *response) { response->set_status("OK"); }), +// Return(grpc::Status::OK) +// )); +// +// raft_node_grpc_client_t client(config, std::move(mockStub)); +// ReplicateEntriesRequest request; +// ReplicateEntriesResponse response; +// +// bool result = client.replicate(request, &response); +// +// EXPECT_TRUE(result); +// EXPECT_EQ(response.status(), "OK"); +// } // ============================================================================ // consensus_module_t Basic Tests @@ -210,7 +222,7 @@ TEST_F(ConsensusModuleTest, Constructor) EXPECT_EQ(consensus->currentTerm(), 0); EXPECT_EQ(consensus->votedFor(), 0); - EXPECT_EQ(consensus->getStateSafe(), NodeState::FOLLOWER); + EXPECT_EQ(consensus->getStateSafe(), raft::v1::NodeState::NODE_STATE_FOLLOWER); } TEST_F(ConsensusModuleTest, InitializationSuccess) @@ -241,7 +253,7 @@ TEST_F(ConsensusModuleTest, WALBasicOperations) EXPECT_EQ(walPtr->size(), 0); EXPECT_TRUE(walPtr->empty()); - LogEntry entry; + consensus_module_t::wal_entry_t entry; entry.set_term(1); entry.set_index(1); entry.set_payload("test payload"); @@ -271,7 +283,7 @@ TEST_F(ConsensusModuleTest, SingleNodeBecomesLeader) // Wait for election timeout and becoming leader std::this_thread::sleep_for(std::chrono::milliseconds(500)); - EXPECT_EQ(consensus->getStateSafe(), NodeState::LEADER); + EXPECT_EQ(consensus->getStateSafe(), raft::v1::NodeState::NODE_STATE_LEADER); EXPECT_GT(consensus->currentTerm(), 0); consensus->stop(); @@ -282,19 +294,19 @@ TEST_F(ConsensusModuleTest, RequestVoteWithHigherTerm) auto consensus = createConsensusModule(); ASSERT_TRUE(consensus->init()); - RequestVoteRequest request; + raft::v1::RequestVoteRequest request; request.set_term(5); - request.set_candidateid(2); - request.set_lastlogindex(0); - request.set_lastlogterm(0); + request.set_candidate_id(2); + request.set_last_log_index(0); + request.set_last_log_term(0); - RequestVoteResponse response; - grpc::ServerContext context; + raft::v1::RequestVoteResponse response; + grpc::ServerContext context; auto status = consensus->RequestVote(&context, &request, &response); EXPECT_TRUE(status.ok()); - EXPECT_EQ(response.votegranted(), 1); + EXPECT_EQ(response.vote_granted(), 1); EXPECT_EQ(consensus->currentTerm(), 5); EXPECT_EQ(consensus->votedFor(), 2); } @@ -305,30 +317,30 @@ TEST_F(ConsensusModuleTest, RequestVoteWithLowerTerm) ASSERT_TRUE(consensus->init()); // Manually set higher term - RequestVoteRequest initialRequest; + raft::v1::RequestVoteRequest initialRequest; initialRequest.set_term(5); - initialRequest.set_candidateid(2); - initialRequest.set_lastlogindex(0); - initialRequest.set_lastlogterm(0); + initialRequest.set_candidate_id(2); + initialRequest.set_last_log_index(0); + initialRequest.set_last_log_term(0); - RequestVoteResponse initialResponse; - grpc::ServerContext initialContext; + raft::v1::RequestVoteResponse initialResponse; + grpc::ServerContext initialContext; consensus->RequestVote(&initialContext, &initialRequest, &initialResponse); // Now request with lower term - RequestVoteRequest request; + raft::v1::RequestVoteRequest request; request.set_term(3); - request.set_candidateid(3); - request.set_lastlogindex(0); - request.set_lastlogterm(0); + request.set_candidate_id(3); + request.set_last_log_index(0); + request.set_last_log_term(0); - RequestVoteResponse response; - grpc::ServerContext context; + raft::v1::RequestVoteResponse response; + grpc::ServerContext context; auto status = consensus->RequestVote(&context, &request, &response); EXPECT_TRUE(status.ok()); - EXPECT_EQ(response.votegranted(), 0); + EXPECT_EQ(response.vote_granted(), 0); EXPECT_EQ(response.term(), 5); } @@ -338,32 +350,32 @@ TEST_F(ConsensusModuleTest, RequestVoteAlreadyVoted) ASSERT_TRUE(consensus->init()); // First vote - RequestVoteRequest request1; + raft::v1::RequestVoteRequest request1; request1.set_term(5); - request1.set_candidateid(2); - request1.set_lastlogindex(0); - request1.set_lastlogterm(0); + request1.set_candidate_id(2); + request1.set_last_log_index(0); + request1.set_last_log_term(0); - RequestVoteResponse response1; - grpc::ServerContext context1; + raft::v1::RequestVoteResponse response1; + grpc::ServerContext context1; consensus->RequestVote(&context1, &request1, &response1); - EXPECT_EQ(response1.votegranted(), 1); + EXPECT_EQ(response1.vote_granted(), 1); // Second vote from different candidate - RequestVoteRequest request2; + raft::v1::RequestVoteRequest request2; request2.set_term(5); - request2.set_candidateid(3); - request2.set_lastlogindex(0); - request2.set_lastlogterm(0); + request2.set_candidate_id(3); + request2.set_last_log_index(0); + request2.set_last_log_term(0); - RequestVoteResponse response2; - grpc::ServerContext context2; + raft::v1::RequestVoteResponse response2; + grpc::ServerContext context2; auto status = consensus->RequestVote(&context2, &request2, &response2); EXPECT_TRUE(status.ok()); - EXPECT_EQ(response2.votegranted(), 0); + EXPECT_EQ(response2.vote_granted(), 0); } // ============================================================================ @@ -375,21 +387,21 @@ TEST_F(ConsensusModuleTest, AppendEntriesBasic) auto consensus = createConsensusModule(); ASSERT_TRUE(consensus->init()); - AppendEntriesRequest request; + raft::v1::AppendEntriesRequest request; request.set_term(1); - request.set_senderid(2); - request.set_prevlogindex(0); - request.set_prevlogterm(0); - request.set_leadercommit(0); + request.set_sender_id(2); + request.set_prev_log_index(0); + request.set_prev_log_term(0); + request.set_leader_commit(0); // Add a log entry - LogEntry *entry = request.add_entries(); + raft::v1::LogEntry *entry = request.add_entries(); entry->set_term(1); entry->set_index(1); entry->set_payload("test payload"); - AppendEntriesResponse response; - grpc::ServerContext context; + raft::v1::AppendEntriesResponse response; + grpc::ServerContext context; auto status = consensus->AppendEntries(&context, &request, &response); @@ -407,7 +419,7 @@ TEST_F(ConsensusModuleTest, AppendEntriesBasic) TEST_F(ConsensusModuleTest, AppendEntriesLogInconsistency) { // Pre-populate WAL with an entry - LogEntry existingEntry; + raft::v1::LogEntry existingEntry; existingEntry.set_term(1); existingEntry.set_index(1); existingEntry.set_payload("existing"); @@ -417,15 +429,15 @@ TEST_F(ConsensusModuleTest, AppendEntriesLogInconsistency) auto consensus = createConsensusModule(); ASSERT_TRUE(consensus->init()); - AppendEntriesRequest request; + raft::v1::AppendEntriesRequest request; request.set_term(2); - request.set_senderid(2); - request.set_prevlogindex(1); - request.set_prevlogterm(2); // Mismatch - existing entry has term 1 - request.set_leadercommit(0); + request.set_sender_id(2); + request.set_prev_log_index(1); + request.set_prev_log_term(2); // Mismatch - existing entry has term 1 + request.set_leader_commit(0); - AppendEntriesResponse response; - grpc::ServerContext context; + raft::v1::AppendEntriesResponse response; + grpc::ServerContext context; auto status = consensus->AppendEntries(&context, &request, &response); @@ -435,47 +447,48 @@ TEST_F(ConsensusModuleTest, AppendEntriesLogInconsistency) TEST_F(ConsensusModuleTest, AppendEntriesWithHigherTerm) { + spdlog::set_level(spdlog::level::debug); auto consensus = createConsensusModule(); ASSERT_TRUE(consensus->init()); - AppendEntriesRequest request; + raft::v1::AppendEntriesRequest request; request.set_term(5); - request.set_senderid(2); - request.set_prevlogindex(0); - request.set_prevlogterm(0); - request.set_leadercommit(0); + request.set_sender_id(2); + request.set_prev_log_index(0); + request.set_prev_log_term(0); + request.set_leader_commit(0); - AppendEntriesResponse response; - grpc::ServerContext context; + raft::v1::AppendEntriesResponse response; + grpc::ServerContext context; auto status = consensus->AppendEntries(&context, &request, &response); EXPECT_TRUE(status.ok()); EXPECT_TRUE(response.success()); EXPECT_EQ(consensus->currentTerm(), 5); - EXPECT_EQ(consensus->getStateSafe(), NodeState::FOLLOWER); + EXPECT_EQ(consensus->getStateSafe(), raft::v1::NodeState::NODE_STATE_FOLLOWER); } // ============================================================================ // Client Replication Tests // ============================================================================ -TEST_F(ConsensusModuleTest, ReplicateAsFollower) -{ - auto consensus = createConsensusModule(); - ASSERT_TRUE(consensus->init()); - - ReplicateEntriesRequest request; - request.add_payloads("test payload"); - - ReplicateEntriesResponse response; - grpc::ServerContext context; - - auto status = consensus->Replicate(&context, &request, &response); - - EXPECT_TRUE(status.ok()); - EXPECT_EQ(response.status(), "FAILED"); -} +// TEST_F(ConsensusModuleTest, ReplicateAsFollower) +// { +// auto consensus = createConsensusModule(); +// ASSERT_TRUE(consensus->init()); +// +// ReplicateEntriesRequest request; +// request.add_payloads("test payload"); +// +// ReplicateEntriesResponse response; +// grpc::ServerContext context; +// +// auto status = consensus->Replicate(&context, &request, &response); +// +// EXPECT_TRUE(status.ok()); +// EXPECT_EQ(response.status(), "FAILED"); +// } TEST_F(ConsensusModuleTest, ReplicateAsLeader) { @@ -486,7 +499,7 @@ TEST_F(ConsensusModuleTest, ReplicateAsLeader) consensus->start(); std::this_thread::sleep_for(std::chrono::milliseconds(500)); // Wait to become leader - if (consensus->getStateSafe() == NodeState::LEADER) + if (consensus->getStateSafe() == raft::v1::NodeState::NODE_STATE_LEADER) { auto future = consensus->replicateAsync("test payload"); @@ -522,17 +535,17 @@ TEST_F(ConsensusModuleTest, PersistentStateRestore) ASSERT_TRUE(consensus1->init()); // Simulate voting for candidate 2 in term 3 - RequestVoteRequest request; + raft::v1::RequestVoteRequest request; request.set_term(3); - request.set_candidateid(2); - request.set_lastlogindex(0); - request.set_lastlogterm(0); + request.set_candidate_id(2); + request.set_last_log_index(0); + request.set_last_log_term(0); - RequestVoteResponse response; - grpc::ServerContext context; + raft::v1::RequestVoteResponse response; + grpc::ServerContext context; consensus1->RequestVote(&context, &request, &response); - EXPECT_EQ(response.votegranted(), 1); + EXPECT_EQ(response.vote_granted(), 1); } // Create new consensus module and verify state restoration @@ -548,10 +561,10 @@ TEST_F(ConsensusModuleTest, PersistentStateRestore) TEST_F(ConsensusModuleTest, WALPersistenceRecovery) { // Test with persistent WAL - std::vector testEntries; + std::vector testEntries; for (int i = 0; i < 5; ++i) { - LogEntry entry; + raft::v1::LogEntry entry; entry.set_term(1); entry.set_index(i + 1); entry.set_payload("payload_" + std::to_string(i)); @@ -568,18 +581,18 @@ TEST_F(ConsensusModuleTest, WALPersistenceRecovery) // Add entries via AppendEntries for (const auto &entry : testEntries) { - AppendEntriesRequest request; + raft::v1::AppendEntriesRequest request; request.set_term(1); - request.set_senderid(2); - request.set_prevlogindex(entry.index() - 1); - request.set_prevlogterm(entry.index() > 1 ? 1 : 0); - request.set_leadercommit(0); + request.set_sender_id(2); + request.set_prev_log_index(entry.index() - 1); + request.set_prev_log_term(entry.index() > 1 ? 1 : 0); + request.set_leader_commit(0); - LogEntry *reqEntry = request.add_entries(); + raft::v1::LogEntry *reqEntry = request.add_entries(); reqEntry->CopyFrom(entry); - AppendEntriesResponse response; - grpc::ServerContext context; + raft::v1::AppendEntriesResponse response; + grpc::ServerContext context; auto status = persistentConsensus->AppendEntries(&context, &request, &response); EXPECT_TRUE(status.ok()); @@ -661,7 +674,7 @@ TEST_F(ConsensusModuleTest, WALResetOperations) // Add some entries to WAL for (int i = 0; i < 5; ++i) { - LogEntry entry; + raft::v1::LogEntry entry; entry.set_term(1); entry.set_index(i + 1); entry.set_payload("entry_" + std::to_string(i)); @@ -700,7 +713,7 @@ TEST_F(ConsensusModuleTest, OnCommitCallback) std::string receivedPayload; consensus->setOnCommitCallback( - [&](const LogEntry &entry) -> bool + [&](const raft::v1::LogEntry &entry) -> bool { callbackCalled = true; receivedPayload = entry.payload(); @@ -711,20 +724,20 @@ TEST_F(ConsensusModuleTest, OnCommitCallback) ASSERT_TRUE(consensus->init()); // Add an entry and set commit index - AppendEntriesRequest request; + raft::v1::AppendEntriesRequest request; request.set_term(1); - request.set_senderid(2); - request.set_prevlogindex(0); - request.set_prevlogterm(0); - request.set_leadercommit(1); + request.set_sender_id(2); + request.set_prev_log_index(0); + request.set_prev_log_term(0); + request.set_leader_commit(1); - LogEntry *entry = request.add_entries(); + raft::v1::LogEntry *entry = request.add_entries(); entry->set_term(1); entry->set_index(1); entry->set_payload("callback_test"); - AppendEntriesResponse response; - grpc::ServerContext context; + raft::v1::AppendEntriesResponse response; + grpc::ServerContext context; auto status = consensus->AppendEntries(&context, &request, &response); EXPECT_TRUE(status.ok()); @@ -766,15 +779,15 @@ TEST_F(ConsensusModuleTest, OnLeaderChangeCallback) EXPECT_TRUE(becameLeader.load()); // Simulate losing leadership through higher term AppendEntries - AppendEntriesRequest request; + raft::v1::AppendEntriesRequest request; request.set_term(consensus->currentTerm() + 1); - request.set_senderid(2); - request.set_prevlogindex(0); - request.set_prevlogterm(0); - request.set_leadercommit(0); + request.set_sender_id(2); + request.set_prev_log_index(0); + request.set_prev_log_term(0); + request.set_leader_commit(0); - AppendEntriesResponse response; - grpc::ServerContext context; + raft::v1::AppendEntriesResponse response; + grpc::ServerContext context; consensus->AppendEntries(&context, &request, &response); @@ -798,9 +811,9 @@ class RaftClusterTest : public ::testing::Test std::filesystem::create_directories("./var/tkvpp/"); // Create mock stubs for network communication - mockStub1 = std::make_unique(); - mockStub2 = std::make_unique(); - mockStub3 = std::make_unique(); + mockStub1 = std::make_unique(); + mockStub2 = std::make_unique(); + mockStub3 = std::make_unique(); // Keep raw pointers for setting expectations mockStubPtr1 = mockStub1.get(); @@ -814,12 +827,14 @@ class RaftClusterTest : public ::testing::Test } std::unique_ptr createNodeWithMockPeers( - id_t nodeId, - const std::string &nodeIp, - std::vector> peerConfigs, - std::vector> peerStubs + id_t nodeId, + const std::string &nodeIp, + std::vector> peerConfigs, + std::vector> peerStubs ) { + auto pDbConfig = config::make_shared(); + pDbConfig->DatabaseConfig.DatabasePath = "./var/tkvpp/"; node_config_t nodeConfig{nodeId, nodeIp}; @@ -827,7 +842,7 @@ class RaftClusterTest : public ::testing::Test auto walPath = std::filesystem::temp_directory_path() / ("cluster_test_wal_" + std::to_string(nodeId) + ".log"); - auto walOpt = wal::wal_builder_t{}.set_file_path(walPath).build( + auto walOpt = wal::wal_builder_t{}.set_file_path(walPath).build( wal::log_storage_type_k::in_memory_k ); @@ -836,26 +851,28 @@ class RaftClusterTest : public ::testing::Test return nullptr; } - auto walPtr = std::make_shared>(std::move(walOpt.value())); + auto walPtr = std::make_shared>(std::move(walOpt.value())); // Create replica clients with mock stubs std::vector replicas; for (size_t i = 0; i < peerConfigs.size() && i < peerStubs.size(); ++i) { - node_config_t peerConfig{peerConfigs[i].first, peerConfigs[i].second}; + node_config_t peerConfig{.m_id = peerConfigs[i].first, .m_ip = peerConfigs[i].second}; replicas.emplace_back(peerConfig, std::move(peerStubs[i])); } - return std::make_unique(nodeConfig, std::move(replicas), walPtr); + return std::make_unique( + pDbConfig, nodeConfig, std::move(replicas), walPtr + ); } - std::unique_ptr mockStub1; - std::unique_ptr mockStub2; - std::unique_ptr mockStub3; + std::unique_ptr mockStub1; + std::unique_ptr mockStub2; + std::unique_ptr mockStub3; - MockRaftServiceStub *mockStubPtr1; - MockRaftServiceStub *mockStubPtr2; - MockRaftServiceStub *mockStubPtr3; + raft::v1::MockRaftServiceStub *mockStubPtr1; + raft::v1::MockRaftServiceStub *mockStubPtr2; + raft::v1::MockRaftServiceStub *mockStubPtr3; }; TEST_F(RaftClusterTest, LeaderElectionWithMockNetwork) @@ -864,11 +881,11 @@ TEST_F(RaftClusterTest, LeaderElectionWithMockNetwork) EXPECT_CALL(*mockStubPtr2, RequestVote(_, _, _)) .WillRepeatedly(DoAll( WithArg<2>( - [](RequestVoteResponse *response) + [](raft::v1::RequestVoteResponse *response) { - response->set_votegranted(1); + response->set_vote_granted(1); response->set_term(1); - response->set_responderid(2); + response->set_responder_id(2); } ), Return(grpc::Status::OK) @@ -877,11 +894,11 @@ TEST_F(RaftClusterTest, LeaderElectionWithMockNetwork) EXPECT_CALL(*mockStubPtr3, RequestVote(_, _, _)) .WillRepeatedly(DoAll( WithArg<2>( - [](RequestVoteResponse *response) + [](raft::v1::RequestVoteResponse *response) { - response->set_votegranted(1); + response->set_vote_granted(1); response->set_term(1); - response->set_responderid(3); + response->set_responder_id(3); } ), Return(grpc::Status::OK) @@ -892,7 +909,7 @@ TEST_F(RaftClusterTest, LeaderElectionWithMockNetwork) {2, "127.0.0.1:5002"}, {3, "127.0.0.1:5003"} }; - std::vector> stubs; + std::vector> stubs; stubs.push_back(std::move(mockStub2)); stubs.push_back(std::move(mockStub3)); @@ -905,7 +922,7 @@ TEST_F(RaftClusterTest, LeaderElectionWithMockNetwork) // Wait for election to complete std::this_thread::sleep_for(std::chrono::milliseconds(600)); - EXPECT_EQ(node1->getStateSafe(), NodeState::LEADER); + EXPECT_EQ(node1->getStateSafe(), raft::v1::NodeState::NODE_STATE_LEADER); EXPECT_GT(node1->currentTerm(), 0); node1->stop(); @@ -917,11 +934,11 @@ TEST_F(RaftClusterTest, HeartbeatCommunication) EXPECT_CALL(*mockStubPtr2, AppendEntries(_, _, _)) .WillRepeatedly(DoAll( WithArg<2>( - [](AppendEntriesResponse *response) + [](raft::v1::AppendEntriesResponse *response) { response->set_success(true); response->set_term(1); - response->set_responderid(2); + response->set_responder_id(2); response->set_match_index(0); } ), @@ -931,11 +948,11 @@ TEST_F(RaftClusterTest, HeartbeatCommunication) EXPECT_CALL(*mockStubPtr3, AppendEntries(_, _, _)) .WillRepeatedly(DoAll( WithArg<2>( - [](AppendEntriesResponse *response) + [](raft::v1::AppendEntriesResponse *response) { response->set_success(true); response->set_term(1); - response->set_responderid(3); + response->set_responder_id(3); response->set_match_index(0); } ), @@ -946,11 +963,11 @@ TEST_F(RaftClusterTest, HeartbeatCommunication) EXPECT_CALL(*mockStubPtr2, RequestVote(_, _, _)) .WillOnce(DoAll( WithArg<2>( - [](RequestVoteResponse *response) + [](raft::v1::RequestVoteResponse *response) { - response->set_votegranted(1); + response->set_vote_granted(1); response->set_term(1); - response->set_responderid(2); + response->set_responder_id(2); } ), Return(grpc::Status::OK) @@ -959,11 +976,11 @@ TEST_F(RaftClusterTest, HeartbeatCommunication) EXPECT_CALL(*mockStubPtr3, RequestVote(_, _, _)) .WillOnce(DoAll( WithArg<2>( - [](RequestVoteResponse *response) + [](raft::v1::RequestVoteResponse *response) { - response->set_votegranted(1); + response->set_vote_granted(1); response->set_term(1); - response->set_responderid(3); + response->set_responder_id(3); } ), Return(grpc::Status::OK) @@ -973,7 +990,7 @@ TEST_F(RaftClusterTest, HeartbeatCommunication) {2, "127.0.0.1:5002"}, {3, "127.0.0.1:5003"} }; - std::vector> stubs; + std::vector> stubs; stubs.push_back(std::move(mockStub2)); stubs.push_back(std::move(mockStub3)); @@ -986,7 +1003,7 @@ TEST_F(RaftClusterTest, HeartbeatCommunication) // Wait longer to ensure heartbeats are sent std::this_thread::sleep_for(std::chrono::milliseconds(800)); - EXPECT_EQ(leader->getStateSafe(), NodeState::LEADER); + EXPECT_EQ(leader->getStateSafe(), raft::v1::NodeState::NODE_STATE_LEADER); leader->stop(); } @@ -997,11 +1014,11 @@ TEST_F(RaftClusterTest, LogReplicationWithMockNetwork) EXPECT_CALL(*mockStubPtr2, AppendEntries(_, _, _)) .WillRepeatedly(DoAll( WithArg<2>( - [](AppendEntriesResponse *response) + [](raft::v1::AppendEntriesResponse *response) { response->set_success(true); response->set_term(1); - response->set_responderid(2); + response->set_responder_id(2); response->set_match_index(1); } ), @@ -1011,11 +1028,11 @@ TEST_F(RaftClusterTest, LogReplicationWithMockNetwork) EXPECT_CALL(*mockStubPtr3, AppendEntries(_, _, _)) .WillRepeatedly(DoAll( WithArg<2>( - [](AppendEntriesResponse *response) + [](raft::v1::AppendEntriesResponse *response) { response->set_success(true); response->set_term(1); - response->set_responderid(3); + response->set_responder_id(3); response->set_match_index(1); } ), @@ -1026,11 +1043,11 @@ TEST_F(RaftClusterTest, LogReplicationWithMockNetwork) EXPECT_CALL(*mockStubPtr2, RequestVote(_, _, _)) .WillOnce(DoAll( WithArg<2>( - [](RequestVoteResponse *response) + [](raft::v1::RequestVoteResponse *response) { - response->set_votegranted(1); + response->set_vote_granted(1); response->set_term(1); - response->set_responderid(2); + response->set_responder_id(2); } ), Return(grpc::Status::OK) @@ -1039,11 +1056,11 @@ TEST_F(RaftClusterTest, LogReplicationWithMockNetwork) EXPECT_CALL(*mockStubPtr3, RequestVote(_, _, _)) .WillOnce(DoAll( WithArg<2>( - [](RequestVoteResponse *response) + [](raft::v1::RequestVoteResponse *response) { - response->set_votegranted(1); + response->set_vote_granted(1); response->set_term(1); - response->set_responderid(3); + response->set_responder_id(3); } ), Return(grpc::Status::OK) @@ -1053,7 +1070,7 @@ TEST_F(RaftClusterTest, LogReplicationWithMockNetwork) {2, "127.0.0.1:5002"}, {3, "127.0.0.1:5003"} }; - std::vector> stubs; + std::vector> stubs; stubs.push_back(std::move(mockStub2)); stubs.push_back(std::move(mockStub3)); @@ -1066,7 +1083,7 @@ TEST_F(RaftClusterTest, LogReplicationWithMockNetwork) // Wait to become leader std::this_thread::sleep_for(std::chrono::milliseconds(600)); - if (leader->getStateSafe() == NodeState::LEADER) + if (leader->getStateSafe() == raft::v1::NodeState::NODE_STATE_LEADER) { // Test async replication auto future = leader->replicateAsync("test_replication_payload"); @@ -1093,11 +1110,11 @@ TEST_F(RaftClusterTest, NetworkPartitionSimulation) EXPECT_CALL(*mockStubPtr3, RequestVote(_, _, _)) .WillOnce(DoAll( WithArg<2>( - [](RequestVoteResponse *response) + [](raft::v1::RequestVoteResponse *response) { - response->set_votegranted(1); + response->set_vote_granted(1); response->set_term(1); - response->set_responderid(3); + response->set_responder_id(3); } ), Return(grpc::Status::OK) @@ -1107,7 +1124,7 @@ TEST_F(RaftClusterTest, NetworkPartitionSimulation) {2, "127.0.0.1:5002"}, {3, "127.0.0.1:5003"} }; - std::vector> stubs; + std::vector> stubs; stubs.push_back(std::move(mockStub2)); stubs.push_back(std::move(mockStub3)); @@ -1122,7 +1139,7 @@ TEST_F(RaftClusterTest, NetworkPartitionSimulation) std::this_thread::sleep_for(std::chrono::milliseconds(800)); // Should be able to become leader with majority (2 out of 3) - EXPECT_EQ(node1->getStateSafe(), NodeState::LEADER); + EXPECT_EQ(node1->getStateSafe(), raft::v1::NodeState::NODE_STATE_LEADER); node1->stop(); } @@ -1133,11 +1150,11 @@ TEST_F(RaftClusterTest, ConflictingTerms) EXPECT_CALL(*mockStubPtr2, RequestVote(_, _, _)) .WillOnce(DoAll( WithArg<2>( - [](RequestVoteResponse *response) + [](raft::v1::RequestVoteResponse *response) { - response->set_votegranted(0); // Don't grant vote - response->set_term(5); // But indicate higher term - response->set_responderid(2); + response->set_vote_granted(0); // Don't grant vote + response->set_term(5); // But indicate higher term + response->set_responder_id(2); } ), Return(grpc::Status::OK) @@ -1146,11 +1163,11 @@ TEST_F(RaftClusterTest, ConflictingTerms) EXPECT_CALL(*mockStubPtr3, RequestVote(_, _, _)) .WillOnce(DoAll( WithArg<2>( - [](RequestVoteResponse *response) + [](raft::v1::RequestVoteResponse *response) { - response->set_votegranted(1); + response->set_vote_granted(1); response->set_term(1); // Lower term - response->set_responderid(3); + response->set_responder_id(3); } ), Return(grpc::Status::OK) @@ -1160,7 +1177,7 @@ TEST_F(RaftClusterTest, ConflictingTerms) {2, "127.0.0.1:5002"}, {3, "127.0.0.1:5003"} }; - std::vector> stubs; + std::vector> stubs; stubs.push_back(std::move(mockStub2)); stubs.push_back(std::move(mockStub3)); @@ -1174,7 +1191,7 @@ TEST_F(RaftClusterTest, ConflictingTerms) // Node should update its term and revert to follower EXPECT_EQ(node1->currentTerm(), 5); - EXPECT_EQ(node1->getStateSafe(), NodeState::FOLLOWER); + EXPECT_EQ(node1->getStateSafe(), raft::v1::NodeState::NODE_STATE_FOLLOWER); node1->stop(); } @@ -1194,7 +1211,7 @@ TEST_F(ConsensusModuleTest, ThreeNodeClusterSetup) // Verify cluster configuration EXPECT_EQ(consensus->currentTerm(), 0); - EXPECT_EQ(consensus->getStateSafe(), NodeState::FOLLOWER); + EXPECT_EQ(consensus->getStateSafe(), raft::v1::NodeState::NODE_STATE_FOLLOWER); // The actual election would require mock expectations for network calls // This is a basic structural test @@ -1224,15 +1241,15 @@ TEST_F(ConsensusModuleTest, AppendEntriesErrorConditions) ASSERT_TRUE(consensus->init()); // Test with missing previous log entry - AppendEntriesRequest request; + raft::v1::AppendEntriesRequest request; request.set_term(1); - request.set_senderid(2); - request.set_prevlogindex(10); // Non-existent index - request.set_prevlogterm(1); - request.set_leadercommit(0); + request.set_sender_id(2); + request.set_prev_log_index(10); // Non-existent index + request.set_prev_log_term(1); + request.set_leader_commit(0); - AppendEntriesResponse response; - grpc::ServerContext context; + raft::v1::AppendEntriesResponse response; + grpc::ServerContext context; auto status = consensus->AppendEntries(&context, &request, &response); @@ -1249,10 +1266,10 @@ TEST_F(ConsensusModuleTest, LogConsistencyWithWAL) auto consensus = createConsensusModule(); ASSERT_TRUE(consensus->init()); - std::vector entries; + std::vector entries; for (int i = 0; i < 3; ++i) { - LogEntry entry; + raft::v1::LogEntry entry; entry.set_term(1); entry.set_index(i + 1); entry.set_payload("consistency_test_" + std::to_string(i)); @@ -1262,18 +1279,18 @@ TEST_F(ConsensusModuleTest, LogConsistencyWithWAL) // Add entries via AppendEntries for (const auto &entry : entries) { - AppendEntriesRequest request; + raft::v1::AppendEntriesRequest request; request.set_term(1); - request.set_senderid(2); - request.set_prevlogindex(entry.index() - 1); - request.set_prevlogterm(entry.index() > 1 ? 1 : 0); - request.set_leadercommit(0); + request.set_sender_id(2); + request.set_prev_log_index(entry.index() - 1); + request.set_prev_log_term(entry.index() > 1 ? 1 : 0); + request.set_leader_commit(0); - LogEntry *reqEntry = request.add_entries(); + raft::v1::LogEntry *reqEntry = request.add_entries(); reqEntry->CopyFrom(entry); - AppendEntriesResponse response; - grpc::ServerContext context; + raft::v1::AppendEntriesResponse response; + grpc::ServerContext context; auto status = consensus->AppendEntries(&context, &request, &response); EXPECT_TRUE(status.ok()); @@ -1305,47 +1322,47 @@ TEST_F(ConsensusModuleTest, LogTruncationWithWAL) ASSERT_TRUE(consensus->init()); // Add initial entries - std::vector initialEntries; + std::vector initialEntries; for (int i = 0; i < 5; ++i) { - LogEntry entry; + raft::v1::LogEntry entry; entry.set_term(1); entry.set_index(i + 1); entry.set_payload("initial_" + std::to_string(i)); initialEntries.push_back(entry); - AppendEntriesRequest request; + raft::v1::AppendEntriesRequest request; request.set_term(1); - request.set_senderid(2); - request.set_prevlogindex(i); - request.set_prevlogterm(i > 0 ? 1 : 0); - request.set_leadercommit(0); + request.set_sender_id(2); + request.set_prev_log_index(i); + request.set_prev_log_term(i > 0 ? 1 : 0); + request.set_leader_commit(0); - LogEntry *reqEntry = request.add_entries(); + raft::v1::LogEntry *reqEntry = request.add_entries(); reqEntry->CopyFrom(entry); - AppendEntriesResponse response; - grpc::ServerContext context; + raft::v1::AppendEntriesResponse response; + grpc::ServerContext context; consensus->AppendEntries(&context, &request, &response); } EXPECT_EQ(walPtr->size(), 5); // Now send conflicting entry that should cause truncation - AppendEntriesRequest conflictRequest; + raft::v1::AppendEntriesRequest conflictRequest; conflictRequest.set_term(2); // Higher term - conflictRequest.set_senderid(3); - conflictRequest.set_prevlogindex(3); // After 3rd entry - conflictRequest.set_prevlogterm(1); - conflictRequest.set_leadercommit(0); + conflictRequest.set_sender_id(3); + conflictRequest.set_prev_log_index(3); // After 3rd entry + conflictRequest.set_prev_log_term(1); + conflictRequest.set_leader_commit(0); - LogEntry *conflictEntry = conflictRequest.add_entries(); + raft::v1::LogEntry *conflictEntry = conflictRequest.add_entries(); conflictEntry->set_term(2); conflictEntry->set_index(4); conflictEntry->set_payload("conflict_entry"); - AppendEntriesResponse conflictResponse; - grpc::ServerContext conflictContext; + raft::v1::AppendEntriesResponse conflictResponse; + grpc::ServerContext conflictContext; auto status = consensus->AppendEntries(&conflictContext, &conflictRequest, &conflictResponse); EXPECT_TRUE(status.ok()); @@ -1363,10 +1380,10 @@ TEST_F(ConsensusModuleTest, LogTruncationWithWAL) TEST_F(ConsensusModuleTest, WALRecordsMethodTest) { // Test the records() method specifically - std::vector testEntries; + std::vector testEntries; for (int i = 0; i < 3; ++i) { - LogEntry entry; + raft::v1::LogEntry entry; entry.set_term(1); entry.set_index(i + 1); entry.set_payload("records_test_" + std::to_string(i)); @@ -1398,20 +1415,20 @@ TEST_F(ConsensusModuleTest, ConsensusModuleLogMethodTest) // Add entries through consensus module for (int i = 0; i < 3; ++i) { - AppendEntriesRequest request; + raft::v1::AppendEntriesRequest request; request.set_term(1); - request.set_senderid(2); - request.set_prevlogindex(i); - request.set_prevlogterm(i > 0 ? 1 : 0); - request.set_leadercommit(0); + request.set_sender_id(2); + request.set_prev_log_index(i); + request.set_prev_log_term(i > 0 ? 1 : 0); + request.set_leader_commit(0); - LogEntry *entry = request.add_entries(); + raft::v1::LogEntry *entry = request.add_entries(); entry->set_term(1); entry->set_index(i + 1); entry->set_payload("consensus_log_test_" + std::to_string(i)); - AppendEntriesResponse response; - grpc::ServerContext context; + raft::v1::AppendEntriesResponse response; + grpc::ServerContext context; consensus->AppendEntries(&context, &request, &response); EXPECT_TRUE(response.success()); } @@ -1440,7 +1457,7 @@ TEST_F(ConsensusModuleTest, WALPerformanceTest) for (size_t i = 0; i < numEntries; ++i) { - LogEntry entry; + raft::v1::LogEntry entry; entry.set_term(1); entry.set_index(i + 1); entry.set_payload("perf_test_" + std::to_string(i)); @@ -1489,22 +1506,22 @@ TEST_F(ConsensusModuleTest, ConcurrentWALAccess) { for (int i = 0; i < entriesPerThread; ++i) { - AppendEntriesRequest request; + raft::v1::AppendEntriesRequest request; request.set_term(1); - request.set_senderid(2); - request.set_prevlogindex(0); // Simplified for concurrent test - request.set_prevlogterm(0); - request.set_leadercommit(0); + request.set_sender_id(2); + request.set_prev_log_index(0); // Simplified for concurrent test + request.set_prev_log_term(0); + request.set_leader_commit(0); - LogEntry *entry = request.add_entries(); + raft::v1::LogEntry *entry = request.add_entries(); entry->set_term(1); entry->set_index(1); // Simplified entry->set_payload( "thread_" + std::to_string(t) + "_entry_" + std::to_string(i) ); - AppendEntriesResponse response; - grpc::ServerContext context; + raft::v1::AppendEntriesResponse response; + grpc::ServerContext context; auto status = consensus->AppendEntries(&context, &request, &response); if (status.ok() && response.success()) @@ -1537,7 +1554,7 @@ TEST_F(ConsensusModuleTest, WALBoundaryConditions) EXPECT_FALSE(walPtr->read(SIZE_MAX).has_value()); // Add one entry - LogEntry entry; + raft::v1::LogEntry entry; entry.set_term(1); entry.set_index(1); entry.set_payload("boundary_test"); @@ -1568,18 +1585,18 @@ TEST_F(ConsensusModuleTest, ConsensusModuleWithEmptyWAL) EXPECT_EQ(log.size(), 0); // Test vote request with empty log - RequestVoteRequest request; + raft::v1::RequestVoteRequest request; request.set_term(1); - request.set_candidateid(2); - request.set_lastlogindex(0); - request.set_lastlogterm(0); + request.set_candidate_id(2); + request.set_last_log_index(0); + request.set_last_log_term(0); - RequestVoteResponse response; - grpc::ServerContext context; + raft::v1::RequestVoteResponse response; + grpc::ServerContext context; auto status = consensus->RequestVote(&context, &request, &response); EXPECT_TRUE(status.ok()); - EXPECT_EQ(response.votegranted(), 1); + EXPECT_EQ(response.vote_granted(), 1); } // ============================================================================ diff --git a/lib/structures/lsmtree/levels/level.cpp b/lib/structures/lsmtree/levels/level.cpp index 5018a8e..48e0b0a 100644 --- a/lib/structures/lsmtree/levels/level.cpp +++ b/lib/structures/lsmtree/levels/level.cpp @@ -241,7 +241,9 @@ void level_t::merge(const segments::regular_segment::shared_ptr_t &pSegment) noe for (const auto ¤tRecord : mergedMemtable) { newMemtable.emplace(currentRecord); - if (newMemtable.size() >= m_pConfig->LSMTreeConfig.SegmentSize) + // TODO(lnikon): Make SegmentSize configurall. + // if (newMemtable.size() >= m_pConfig->LSMTreeConfig.SegmentSize) + if (newMemtable.size() >= 1024) { auto name{fmt::format("{}_{}", helpers::segment_name(), index())}; newSegments.emplace( diff --git a/src/main.cpp b/src/main.cpp index 602850e..bcc0ec7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -57,12 +57,6 @@ namespace return std::nullopt; } - if (pConfig->ServerConfig.peers.empty()) - { - spdlog::error("maybe_create_consensus_module: List of node IPs can't be empty"); - return std::nullopt; - } - std::vector replicas; for (consensus::id_t replicaId{1}; const auto &replicaIp : pConfig->ServerConfig.peers) { @@ -79,8 +73,9 @@ namespace ++replicaId; } - auto pConsensusModule = - std::make_shared(nodeConfig, std::move(replicas), pWAL); + auto pConsensusModule = std::make_shared( + pConfig, nodeConfig, std::move(replicas), pWAL + ); if (!pConsensusModule->init()) { spdlog::error("maybe_create_consensus_module: Failed to initialize the consensus module"); diff --git a/tests/go/kvtest/cmd/kvtest/kvtest.yml b/tests/go/kvtest/cmd/kvtest/kvtest.yml index 03c0172..13b7366 100644 --- a/tests/go/kvtest/cmd/kvtest/kvtest.yml +++ b/tests/go/kvtest/cmd/kvtest/kvtest.yml @@ -3,7 +3,7 @@ log_level: info adapter: type: tinykvpp - address: localhost:9891 + address: 0.0.0.0:9891 connection_timeout: 5s request_timeout: 1s max_retries: 3 From 813a7e91fb4792966c97e6d8700e0de395560307 Mon Sep 17 00:00:00 2001 From: lnikon Date: Tue, 21 Oct 2025 00:50:25 +0400 Subject: [PATCH 12/17] Integration & load testing: - Improve logging - Simplify compaction threshold calculation - Update infra files --- infra/assets/tkvpp_config_standalone.json | 8 +-- lib/db/CMakeLists.txt | 7 ++- lib/db/db.cpp | 3 -- lib/db/db_test.cpp | 34 +++++++------ lib/raft/CMakeLists.txt | 3 ++ lib/raft/raft.cpp | 49 +++---------------- lib/raft/raft_test.cpp | 6 ++- lib/structures/lsmtree/levels/level.cpp | 33 +++++++------ lib/structures/lsmtree/levels/level.h | 12 +++-- lib/structures/lsmtree/levels/levels.cpp | 41 ++++++++-------- lib/structures/lsmtree/lsmtree_config.h | 6 +-- tests/go/kvtest/cmd/kvtest/kvtest.yml | 4 +- .../pkg/scenarios/storage/memtable_stress.go | 4 +- 13 files changed, 91 insertions(+), 119 deletions(-) diff --git a/infra/assets/tkvpp_config_standalone.json b/infra/assets/tkvpp_config_standalone.json index 785af3a..e469c29 100644 --- a/infra/assets/tkvpp_config_standalone.json +++ b/infra/assets/tkvpp_config_standalone.json @@ -14,20 +14,20 @@ "storageType": "persistent" }, "lsm": { - "flushThreshold": 64000000, + "flushThreshold": 1024, "maximumLevels": 12, "levelZeroCompaction": { "compactionStrategy": "levelled", - "compactionThreshold": 256000000 + "compactionThreshold": 8096 }, "levelNonZeroCompaction": { "compactionStrategy": "tiered", - "compactionThreshold": 256000000 + "compactionThreshold": 4096 } }, "server": { "transport": "grpc", - "host": "127.0.0.1", + "host": "0.0.0.0", "port": 9891, "id": 1, "peers": [] diff --git a/lib/db/CMakeLists.txt b/lib/db/CMakeLists.txt index f042a44..266aabe 100644 --- a/lib/db/CMakeLists.txt +++ b/lib/db/CMakeLists.txt @@ -27,7 +27,7 @@ add_executable(DBTest db_test.cpp) set_target_properties(DBTest PROPERTIES CXX_STANDARD 23) target_link_libraries( DBTest - Catch2::Catch2WithMain + gtest::gtest spdlog::spdlog fmt::fmt LSMTree @@ -36,6 +36,5 @@ target_link_libraries( Config HashIndex) -# Register Catch2 tests with CTest -include(Catch) -catch_discover_tests(DBTest) +include(GoogleTest) +gtest_discover_tests(DBTest) diff --git a/lib/db/db.cpp b/lib/db/db.cpp index 8e609f8..d3a213a 100644 --- a/lib/db/db.cpp +++ b/lib/db/db.cpp @@ -182,7 +182,6 @@ db_t::put(const tinykvpp::v1::PutRequest *pRequest, tinykvpp::v1::PutResponse *p } auto waitStatus = future.wait_for(m_config->DatabaseConfig.requestTimeout); - spdlog::info("waitStatus={}", magic_enum::enum_name(waitStatus)); if (waitStatus == std::future_status::ready) { if (!future.get()) @@ -267,8 +266,6 @@ void db_t::processRequests() continue; } - spdlog::info("Processing request: {}", request->requestId); - m_requestPool.enqueue([this, request = std::move(request.value())] mutable { handleClientRequest(std::move(request)); }); } diff --git a/lib/db/db_test.cpp b/lib/db/db_test.cpp index abf7ecd..df5b526 100644 --- a/lib/db/db_test.cpp +++ b/lib/db/db_test.cpp @@ -1,22 +1,24 @@ -#include "manifest/manifest.h" -#include "wal/wal.h" -#include +#include +#include -#include #include -#include -TEST_CASE("db interface validation", "[db]") +// ============================================================================ +// Main Test Runner +// ============================================================================ + +int main(int argc, char **argv) { - config::shared_ptr_t pConfig{config::make_shared()}; - auto pSegmentStorage{structures::lsmtree::segments::storage::make_shared()}; - auto manifest{db::manifest::make_shared(pConfig)}; - auto wal{wal::make_shared("wal")}; - auto lsmTree{structures::lsmtree::lsmtree_t{pConfig, manifest, wal}}; + ::testing::InitGoogleTest(&argc, argv); + ::testing::InitGoogleMock(&argc, argv); + + // Create test directories + std::filesystem::create_directories("./var/tkvpp/"); + + int result = RUN_ALL_TESTS(); + + // Cleanup + std::filesystem::remove_all("./var/tkvpp/"); - SECTION("fail when db path is empty") - { - db::db_t db(pConfig); - // REQUIRE(db.open() == false); - } + return result; } diff --git a/lib/raft/CMakeLists.txt b/lib/raft/CMakeLists.txt index 81f4800..1eb3850 100644 --- a/lib/raft/CMakeLists.txt +++ b/lib/raft/CMakeLists.txt @@ -17,3 +17,6 @@ add_executable(RaftTest "raft_test.cpp") set_target_properties(RaftTest PROPERTIES CXX_STANDARD 23) target_link_libraries(RaftTest gtest::gtest Raft RaftProtoObjects TKVProtoObjects Concurrency) + +include(GoogleTest) +gtest_discover_tests(RaftTest) diff --git a/lib/raft/raft.cpp b/lib/raft/raft.cpp index e34c1fc..21d29eb 100644 --- a/lib/raft/raft.cpp +++ b/lib/raft/raft.cpp @@ -212,7 +212,7 @@ auto consensus_module_t::init() -> bool return false; } - m_replicationMonitor = std::thread([this] { monitorPendingReplications(); }); + m_replicationMonitor = std::thread([this] -> void { monitorPendingReplications(); }); return true; } @@ -395,11 +395,6 @@ auto consensus_module_t::AppendEntries( m_log->add(entry); } - if (!pRequest->entries().empty()) - { - spdlog::info("leaderCommit={}, m_commitIndex={}", pRequest->leader_commit(), m_commitIndex); - } - if (pRequest->leader_commit() > m_commitIndex) { // Update m_commitIndex @@ -417,17 +412,6 @@ auto consensus_module_t::AppendEntries( } } - if (!entries.empty()) - { - spdlog::info( - "Follower Node={} PREPARING applying m_lastApplied+1={}, m_commitIndex={}", - m_nodeConfig.m_id, - m_lastApplied + 1, - m_commitIndex - ); - } - - // Apply committed entries to the state machine applyCommittedEntries(); pResponse->set_term(m_currentTerm); @@ -441,9 +425,7 @@ auto consensus_module_t::AppendEntries( spdlog::error("Node={} is unable to persist votedFor", m_nodeConfig.m_id, m_votedFor); } - { - m_leaderHeartbeatReceived.store(true); - } + m_leaderHeartbeatReceived.store(true); spdlog::debug( "Node={} is resetting election timeout at term={}", m_nodeConfig.m_id, m_currentTerm @@ -508,8 +490,6 @@ auto consensus_module_t::RequestVote( ); } - spdlog::info("Node={} votedFor={}", m_nodeConfig.m_id, pRequest->candidate_id()); - pResponse->set_term(m_currentTerm); pResponse->set_vote_granted(1); } @@ -681,7 +661,9 @@ void consensus_module_t::becomeFollower(uint32_t newTerm) if (m_onLeaderChangeCbk) { auto callback = m_onLeaderChangeCbk; - m_internalThreadPool->enqueue([callback = std::move(callback)] { callback(false); }); + m_internalThreadPool->enqueue( + [callback = std::move(callback)] -> void { callback(false); } + ); } } @@ -710,7 +692,6 @@ void consensus_module_t::becomeLeader() if (m_onLeaderChangeCbk) { - spdlog::info("Executing onLeaderChangeCbk"); auto callback = m_onLeaderChangeCbk; m_internalThreadPool->enqueue([callback = std::move(callback)] { callback(true); }); } @@ -925,15 +906,8 @@ auto consensus_module_t::onSendAppendEntriesRPC( if (m_log->size() >= majorityIndex && majorityIndex > m_commitIndex) { - spdlog::info("Node={} found majorityIndex={}", m_nodeConfig.m_id, majorityIndex); if (m_log->read(majorityIndex - 1)->term() == m_currentTerm) { - spdlog::info( - "Node={} is updating commitIndex to majorityIndex={}", - m_nodeConfig.m_id, - majorityIndex - ); - if (!updatePersistentState(majorityIndex, std::nullopt)) { spdlog::error( @@ -1007,7 +981,7 @@ void consensus_module_t::startElection() spdlog::info("Node={} starts election. New term={}", m_nodeConfig.m_id, m_currentTerm); // Node in a canditate state should vote for itself. - m_voteCount++; + m_voteCount = 1; if (!updatePersistentState(std::nullopt, m_nodeConfig.m_id)) { spdlog::error( @@ -1018,10 +992,6 @@ void consensus_module_t::startElection() // Early majority check for a single node cluster if (hasMajority(m_voteCount.load())) { - spdlog::info( - "Node={} achieved majority (single node cluster), becoming leader", - m_nodeConfig.m_id - ); becomeLeader(); return; } @@ -1163,7 +1133,6 @@ auto consensus_module_t::findMajorityIndexMatch() const -> uint32_t } std::ranges::sort(matchIndexes); - // spdlog::info("Node={} matchIndexes={}", m_config.m_id, fmt::join(matchIndexes, ", ")); return matchIndexes[matchIndexes.size() / 2]; } @@ -1179,10 +1148,6 @@ auto consensus_module_t::waitForMajorityReplication(uint32_t logIndex) const -> return hasMajority(count); }; - spdlog::info( - "Node={} is waiting for majority to agree on logIndex={}", m_nodeConfig.m_id, logIndex - ); - const auto timeout{absl::Seconds(generate_raft_timeout())}; return m_stateMutex.AwaitWithTimeout(absl::Condition(&hasMajorityLambda), timeout); } @@ -1364,8 +1329,6 @@ void consensus_module_t::applyCommittedEntries() m_internalThreadPool->enqueue( [this, entry = std::move(logEntry.value()), applyIndex] { - spdlog::info("Applying log entry {} to state machine", applyIndex); - try { if (m_onCommitCbk(entry)) diff --git a/lib/raft/raft_test.cpp b/lib/raft/raft_test.cpp index a5a42dd..a85f74a 100644 --- a/lib/raft/raft_test.cpp +++ b/lib/raft/raft_test.cpp @@ -836,7 +836,7 @@ class RaftClusterTest : public ::testing::Test auto pDbConfig = config::make_shared(); pDbConfig->DatabaseConfig.DatabasePath = "./var/tkvpp/"; - node_config_t nodeConfig{nodeId, nodeIp}; + node_config_t nodeConfig{.m_id = nodeId, .m_ip = nodeIp}; // Create WAL for this node auto walPath = std::filesystem::temp_directory_path() / @@ -861,6 +861,8 @@ class RaftClusterTest : public ::testing::Test replicas.emplace_back(peerConfig, std::move(peerStubs[i])); } + spdlog::info("replicas.size()={}", replicas.size()); + return std::make_unique( pDbConfig, nodeConfig, std::move(replicas), walPtr ); @@ -1110,7 +1112,7 @@ TEST_F(RaftClusterTest, NetworkPartitionSimulation) EXPECT_CALL(*mockStubPtr3, RequestVote(_, _, _)) .WillOnce(DoAll( WithArg<2>( - [](raft::v1::RequestVoteResponse *response) + [](raft::v1::RequestVoteResponse *response) -> void { response->set_vote_granted(1); response->set_term(1); diff --git a/lib/structures/lsmtree/levels/level.cpp b/lib/structures/lsmtree/levels/level.cpp index 48e0b0a..cc2a4ef 100644 --- a/lib/structures/lsmtree/levels/level.cpp +++ b/lib/structures/lsmtree/levels/level.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -116,24 +117,25 @@ auto level_t::record(const key_t &key) const noexcept auto level_t::compact() const noexcept -> segments::regular_segment::shared_ptr_t { absl::ReaderMutexLock lock{&m_mutex}; - const auto bytesUsedForLevel{bytes_used()}; - // If level size hasn't reached the size limit then skip the compaction - const bool isZeroLevel{index() == 0}; - const auto zeroLevelThreshold{m_pConfig->LSMTreeConfig.LevelZeroCompactionThreshold}; - if (isZeroLevel && bytesUsedForLevel < zeroLevelThreshold) - { - return nullptr; - } - - const auto nonZeroLevelThreshold{ - m_pConfig->LSMTreeConfig.LevelNonZeroCompactionThreshold * std::pow(10, index()) + // Skip compaction if size of the level is less than the threshold + const std::uint64_t used{bytes_used()}; + const std::uint64_t threshold{ + index() == 0 ? m_pConfig->LSMTreeConfig.LevelZeroCompactionThreshold + : m_pConfig->LSMTreeConfig.LevelNonZeroCompactionThreshold * + static_cast(std::pow(10, index())) }; - if (!isZeroLevel && bytesUsedForLevel < nonZeroLevelThreshold) + if (used < threshold) { + spdlog::info( + "Skipping level={} compaction. Used={}, threshold={}", index(), used, threshold + ); return nullptr; } + spdlog::info("Compacting level={}. Used={}, threshold={}", index(), used, threshold); + + // Merge segments from the current level std::priority_queue< std::pair< typename memtable::memtable_t::const_iterator, @@ -284,19 +286,20 @@ void level_t::merge(const segments::regular_segment::shared_ptr_t &pSegment) noe void level_t::purge() noexcept { absl::WriterMutexLock lock{&m_mutex}; - spdlog::debug("Purging level {} with {} segments", index(), m_storage.size()); + spdlog::info("Purging level {} with {} segments", index(), m_storage.size()); const auto idx{index()}; for (auto &pSegment : m_storage) { pSegment->remove_from_disk(); - ASSERT(m_manifest->add( + auto ok{m_manifest->add( db::manifest::manifest_t::segment_record_t{ .op = segment_operation_k::remove_segment_k, .name = pSegment->get_name(), .level = idx } - )); + )}; + ASSERT(ok); } m_storage.clear(); } diff --git a/lib/structures/lsmtree/levels/level.h b/lib/structures/lsmtree/levels/level.h index acda9d3..06c5bd3 100644 --- a/lib/structures/lsmtree/levels/level.h +++ b/lib/structures/lsmtree/levels/level.h @@ -1,19 +1,21 @@ #pragma once -#include "structures/lsmtree/segments/lsmtree_regular_segment.h" -#include -#include -#include +#include #include +#include "structures/lsmtree/segments/lsmtree_regular_segment.h" +#include "db/manifest/manifest.h" +#include "config/config.h" +#include "structures/lsmtree/segments/segment_storage.h" + namespace structures::lsmtree::level { class level_t { public: - using level_index_type_t = std::size_t; + using level_index_type_t = std::uint64_t; using record_t = structures::memtable::memtable_t::record_t; using key_t = record_t::key_t; diff --git a/lib/structures/lsmtree/levels/levels.cpp b/lib/structures/lsmtree/levels/levels.cpp index c6c9c46..9aefbe8 100644 --- a/lib/structures/lsmtree/levels/levels.cpp +++ b/lib/structures/lsmtree/levels/levels.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include @@ -90,15 +92,18 @@ auto levels_t::compact() -> segments::regular_segment::shared_ptr_t continue; } - // Update manifest with compacted level - // TODO(lnikon): Replace this and following ASSERTS()s with manifest batching! - ASSERT(m_pManifest->add( - db::manifest::manifest_t::level_record_t{ - .op = level_operation_k::compact_level_k, .level = currentLevel->index() - } - )); + { + // TODO(lnikon): Replace this and following ASSERTS()s with manifest batching! + auto ok{m_pManifest->add( + db::manifest::manifest_t::level_record_t{ + .op = level_operation_k::compact_level_k, .level = currentLevel->index() + } + )}; + ASSERT(ok); + } { + // Update manifest with compacted level // Update manifest with new segment auto ok{m_pManifest->add( db::manifest::manifest_t::segment_record_t{ @@ -197,11 +202,12 @@ auto levels_t::size() const noexcept -> levels_t::levels_storage_t::size_type // Generate name for the segment and add it to the manifest auto name{fmt::format("{}_{}", segments::helpers::segment_name(), 0)}; // TODO(lnikon): Replace this and following ASSERTS()s with manifest batching! - ASSERT(m_pManifest->add( + auto ok{m_pManifest->add( db::manifest::manifest_t::segment_record_t{ .op = segment_operation_k::add_segment_k, .name = name, .level = 0 } - )); + )}; + ASSERT(ok); auto pSegment{m_levels[0]->segment(std::move(memtable), name)}; if (pSegment) @@ -225,21 +231,16 @@ auto levels_t::restore() noexcept -> void void levels_t::compaction_task(std::stop_token stoken) noexcept { - while (true) + while (!stoken.stop_requested()) { - if (stoken.stop_requested()) - { - return; - } + m_level0_segment_flushed_notification.WaitForNotification(); - if (m_level0_segment_flushed_notification.WaitForNotificationWithTimeout(absl::Seconds(1))) + auto ok{compact()}; + if (!ok) { - compact(); - } - else - { - continue; + spdlog::error("Compaction failed."); } + ASSERT(ok); } } diff --git a/lib/structures/lsmtree/lsmtree_config.h b/lib/structures/lsmtree/lsmtree_config.h index a1549bd..969235f 100644 --- a/lib/structures/lsmtree/lsmtree_config.h +++ b/lib/structures/lsmtree/lsmtree_config.h @@ -12,13 +12,13 @@ struct lsmtree_config_t * Determines the size (in Mb) of the in-memory memtable after which it * should be flushed onto the disk. */ - uint64_t DiskFlushThresholdSize{8 * 1024 * 1024}; + std::uint64_t DiskFlushThresholdSize{8 * 1024 * 1024}; /** * Determines number of segments after which compaction process should * start for level 0. */ - uint64_t LevelZeroCompactionThreshold{0}; + std::uint64_t LevelZeroCompactionThreshold{0}; /** * Determines strategy used by compaction process for level 0. @@ -30,7 +30,7 @@ struct lsmtree_config_t * Determines number of segments after which compaction process should * start for level 1 and below. */ - uint64_t LevelNonZeroCompactionThreshold{0}; + std::uint64_t LevelNonZeroCompactionThreshold{0}; /** * Determines strategy used by compaction process for level 1 and below. diff --git a/tests/go/kvtest/cmd/kvtest/kvtest.yml b/tests/go/kvtest/cmd/kvtest/kvtest.yml index 13b7366..8e5115e 100644 --- a/tests/go/kvtest/cmd/kvtest/kvtest.yml +++ b/tests/go/kvtest/cmd/kvtest/kvtest.yml @@ -18,8 +18,8 @@ scenarios: type: memtable_stress enabled: true parameters: - record_size: 16000 - record_count: 8096 + record_size: 33 + record_count: 128 concurrent_writers: 1 flush_threshold: 6400000 - name: basic_crud diff --git a/tests/go/kvtest/pkg/scenarios/storage/memtable_stress.go b/tests/go/kvtest/pkg/scenarios/storage/memtable_stress.go index 403c176..f82db22 100644 --- a/tests/go/kvtest/pkg/scenarios/storage/memtable_stress.go +++ b/tests/go/kvtest/pkg/scenarios/storage/memtable_stress.go @@ -53,9 +53,9 @@ func NewMemtableStressScenario(params map[string]interface{}) *MemtableStressSce } func (s *MemtableStressScenario) Execute(ctx *core.TestContext) error { - log.Printf("[%s] === Test 1: Sequential Writes (Flush Detection) === ", s.name) + log.Printf("[%s] === Test 1: Memtable Stress (Flush Detection) === ", s.name) if err := s.testSequentialWrites(ctx); err != nil { - return fmt.Errorf("sequantial writes failec: %w", err) + return fmt.Errorf("sequantial writes failed: %w", err) } log.Printf("[%s] Sequential writes test passed", s.name) From 4588c83d7c402436e7ac78ef8fb903946f1485b9 Mon Sep 17 00:00:00 2001 From: lnikon Date: Sun, 26 Oct 2025 21:51:28 +0400 Subject: [PATCH 13/17] Integration & load testing: - Project restructure --- CMakeLists.txt | 6 +- examples/absl/CMakeLists.txt | 5 +- examples/absl/concurrency/CMakeLists.txt | 2 +- lib/CMakeLists.txt | 12 --- lib/concurrency/CMakeLists.txt | 14 ---- lib/concurrency/thread_safe_queue.cpp | 1 - lib/config/CMakeLists.txt | 12 --- lib/config/config.cpp | 54 ------------- lib/db/CMakeLists.txt | 40 ---------- lib/db/db_config.cpp | 1 - lib/fs/CMakeLists.txt | 12 --- lib/posix_wrapper/open_flag.cpp | 0 lib/raft/CMakeLists.txt | 22 ------ lib/server/CMakeLists.txt | 5 -- lib/server/tcp_server.cpp | 16 ---- lib/server/tcp_server.h | 21 ------ lib/structures/lsmtree/CMakeLists.txt | 49 ------------ lib/structures/lsmtree/lsmtree_compactor.cpp | 10 --- lib/structures/lsmtree/lsmtree_compactor.h | 6 -- .../lsmtree/lsmtree_reader_writer.h | 50 ------------- .../segments/lsmtree_segment_index.cpp | 16 ---- .../memtable/memtable_reader_writer.cpp | 5 -- lib/structures/skiplist/CMakeLists.txt | 26 ------- lib/structures/skiplist/skiplist.cpp | 1 - lib/structures/sorted_vector/CMakeLists.txt | 24 ------ .../sorted_vector/sorted_vector.cpp | 1 - mise.toml | 57 ++++++++++++++ src/CMakeLists.txt | 25 +++++-- src/concurrency/CMakeLists.txt | 18 +++++ {lib => src}/concurrency/helpers.cpp | 4 +- {lib => src}/concurrency/thread_pool.cpp | 2 +- src/concurrency/thread_safe_queue.cpp | 1 + src/config.h | 29 ------- src/config/CMakeLists.txt | 21 ++++++ src/{ => config}/config.cpp | 75 +++++++++++++++---- src/db/CMakeLists.txt | 43 +++++++++++ {lib => src}/db/db.cpp | 6 +- src/db/db_config.cpp | 1 + {lib => src}/db/db_test.cpp | 0 {lib => src}/db/manifest/manifest.cpp | 0 src/fs/CMakeLists.txt | 22 ++++++ {lib => src}/fs/append_only_file.cpp | 12 +-- {lib => src}/fs/common.cpp | 7 +- {lib => src}/fs/random_access_file.cpp | 2 +- {lib => src}/fs/random_access_file_test.cpp | 2 +- {lib => src/include}/concurrency/helpers.h | 0 .../include}/concurrency/thread_pool.h | 1 - .../include}/concurrency/thread_safe_queue.h | 0 {lib => src/include}/config/config.h | 22 ++++++ {lib => src/include}/db/db.h | 0 {lib => src/include}/db/db_config.h | 0 {lib => src/include}/db/manifest/manifest.h | 2 +- {lib => src/include}/fs/append_only_file.h | 6 +- {lib => src/include}/fs/common.h | 0 {lib => src/include}/fs/random_access_file.h | 0 {lib => src/include}/fs/types.h | 0 .../include}/posix_wrapper/open_flag.h | 0 {lib => src/include}/raft/persistence.h | 0 {lib => src/include}/raft/raft.h | 0 {lib => src/include}/server/grpc_server.h | 8 +- {lib => src/include}/server/server.h | 2 +- {lib => src/include}/server/server_concept.h | 4 +- {lib => src/include}/server/server_config.h | 0 {lib => src/include}/server/server_factory.h | 17 ++--- {lib => src/include}/server/server_kind.h | 3 +- .../include}/structures/hashindex/hashindex.h | 0 .../lsmtree/compaction/compactation.h | 0 .../lsmtree/compaction/compaction_trigger.h | 0 .../compaction/level_zero_compactation.h | 0 .../structures/lsmtree/levels/level.h | 0 .../structures/lsmtree/levels/levels.h | 0 .../include}/structures/lsmtree/lsmtree.h | 2 +- .../structures/lsmtree/lsmtree_config.h | 0 .../structures/lsmtree/lsmtree_types.h | 0 .../structures/lsmtree/segments/helpers.h | 0 .../segments/lsmtree_regular_segment.h | 0 .../segments/lsmtree_segment_factory.h | 3 +- .../lsmtree/segments/lsmtree_segment_index.h | 6 +- .../lsmtree/segments/segment_config.h | 0 .../lsmtree/segments/segment_storage.h | 0 .../structures/lsmtree/segments/types.h | 0 .../structures/lsmtree/segments/uuid.h | 0 .../include}/structures/memtable/memtable.h | 0 .../memtable/memtable_reader_writer.h | 0 .../include}/structures/skiplist/skiplist.h | 0 .../structures/sorted_vector/sorted_vector.h | 0 .../append_only_file_storage_backend.h | 4 +- {lib => src/include}/wal/backend/backend.h | 2 +- {lib => src/include}/wal/common.h | 0 {lib => src/include}/wal/concepts.h | 0 {lib => src/include}/wal/config.h | 0 .../include}/wal/in_memory_log_storage.h | 0 .../include}/wal/persistent_log_storage.h | 6 +- {lib => src/include}/wal/wal.h | 6 +- src/main.cpp | 22 +++--- {lib => src}/posix_wrapper/CMakeLists.txt | 8 +- src/posix_wrapper/open_flag.cpp | 1 + .../posix_wrapper/posix_wrapper_test.cpp | 2 +- src/raft/CMakeLists.txt | 31 ++++++++ {lib => src}/raft/persistence.cpp | 2 +- {lib => src}/raft/raft.cpp | 4 +- {lib => src}/raft/raft_test.cpp | 2 +- src/server/CMakeLists.txt | 6 ++ {lib => src}/server/grpc_server.cpp | 2 +- {lib => src}/server/server_kind.cpp | 7 -- {lib => src}/structures/CMakeLists.txt | 0 .../structures/hashindex/CMakeLists.txt | 12 +-- .../structures/hashindex/hashindex.cpp | 0 src/structures/lsmtree/CMakeLists.txt | 53 +++++++++++++ .../compaction/level_zero_compactation.cpp | 0 .../structures/lsmtree/levels/level.cpp | 4 +- .../structures/lsmtree/levels/levels.cpp | 9 +-- {lib => src}/structures/lsmtree/lsmtree.cpp | 0 .../structures/lsmtree/lsmtree_test.cpp | 0 .../structures/lsmtree/segments/gsl/gsl | 0 .../lsmtree/segments/gsl/gsl_algorithm | 0 .../lsmtree/segments/gsl/gsl_assert | 0 .../structures/lsmtree/segments/gsl/gsl_byte | 0 .../structures/lsmtree/segments/gsl/gsl_util | 0 .../lsmtree/segments/gsl/multi_span | 0 .../structures/lsmtree/segments/gsl/pointers | 0 .../structures/lsmtree/segments/gsl/span | 0 .../lsmtree/segments/gsl/string_span | 0 .../structures/lsmtree/segments/helpers.cpp | 10 +-- .../segments/lsmtree_regular_segment.cpp | 1 + .../segments/lsmtree_segment_factory.cpp | 8 +- .../lsmtree/segments/segment_storage.cpp | 22 +++--- .../structures/memtable/CMakeLists.txt | 14 +++- {lib => src}/structures/memtable/memtable.cpp | 2 +- .../structures/memtable/memtable_test.cpp | 0 src/structures/skiplist/CMakeLists.txt | 27 +++++++ src/structures/skiplist/skiplist.cpp | 1 + .../structures/skiplist/skiplist_test.cpp | 0 src/structures/sorted_vector/CMakeLists.txt | 27 +++++++ .../sorted_vector/sorted_vector.cpp | 1 + .../sorted_vector/sorted_vector_test.cpp | 0 {lib => src}/wal/CMakeLists.txt | 3 +- {lib => src}/wal/common.cpp | 2 +- {lib => src}/wal/wal.cpp | 2 +- {lib => src}/wal/wal_test.cpp | 0 140 files changed, 532 insertions(+), 585 deletions(-) delete mode 100644 lib/CMakeLists.txt delete mode 100644 lib/concurrency/CMakeLists.txt delete mode 100644 lib/concurrency/thread_safe_queue.cpp delete mode 100644 lib/config/CMakeLists.txt delete mode 100644 lib/config/config.cpp delete mode 100644 lib/db/CMakeLists.txt delete mode 100644 lib/db/db_config.cpp delete mode 100644 lib/fs/CMakeLists.txt delete mode 100644 lib/posix_wrapper/open_flag.cpp delete mode 100644 lib/raft/CMakeLists.txt delete mode 100644 lib/server/CMakeLists.txt delete mode 100644 lib/server/tcp_server.cpp delete mode 100644 lib/server/tcp_server.h delete mode 100644 lib/structures/lsmtree/CMakeLists.txt delete mode 100644 lib/structures/lsmtree/lsmtree_compactor.cpp delete mode 100644 lib/structures/lsmtree/lsmtree_compactor.h delete mode 100644 lib/structures/lsmtree/lsmtree_reader_writer.h delete mode 100644 lib/structures/lsmtree/segments/lsmtree_segment_index.cpp delete mode 100644 lib/structures/memtable/memtable_reader_writer.cpp delete mode 100644 lib/structures/skiplist/CMakeLists.txt delete mode 100644 lib/structures/skiplist/skiplist.cpp delete mode 100644 lib/structures/sorted_vector/CMakeLists.txt delete mode 100644 lib/structures/sorted_vector/sorted_vector.cpp create mode 100644 mise.toml create mode 100644 src/concurrency/CMakeLists.txt rename {lib => src}/concurrency/helpers.cpp (91%) rename {lib => src}/concurrency/thread_pool.cpp (97%) create mode 100644 src/concurrency/thread_safe_queue.cpp delete mode 100644 src/config.h create mode 100644 src/config/CMakeLists.txt rename src/{ => config}/config.cpp (83%) create mode 100644 src/db/CMakeLists.txt rename {lib => src}/db/db.cpp (99%) create mode 100644 src/db/db_config.cpp rename {lib => src}/db/db_test.cpp (100%) rename {lib => src}/db/manifest/manifest.cpp (100%) create mode 100644 src/fs/CMakeLists.txt rename {lib => src}/fs/append_only_file.cpp (99%) rename {lib => src}/fs/common.cpp (84%) rename {lib => src}/fs/random_access_file.cpp (99%) rename {lib => src}/fs/random_access_file_test.cpp (99%) rename {lib => src/include}/concurrency/helpers.h (100%) rename {lib => src/include}/concurrency/thread_pool.h (98%) rename {lib => src/include}/concurrency/thread_safe_queue.h (100%) rename {lib => src/include}/config/config.h (54%) rename {lib => src/include}/db/db.h (100%) rename {lib => src/include}/db/db_config.h (100%) rename {lib => src/include}/db/manifest/manifest.h (99%) rename {lib => src/include}/fs/append_only_file.h (96%) rename {lib => src/include}/fs/common.h (100%) rename {lib => src/include}/fs/random_access_file.h (100%) rename {lib => src/include}/fs/types.h (100%) rename {lib => src/include}/posix_wrapper/open_flag.h (100%) rename {lib => src/include}/raft/persistence.h (100%) rename {lib => src/include}/raft/raft.h (100%) rename {lib => src/include}/server/grpc_server.h (93%) rename {lib => src/include}/server/server.h (94%) rename {lib => src/include}/server/server_concept.h (100%) rename {lib => src/include}/server/server_config.h (100%) rename {lib => src/include}/server/server_factory.h (62%) rename {lib => src/include}/server/server_kind.h (87%) rename {lib => src/include}/structures/hashindex/hashindex.h (100%) rename {lib => src/include}/structures/lsmtree/compaction/compactation.h (100%) rename {lib => src/include}/structures/lsmtree/compaction/compaction_trigger.h (100%) rename {lib => src/include}/structures/lsmtree/compaction/level_zero_compactation.h (100%) rename {lib => src/include}/structures/lsmtree/levels/level.h (100%) rename {lib => src/include}/structures/lsmtree/levels/levels.h (100%) rename {lib => src/include}/structures/lsmtree/lsmtree.h (98%) rename {lib => src/include}/structures/lsmtree/lsmtree_config.h (100%) rename {lib => src/include}/structures/lsmtree/lsmtree_types.h (100%) rename {lib => src/include}/structures/lsmtree/segments/helpers.h (100%) rename {lib => src/include}/structures/lsmtree/segments/lsmtree_regular_segment.h (100%) rename {lib => src/include}/structures/lsmtree/segments/lsmtree_segment_factory.h (72%) rename {lib => src/include}/structures/lsmtree/segments/lsmtree_segment_index.h (86%) rename {lib => src/include}/structures/lsmtree/segments/segment_config.h (100%) rename {lib => src/include}/structures/lsmtree/segments/segment_storage.h (100%) rename {lib => src/include}/structures/lsmtree/segments/types.h (100%) rename {lib => src/include}/structures/lsmtree/segments/uuid.h (100%) rename {lib => src/include}/structures/memtable/memtable.h (100%) rename {lib => src/include}/structures/memtable/memtable_reader_writer.h (100%) rename {lib => src/include}/structures/skiplist/skiplist.h (100%) rename {lib => src/include}/structures/sorted_vector/sorted_vector.h (100%) rename {lib => src/include}/wal/backend/append_only_file_storage_backend.h (98%) rename {lib => src/include}/wal/backend/backend.h (99%) rename {lib => src/include}/wal/common.h (100%) rename {lib => src/include}/wal/concepts.h (100%) rename {lib => src/include}/wal/config.h (100%) rename {lib => src/include}/wal/in_memory_log_storage.h (100%) rename {lib => src/include}/wal/persistent_log_storage.h (98%) rename {lib => src/include}/wal/wal.h (97%) rename {lib => src}/posix_wrapper/CMakeLists.txt (67%) create mode 100644 src/posix_wrapper/open_flag.cpp rename {lib => src}/posix_wrapper/posix_wrapper_test.cpp (96%) create mode 100644 src/raft/CMakeLists.txt rename {lib => src}/raft/persistence.cpp (56%) rename {lib => src}/raft/raft.cpp (99%) rename {lib => src}/raft/raft_test.cpp (99%) create mode 100644 src/server/CMakeLists.txt rename {lib => src}/server/grpc_server.cpp (99%) rename {lib => src}/server/server_kind.cpp (77%) rename {lib => src}/structures/CMakeLists.txt (100%) rename {lib => src}/structures/hashindex/CMakeLists.txt (63%) rename {lib => src}/structures/hashindex/hashindex.cpp (100%) create mode 100644 src/structures/lsmtree/CMakeLists.txt rename {lib => src}/structures/lsmtree/compaction/level_zero_compactation.cpp (100%) rename {lib => src}/structures/lsmtree/levels/level.cpp (99%) rename {lib => src}/structures/lsmtree/levels/levels.cpp (97%) rename {lib => src}/structures/lsmtree/lsmtree.cpp (100%) rename {lib => src}/structures/lsmtree/lsmtree_test.cpp (100%) rename {lib => src}/structures/lsmtree/segments/gsl/gsl (100%) rename {lib => src}/structures/lsmtree/segments/gsl/gsl_algorithm (100%) rename {lib => src}/structures/lsmtree/segments/gsl/gsl_assert (100%) rename {lib => src}/structures/lsmtree/segments/gsl/gsl_byte (100%) rename {lib => src}/structures/lsmtree/segments/gsl/gsl_util (100%) rename {lib => src}/structures/lsmtree/segments/gsl/multi_span (100%) rename {lib => src}/structures/lsmtree/segments/gsl/pointers (100%) rename {lib => src}/structures/lsmtree/segments/gsl/span (100%) rename {lib => src}/structures/lsmtree/segments/gsl/string_span (100%) rename {lib => src}/structures/lsmtree/segments/helpers.cpp (89%) rename {lib => src}/structures/lsmtree/segments/lsmtree_regular_segment.cpp (99%) rename {lib => src}/structures/lsmtree/segments/lsmtree_segment_factory.cpp (63%) rename {lib => src}/structures/lsmtree/segments/segment_storage.cpp (89%) rename {lib => src}/structures/memtable/CMakeLists.txt (50%) rename {lib => src}/structures/memtable/memtable.cpp (99%) rename {lib => src}/structures/memtable/memtable_test.cpp (100%) create mode 100644 src/structures/skiplist/CMakeLists.txt create mode 100644 src/structures/skiplist/skiplist.cpp rename {lib => src}/structures/skiplist/skiplist_test.cpp (100%) create mode 100644 src/structures/sorted_vector/CMakeLists.txt create mode 100644 src/structures/sorted_vector/sorted_vector.cpp rename {lib => src}/structures/sorted_vector/sorted_vector_test.cpp (100%) rename {lib => src}/wal/CMakeLists.txt (76%) rename {lib => src}/wal/common.cpp (96%) rename {lib => src}/wal/wal.cpp (88%) rename {lib => src}/wal/wal_test.cpp (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 110b004..779f998 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,13 +49,9 @@ find_package(libassert REQUIRED) find_package(magic_enum REQUIRED) find_package(liburing REQUIRED) -include_directories(lib) - add_subdirectory(proto) -add_subdirectory(lib) add_subdirectory(src) -add_subdirectory(bench) -add_subdirectory(examples) +# add_subdirectory(bench) # Custom targets to build docker images and run tests add_custom_target( diff --git a/examples/absl/CMakeLists.txt b/examples/absl/CMakeLists.txt index cd775fc..3a603c6 100644 --- a/examples/absl/CMakeLists.txt +++ b/examples/absl/CMakeLists.txt @@ -3,5 +3,6 @@ project(frankie) add_executable(absl_try_mutex absl_try_mutex.cpp) set_target_properties(absl_try_mutex PROPERTIES CXX_STANDARD 23) -target_include_directories(absl_try_mutex INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(absl_try_mutex abseil::abseil) \ No newline at end of file +target_include_directories(absl_try_mutex PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(absl_try_mutex abseil::abseil) + diff --git a/examples/absl/concurrency/CMakeLists.txt b/examples/absl/concurrency/CMakeLists.txt index fdc6f4c..faf65b9 100644 --- a/examples/absl/concurrency/CMakeLists.txt +++ b/examples/absl/concurrency/CMakeLists.txt @@ -4,7 +4,7 @@ project(frankie) add_library(Concurrency thread_safe_queue.h thread_safe_queue.cpp) set_target_properties(Concurrency PROPERTIES CXX_STANDARD 23) -target_include_directories(Concurrency INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(Concurrency PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(Concurrency abseil::abseil spdlog::spdlog fmt::fmt) # add_executable(DBTest "db_test.cpp") diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt deleted file mode 100644 index 878cf89..0000000 --- a/lib/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -cmake_minimum_required(VERSION 3.25) -project(frankie) - -add_subdirectory(config) -add_subdirectory(db) -add_subdirectory(structures) -add_subdirectory(fs) -add_subdirectory(concurrency) -add_subdirectory(server) -add_subdirectory(raft) -add_subdirectory(wal) -add_subdirectory(posix_wrapper) diff --git a/lib/concurrency/CMakeLists.txt b/lib/concurrency/CMakeLists.txt deleted file mode 100644 index 41a820d..0000000 --- a/lib/concurrency/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -cmake_minimum_required(VERSION 3.25) -project(frankie) - -add_library(Concurrency thread_safe_queue.h thread_safe_queue.cpp thread_pool.h - thread_pool.cpp helpers.cpp) - -set_target_properties(Concurrency PROPERTIES CXX_STANDARD 23) -target_include_directories(Concurrency INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(Concurrency PUBLIC abseil::abseil spdlog::spdlog fmt::fmt - libassert::assert) - -# add_executable(DBTest "db_test.cpp") set_target_properties(DBTest PROPERTIES -# CXX_STANDARD 23) target_link_libraries( DBTest Catch2::Catch2WithMain -# spdlog::spdlog fmt::fmt LSMTree MemTable DB Config HashIndex) diff --git a/lib/concurrency/thread_safe_queue.cpp b/lib/concurrency/thread_safe_queue.cpp deleted file mode 100644 index 21d8d72..0000000 --- a/lib/concurrency/thread_safe_queue.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "thread_safe_queue.h" \ No newline at end of file diff --git a/lib/config/CMakeLists.txt b/lib/config/CMakeLists.txt deleted file mode 100644 index 8b135d5..0000000 --- a/lib/config/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -cmake_minimum_required(VERSION 3.25) -project(frankie) - -add_library(Config "config.cpp") - -set_target_properties(Config PROPERTIES CXX_STANDARD 23) -target_include_directories(Config INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(Config PUBLIC libassert::assert fmt::fmt) - -# add_executable(ConfigTest "db_test.cpp") set_target_properties(ConfigTest -# PROPERTIES CXX_STANDARD 23) target_link_libraries( ConfigTest boost::boost -# Catch2::Catch2 spdlog::spdlog fmt::fmt LSMTree MemTable Config) diff --git a/lib/config/config.cpp b/lib/config/config.cpp deleted file mode 100644 index c368b5e..0000000 --- a/lib/config/config.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include -#include - -#include - -#include "config.h" -#include "structures/lsmtree/segments/helpers.h" - -namespace config -{ - -static constexpr const std::string_view SegmentsDirectoryName{"segments"}; -static constexpr const std::string_view ManifestCurrentFilename("current"); - -[[nodiscard]] auto config_t::datadir_path() const -> fs::path_t -{ - return DatabaseConfig.DatabasePath / SegmentsDirectoryName; -} - -[[nodiscard]] auto config_t::manifest_path() const -> fs::path_t -{ - const auto current_path{DatabaseConfig.DatabasePath / ManifestCurrentFilename}; - - std::ifstream current(current_path); - if (!current.is_open()) - { - // File doesn't exist, create it - std::ofstream current_out(current_path); - if (!current_out.is_open()) - { - throw std::runtime_error("unable to create \"current\" " + current_path.string()); - } - - const auto new_filename{ - DatabaseConfig.DatabasePath / - fmt::format("manifest_{}", structures::lsmtree::segments::helpers::uuid()) - }; - current_out << new_filename.string() << '\n'; - current_out.flush(); - current_out.close(); - - return new_filename; - } - - std::string latest_filename; - if (!(current >> latest_filename) || latest_filename.empty()) - { - throw std::runtime_error("invalid or empty manifest filename in \"current\" file"); - } - - return latest_filename; -} - -} // namespace config diff --git a/lib/db/CMakeLists.txt b/lib/db/CMakeLists.txt deleted file mode 100644 index 266aabe..0000000 --- a/lib/db/CMakeLists.txt +++ /dev/null @@ -1,40 +0,0 @@ -cmake_minimum_required(VERSION 3.25) -project(frankie) - -add_library(DB db.cpp db_config.cpp manifest/manifest.cpp) - -set_target_properties(DB PROPERTIES CXX_STANDARD 23) -target_include_directories(DB INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries( - DB - PUBLIC abseil::abseil - spdlog::spdlog - fmt::fmt - nlohmann_json::nlohmann_json - nlohmann_json_schema_validator - cxxopts::cxxopts - gRPC::grpc - libassert::assert - magic_enum::magic_enum - TKVProtoObjects - LSMTree - MemTable - Config - HashIndex - WAL) - -add_executable(DBTest db_test.cpp) -set_target_properties(DBTest PROPERTIES CXX_STANDARD 23) -target_link_libraries( - DBTest - gtest::gtest - spdlog::spdlog - fmt::fmt - LSMTree - MemTable - DB - Config - HashIndex) - -include(GoogleTest) -gtest_discover_tests(DBTest) diff --git a/lib/db/db_config.cpp b/lib/db/db_config.cpp deleted file mode 100644 index 3237fd8..0000000 --- a/lib/db/db_config.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "db_config.h" diff --git a/lib/fs/CMakeLists.txt b/lib/fs/CMakeLists.txt deleted file mode 100644 index 6a6c90d..0000000 --- a/lib/fs/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -cmake_minimum_required(VERSION 3.25) -project(frankie) - -add_library(FS append_only_file.cpp random_access_file.cpp common.cpp) - -set_target_properties(FS PROPERTIES CXX_STANDARD 23) -target_include_directories(FS INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(FS PUBLIC liburing::liburing spdlog::spdlog magic_enum::magic_enum POSIXWrapper) - -add_executable(FSTest random_access_file_test.cpp) -set_target_properties(FSTest PROPERTIES CXX_STANDARD 23) -target_link_libraries(FSTest gtest::gtest FS liburing::liburing spdlog::spdlog POSIXWrapper) diff --git a/lib/posix_wrapper/open_flag.cpp b/lib/posix_wrapper/open_flag.cpp deleted file mode 100644 index e69de29..0000000 diff --git a/lib/raft/CMakeLists.txt b/lib/raft/CMakeLists.txt deleted file mode 100644 index 1eb3850..0000000 --- a/lib/raft/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -cmake_minimum_required(VERSION 3.25) -project(frankie) - -add_library(Raft raft.cpp) -set_target_properties(Raft PROPERTIES CXX_STANDARD 23) -target_link_libraries( - Raft - RaftProtoObjects - FS - WAL - libassert::assert - spdlog::spdlog - fmt::fmt - magic_enum::magic_enum) - -add_executable(RaftTest "raft_test.cpp") -set_target_properties(RaftTest PROPERTIES CXX_STANDARD 23) -target_link_libraries(RaftTest gtest::gtest Raft RaftProtoObjects - TKVProtoObjects Concurrency) - -include(GoogleTest) -gtest_discover_tests(RaftTest) diff --git a/lib/server/CMakeLists.txt b/lib/server/CMakeLists.txt deleted file mode 100644 index bb56e53..0000000 --- a/lib/server/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -cmake_minimum_required(VERSION 3.25) -project(frankie) - -add_library(Server "server_kind.cpp" "tcp_server.cpp" "grpc_server.cpp") -target_link_libraries(Server DB fmt::fmt) diff --git a/lib/server/tcp_server.cpp b/lib/server/tcp_server.cpp deleted file mode 100644 index 430ee07..0000000 --- a/lib/server/tcp_server.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "server/tcp_server.h" - -namespace server::tcp_communication -{ - -void tcp_communication_t::start(db::shared_ptr_t database) const noexcept -{ - (void)database; - spdlog::error("TCP server is not implemented yet"); -} - -void tcp_communication_t::shutdown() const noexcept -{ -} - -} // namespace server::tcp_communication diff --git a/lib/server/tcp_server.h b/lib/server/tcp_server.h deleted file mode 100644 index 4ec3cfa..0000000 --- a/lib/server/tcp_server.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include "server/server_concept.h" -#include "server_kind.h" - -namespace server::tcp_communication -{ - -class tcp_communication_t final -{ - public: - static constexpr const auto kind = communication_strategy_kind_k::tcp_k; - - void start(db::shared_ptr_t database) const noexcept; - void shutdown() const noexcept; -}; - -static_assert(server::communication_strategy_t, - "TCPServerCommunication must satisfy CommunicationStrategy concept"); - -} // namespace server::tcp_communication diff --git a/lib/structures/lsmtree/CMakeLists.txt b/lib/structures/lsmtree/CMakeLists.txt deleted file mode 100644 index 8a07903..0000000 --- a/lib/structures/lsmtree/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ -cmake_minimum_required(VERSION 3.25) - -add_library( - LSMTree - "lsmtree.cpp" - "segments/lsmtree_regular_segment.cpp" - "segments/lsmtree_segment_factory.cpp" - "segments/segment_storage.cpp" - "segments/uuid.h" - "segments/helpers.cpp" - "segments/types.h" - "levels/levels.cpp" - "levels/level.cpp") - -set_target_properties(LSMTree PROPERTIES CXX_STANDARD 23) -target_include_directories( - LSMTree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/segments) -target_link_libraries( - LSMTree - PUBLIC spdlog::spdlog - abseil::abseil - libassert::assert - magic_enum::magic_enum - Concurrency - WAL - Raft - RaftProtoObjects) - -add_executable(LSMTreeTest "lsmtree_test.cpp") -set_target_properties(LSMTreeTest PROPERTIES CXX_STANDARD 23) -target_link_libraries( - LSMTreeTest - PRIVATE Catch2::Catch2WithMain - spdlog::spdlog - fmt::fmt - LSMTree - MemTable - Config - HashIndex - WAL - Raft - FS - DB - Concurrency) - -# Register Catch2 tests with CTest -include(Catch) -catch_discover_tests(LSMTreeTest) diff --git a/lib/structures/lsmtree/lsmtree_compactor.cpp b/lib/structures/lsmtree/lsmtree_compactor.cpp deleted file mode 100644 index a0c843c..0000000 --- a/lib/structures/lsmtree/lsmtree_compactor.cpp +++ /dev/null @@ -1,10 +0,0 @@ -// -// Created by nikon on 1/26/22. -// - -// #include "LSMTreeCompactor.h" - -namespace structures::lsmtree -{ -// void compactLSMTree(MemTableUniquePtr memTable) {} -} // namespace structures::lsmtree diff --git a/lib/structures/lsmtree/lsmtree_compactor.h b/lib/structures/lsmtree/lsmtree_compactor.h deleted file mode 100644 index 71985b3..0000000 --- a/lib/structures/lsmtree/lsmtree_compactor.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once - -namespace structures::lsmtree -{ - -} // namespace structures::lsmtree diff --git a/lib/structures/lsmtree/lsmtree_reader_writer.h b/lib/structures/lsmtree/lsmtree_reader_writer.h deleted file mode 100644 index 2c1a999..0000000 --- a/lib/structures/lsmtree/lsmtree_reader_writer.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include - -namespace structures::lsmtree -{ - -// TOD(vahag): WTF is this file? - -template struct generic_reader_t -{ - void read(T &result) - { - } -}; - -template struct generic_writer_t -{ - void write(T &result) - { - } -}; - -struct lsmtree_segment_t -{ - void read() - { - } - void write() - { - } -}; - -struct lsmtree_writer_t : generic_writer_t -{ -}; - -struct lsmtree_reader_t : generic_reader_t -{ -}; - -struct lsmtree_segment_writer_t : generic_writer_t -{ -}; - -struct lsmtree_segment_reader_t : generic_writer_t -{ -}; - -} // namespace structures::lsmtree diff --git a/lib/structures/lsmtree/segments/lsmtree_segment_index.cpp b/lib/structures/lsmtree/segments/lsmtree_segment_index.cpp deleted file mode 100644 index 8118fba..0000000 --- a/lib/structures/lsmtree/segments/lsmtree_segment_index.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include - -namespace structures::lsmtree -{ - -lsmtree_segment_index_t::offset_type_t lsmtree_segment_index_t::write(const key_t &key) -{ - return m_lastOffset; -} - -lsmtree_segment_index_t::offset_type_t lsmtree_segment_index_t::read(const key_t &key) const -{ - return m_lastOffset; -} - -} // namespace structures::lsmtree diff --git a/lib/structures/memtable/memtable_reader_writer.cpp b/lib/structures/memtable/memtable_reader_writer.cpp deleted file mode 100644 index 9667b86..0000000 --- a/lib/structures/memtable/memtable_reader_writer.cpp +++ /dev/null @@ -1,5 +0,0 @@ -// -// Created by nikon on 1/26/22. -// - -#include "memtable_reader_writer.h" diff --git a/lib/structures/skiplist/CMakeLists.txt b/lib/structures/skiplist/CMakeLists.txt deleted file mode 100644 index 3554acb..0000000 --- a/lib/structures/skiplist/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -cmake_minimum_required(VERSION 3.25) -project(frankie) - -add_library( - SkipList - "skiplist.cpp") - -set_target_properties(SkipList PROPERTIES CXX_STANDARD 23) -target_include_directories(SkipList INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) - -add_executable(SkipListTest "skiplist_test.cpp") -set_target_properties(SkipListTest PROPERTIES CXX_STANDARD 23) -target_link_libraries( - SkipListTest - Catch2::Catch2WithMain - spdlog::spdlog - fmt::fmt - LSMTree - SkipList - MemTable - Config -) - -# Register Catch2 tests with CTest -include(Catch) -catch_discover_tests(SkipListTest) diff --git a/lib/structures/skiplist/skiplist.cpp b/lib/structures/skiplist/skiplist.cpp deleted file mode 100644 index bf10872..0000000 --- a/lib/structures/skiplist/skiplist.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "skiplist.h" \ No newline at end of file diff --git a/lib/structures/sorted_vector/CMakeLists.txt b/lib/structures/sorted_vector/CMakeLists.txt deleted file mode 100644 index 830cce4..0000000 --- a/lib/structures/sorted_vector/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -cmake_minimum_required(VERSION 3.25) -project(frankie) - -add_library(SortedVector "sorted_vector.cpp") -target_compile_features(SortedVector PUBLIC cxx_std_23) -target_include_directories(SortedVector INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) - -add_executable(SortedVectorTest "sorted_vector_test.cpp") -set_target_properties(SortedVectorTest PROPERTIES CXX_STANDARD 23) -target_link_libraries( - SortedVectorTest - Catch2::Catch2WithMain - spdlog::spdlog - fmt::fmt - LSMTree - SkipList - MemTable - Config - SortedVector) - - -# Register Catch2 tests with CTest -include(Catch) -catch_discover_tests(SortedVectorTest) diff --git a/lib/structures/sorted_vector/sorted_vector.cpp b/lib/structures/sorted_vector/sorted_vector.cpp deleted file mode 100644 index 56aaf69..0000000 --- a/lib/structures/sorted_vector/sorted_vector.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "sorted_vector.h" diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..2a93154 --- /dev/null +++ b/mise.toml @@ -0,0 +1,57 @@ +[tools] +node = "latest" +python = "latest" +buf = "latest" + +[env] +PROFILE = "debug" +COMPILER = "clang" +JOBS = "24" +IMAGE = "silkeh/clang" +IMAGE_VERSION = "latest" + +[tasks.conan] +run = [ + "conan install . --output-folder=build --build=missing --profile=./conan/profiles/$PROFILE-$COMPILER", + "cmake --preset conan-$PROFILE", +] + +[tasks.build] +run = ["cmake --build ./build -j$JOBS"] + +[tasks.image] +run = """ + podman build \ + --build-arg TARGET=$IMAGE:$IMAGE_VERSION \ + --build-arg COMPILER=$COMPILER \ + --build-arg BUILD_TYPE=$PROFILE . --file Dockerfile --tag tinykvpp-$COMPILER + """ + +[tasks.up_standalone] +run = """ + cd infra + podman compose -f docker-compose.standalone.yaml up -d +""" + +[tasks.down_standalone] +run = """ + cd infra + podman compose -f docker-compose.standalone.yaml down -v +""" + +[tasks.up_cluster] +run = """ + cd infra + podman compose -f docker-compose.cluster.yaml up -d +""" + +[tasks.down_cluster] +run = """ + cd infra + podman compose -f docker-compose.cluster.yaml down -v +""" + +[tasks.kvtest] +run = """ + ./tests/go/kvtest/cmd/kvtest/kvtest -c {{arg(name="kvtest.yml", default="./tests/go/kvtest/cmd/kvtest/kvtest.yml")}} run +""" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 49a2780..fb20add 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,13 +1,26 @@ cmake_minimum_required(VERSION 3.25) project(frankie) -add_executable(Main main.cpp config.cpp) +add_subdirectory(config) +add_subdirectory(db) +add_subdirectory(structures) +add_subdirectory(fs) +add_subdirectory(concurrency) +add_subdirectory(server) +add_subdirectory(raft) +add_subdirectory(wal) +add_subdirectory(posix_wrapper) +add_executable(Main main.cpp) set_target_properties(Main PROPERTIES CXX_STANDARD 23) -target_include_directories(Main INTERFACE ${proto_import_dirs} - ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(Main PUBLIC WAL DB Server Raft - RaftProtoObjects) +target_include_directories( + Main + PUBLIC ${proto_import_dirs} ${CMAKE_CURRENT_SOURCE_DIR}/include +) +target_link_libraries( + Main + PUBLIC WAL DB Server Raft Config RaftProtoObjects +) -# target_include_directories(GrpcAppMain INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} +# target_include_directories(GrpcAppMain PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} # ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/src/concurrency/CMakeLists.txt b/src/concurrency/CMakeLists.txt new file mode 100644 index 0000000..ae1c179 --- /dev/null +++ b/src/concurrency/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.25) +project(frankie) + +add_library(Concurrency thread_safe_queue.cpp thread_pool.cpp helpers.cpp) + +set_target_properties(Concurrency PROPERTIES CXX_STANDARD 23) +target_include_directories( + Concurrency + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include +) +target_link_libraries( + Concurrency + PUBLIC abseil::abseil spdlog::spdlog fmt::fmt libassert::assert +) + +# add_executable(DBTest "db_test.cpp") set_target_properties(DBTest PROPERTIES +# CXX_STANDARD 23) target_link_libraries( DBTest Catch2::Catch2WithMain +# spdlog::spdlog fmt::fmt LSMTree MemTable DB Config HashIndex) diff --git a/lib/concurrency/helpers.cpp b/src/concurrency/helpers.cpp similarity index 91% rename from lib/concurrency/helpers.cpp rename to src/concurrency/helpers.cpp index e124d6c..ed06977 100644 --- a/lib/concurrency/helpers.cpp +++ b/src/concurrency/helpers.cpp @@ -1,6 +1,6 @@ #include -#include "helpers.h" +#include "concurrency/helpers.h" namespace concurrency { @@ -31,4 +31,4 @@ absl_dual_mutex_lock_guard::~absl_dual_mutex_lock_guard() noexcept m_first_mutex->Unlock(); } -} // namespace concurrency \ No newline at end of file +} // namespace concurrency diff --git a/lib/concurrency/thread_pool.cpp b/src/concurrency/thread_pool.cpp similarity index 97% rename from lib/concurrency/thread_pool.cpp rename to src/concurrency/thread_pool.cpp index 1090d55..98bbc4c 100644 --- a/lib/concurrency/thread_pool.cpp +++ b/src/concurrency/thread_pool.cpp @@ -1,4 +1,4 @@ -#include "thread_pool.h" +#include "concurrency/thread_pool.h" #include diff --git a/src/concurrency/thread_safe_queue.cpp b/src/concurrency/thread_safe_queue.cpp new file mode 100644 index 0000000..47054d1 --- /dev/null +++ b/src/concurrency/thread_safe_queue.cpp @@ -0,0 +1 @@ +#include "concurrency/thread_safe_queue.h" diff --git a/src/config.h b/src/config.h deleted file mode 100644 index e76d20f..0000000 --- a/src/config.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include -#include - -#include "config/config.h" - -using nlohmann::json; -using nlohmann::json_schema::json_validator; -using json = nlohmann::json; - -auto loadConfigJson(const std::string &configPath) -> json; - -void validateConfigJson(const json &configJson); - -void configureLogging(const std::string &loggingLevel); - -auto loadDatabaseConfig(const json &configJson) -> config::shared_ptr_t; - -void loadWALConfig(const json &walConfig, config::shared_ptr_t dbConfig); - -void loadLSMTreeConfig( - const json &lsmtreeConfig, config::shared_ptr_t dbConfig, const std::string &configPath -); - -auto loadServerConfig(const json &configJson, config::shared_ptr_t dbConfig); - -auto initializeDatabaseConfig(const json &configJson, const std::string &configPath) - -> config::shared_ptr_t; diff --git a/src/config/CMakeLists.txt b/src/config/CMakeLists.txt new file mode 100644 index 0000000..7430291 --- /dev/null +++ b/src/config/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.25) +project(frankie) + +add_library(Config config.cpp) + +set_target_properties(Config PROPERTIES CXX_STANDARD 23) +target_include_directories(Config PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include) +target_link_libraries( + Config + PUBLIC + libassert::assert + fmt::fmt + nlohmann_json::nlohmann_json + nlohmann_json_schema_validator + cxxopts::cxxopts + magic_enum::magic_enum +) + +# add_executable(ConfigTest "db_test.cpp") set_target_properties(ConfigTest +# PROPERTIES CXX_STANDARD 23) target_link_libraries( ConfigTest boost::boost +# Catch2::Catch2 spdlog::spdlog fmt::fmt LSMTree MemTable Config) diff --git a/src/config.cpp b/src/config/config.cpp similarity index 83% rename from src/config.cpp rename to src/config/config.cpp index 5ab3fcf..03b3109 100644 --- a/src/config.cpp +++ b/src/config/config.cpp @@ -1,17 +1,24 @@ +#include #include #include #include #include -#include #include +#include -#include "config.h" -#include "db_config.h" +#include "config/config.h" #include "wal/common.h" +#include "structures/lsmtree/segments/helpers.h" + +namespace config +{ + +static constexpr const std::string_view SegmentsDirectoryName{"segments"}; +static constexpr const std::string_view ManifestCurrentFilename("current"); // JSON schema for the configuration file -static const json database_config_schema = R"( +static const nlohmann::json database_config_schema = R"( { "$id": "https://json-schema.hyperjump.io/schema", "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -150,21 +157,60 @@ static const json database_config_schema = R"( } })"_json; -auto loadConfigJson(const std::string &configPath) -> json +[[nodiscard]] auto config_t::datadir_path() const -> fs::path_t +{ + return DatabaseConfig.DatabasePath / SegmentsDirectoryName; +} + +[[nodiscard]] auto config_t::manifest_path() const -> fs::path_t +{ + const auto current_path{DatabaseConfig.DatabasePath / ManifestCurrentFilename}; + + std::ifstream current(current_path); + if (!current.is_open()) + { + // File doesn't exist, create it + std::ofstream current_out(current_path); + if (!current_out.is_open()) + { + throw std::runtime_error("unable to create \"current\" " + current_path.string()); + } + + const auto new_filename{ + DatabaseConfig.DatabasePath / + fmt::format("manifest_{}", structures::lsmtree::segments::helpers::uuid()) + }; + current_out << new_filename.string() << '\n'; + current_out.flush(); + current_out.close(); + + return new_filename; + } + + std::string latest_filename; + if (!(current >> latest_filename) || latest_filename.empty()) + { + throw std::runtime_error("invalid or empty manifest filename in \"current\" file"); + } + + return latest_filename; +} + +auto loadConfigJson(const std::string &configPath) -> nlohmann::json { std::fstream configStream(configPath, std::fstream::in); if (!configStream.is_open()) { throw std::runtime_error(fmt::format("Unable to open config file: {}", configPath)); } - return json::parse(configStream); + return nlohmann::json::parse(configStream); } -void validateConfigJson(const json &configJson) +void validateConfigJson(const nlohmann::json &configJson) { try { - json_validator validator; + nlohmann::json_schema::json_validator validator; validator.set_root_schema(database_config_schema); validator.validate(configJson); } @@ -199,7 +245,7 @@ void configureLogging(const std::string &loggingLevel) } } -auto loadDatabaseConfig(const json &configJson) -> config::shared_ptr_t +auto loadDatabaseConfig(const nlohmann::json &configJson) -> config::shared_ptr_t { auto dbConfig = config::make_shared(); @@ -223,7 +269,7 @@ auto loadDatabaseConfig(const json &configJson) -> config::shared_ptr_t return dbConfig; } -void loadWALConfig(const json &walConfig, config::shared_ptr_t dbConfig) +void loadWALConfig(const nlohmann::json &walConfig, config::shared_ptr_t dbConfig) { if (walConfig.contains("enable")) { @@ -255,7 +301,9 @@ void loadWALConfig(const json &walConfig, config::shared_ptr_t dbConfig) } void loadLSMTreeConfig( - const json &lsmtreeConfig, config::shared_ptr_t dbConfig, const std::string &configPath + const nlohmann::json &lsmtreeConfig, + config::shared_ptr_t dbConfig, + const std::string &configPath ) { if (lsmtreeConfig.contains("flushThreshold")) @@ -345,7 +393,7 @@ void loadLSMTreeConfig( } } -auto loadServerConfig(const json &configJson, config::shared_ptr_t dbConfig) +auto loadServerConfig(const nlohmann::json &configJson, config::shared_ptr_t dbConfig) { if (configJson.contains("host")) { @@ -393,7 +441,7 @@ auto loadServerConfig(const json &configJson, config::shared_ptr_t dbConfig) } } -auto initializeDatabaseConfig(const json &configJson, const std::string &configPath) +auto initializeDatabaseConfig(const nlohmann::json &configJson, const std::string &configPath) -> config::shared_ptr_t { auto dbConfig = loadDatabaseConfig(configJson); @@ -427,3 +475,4 @@ auto initializeDatabaseConfig(const json &configJson, const std::string &configP return dbConfig; } +} // namespace config diff --git a/src/db/CMakeLists.txt b/src/db/CMakeLists.txt new file mode 100644 index 0000000..293400f --- /dev/null +++ b/src/db/CMakeLists.txt @@ -0,0 +1,43 @@ +cmake_minimum_required(VERSION 3.25) +project(frankie) + +add_library(DB db.cpp db_config.cpp manifest/manifest.cpp) + +set_target_properties(DB PROPERTIES CXX_STANDARD 23) +target_include_directories(DB PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include) +target_link_libraries( + DB + PUBLIC + abseil::abseil + spdlog::spdlog + fmt::fmt + nlohmann_json::nlohmann_json + nlohmann_json_schema_validator + cxxopts::cxxopts + gRPC::grpc + libassert::assert + magic_enum::magic_enum + TKVProtoObjects + LSMTree + MemTable + Config + HashIndex + WAL +) + +add_executable(DBTest db_test.cpp) +set_target_properties(DBTest PROPERTIES CXX_STANDARD 23) +target_link_libraries( + DBTest + gtest::gtest + spdlog::spdlog + fmt::fmt + LSMTree + MemTable + DB + Config + HashIndex +) + +include(GoogleTest) +gtest_discover_tests(DBTest) diff --git a/lib/db/db.cpp b/src/db/db.cpp similarity index 99% rename from lib/db/db.cpp rename to src/db/db.cpp index d3a213a..8a6c922 100644 --- a/lib/db/db.cpp +++ b/src/db/db.cpp @@ -5,10 +5,10 @@ #include #include -#include "db.h" +#include "db/db.h" #include "db/manifest/manifest.h" -#include "lsmtree.h" -#include "memtable.h" +#include "structures/lsmtree/lsmtree.h" +#include "structures/memtable/memtable.h" #include "raft/raft.h" namespace diff --git a/src/db/db_config.cpp b/src/db/db_config.cpp new file mode 100644 index 0000000..8a26f7c --- /dev/null +++ b/src/db/db_config.cpp @@ -0,0 +1 @@ +#include "db/db_config.h" diff --git a/lib/db/db_test.cpp b/src/db/db_test.cpp similarity index 100% rename from lib/db/db_test.cpp rename to src/db/db_test.cpp diff --git a/lib/db/manifest/manifest.cpp b/src/db/manifest/manifest.cpp similarity index 100% rename from lib/db/manifest/manifest.cpp rename to src/db/manifest/manifest.cpp diff --git a/src/fs/CMakeLists.txt b/src/fs/CMakeLists.txt new file mode 100644 index 0000000..5859b02 --- /dev/null +++ b/src/fs/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.25) +project(frankie) + +add_library(FS append_only_file.cpp random_access_file.cpp common.cpp) + +set_target_properties(FS PROPERTIES CXX_STANDARD 23) +target_include_directories(FS PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include) +target_link_libraries( + FS + PUBLIC liburing::liburing spdlog::spdlog magic_enum::magic_enum POSIXWrapper +) + +add_executable(FSTest random_access_file_test.cpp) +set_target_properties(FSTest PROPERTIES CXX_STANDARD 23) +target_link_libraries( + FSTest + gtest::gtest + FS + liburing::liburing + spdlog::spdlog + POSIXWrapper +) diff --git a/lib/fs/append_only_file.cpp b/src/fs/append_only_file.cpp similarity index 99% rename from lib/fs/append_only_file.cpp rename to src/fs/append_only_file.cpp index 5ead4c8..254e4b3 100644 --- a/lib/fs/append_only_file.cpp +++ b/src/fs/append_only_file.cpp @@ -1,13 +1,13 @@ -#include "append_only_file.h" -#include "fs/common.h" -#include "posix_wrapper/open_flag.h" +#include +#include +#include #include #include -#include -#include -#include +#include "fs/append_only_file.h" +#include "fs/common.h" +#include "posix_wrapper/open_flag.h" namespace pw = posix_wrapper; diff --git a/lib/fs/common.cpp b/src/fs/common.cpp similarity index 84% rename from lib/fs/common.cpp rename to src/fs/common.cpp index cbe5fa3..7f8ec50 100644 --- a/lib/fs/common.cpp +++ b/src/fs/common.cpp @@ -1,4 +1,4 @@ -#include "common.h" +#include "fs/common.h" #include @@ -14,11 +14,12 @@ auto fs::file_error_t::from_errno(file_error_code_k code, int err, const char *c -> file_error_t { return { - .code = code, .system_errno = err, .message = std::string(context) + ": " + strerror(err)}; + .code = code, .system_errno = err, .message = std::string(context) + ": " + strerror(err) + }; } auto fs::file_error_t::success() noexcept -> file_error_t { return file_error_t{}; } -} // namespace fs \ No newline at end of file +} // namespace fs diff --git a/lib/fs/random_access_file.cpp b/src/fs/random_access_file.cpp similarity index 99% rename from lib/fs/random_access_file.cpp rename to src/fs/random_access_file.cpp index a886135..f17a095 100644 --- a/lib/fs/random_access_file.cpp +++ b/src/fs/random_access_file.cpp @@ -8,7 +8,7 @@ #include #include -#include "random_access_file.h" +#include "fs/random_access_file.h" #include "fs/common.h" namespace fs::random_access_file diff --git a/lib/fs/random_access_file_test.cpp b/src/fs/random_access_file_test.cpp similarity index 99% rename from lib/fs/random_access_file_test.cpp rename to src/fs/random_access_file_test.cpp index 2d24664..9d13a26 100644 --- a/lib/fs/random_access_file_test.cpp +++ b/src/fs/random_access_file_test.cpp @@ -11,7 +11,7 @@ #include #include -#include "random_access_file.h" +#include "fs/random_access_file.h" using namespace fs; using namespace fs::random_access_file; diff --git a/lib/concurrency/helpers.h b/src/include/concurrency/helpers.h similarity index 100% rename from lib/concurrency/helpers.h rename to src/include/concurrency/helpers.h diff --git a/lib/concurrency/thread_pool.h b/src/include/concurrency/thread_pool.h similarity index 98% rename from lib/concurrency/thread_pool.h rename to src/include/concurrency/thread_pool.h index 2deb3d7..2649200 100644 --- a/lib/concurrency/thread_pool.h +++ b/src/include/concurrency/thread_pool.h @@ -6,7 +6,6 @@ #include #include #include -#include #include diff --git a/lib/concurrency/thread_safe_queue.h b/src/include/concurrency/thread_safe_queue.h similarity index 100% rename from lib/concurrency/thread_safe_queue.h rename to src/include/concurrency/thread_safe_queue.h diff --git a/lib/config/config.h b/src/include/config/config.h similarity index 54% rename from lib/config/config.h rename to src/include/config/config.h index 703b660..56b8964 100644 --- a/lib/config/config.h +++ b/src/include/config/config.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "db/db_config.h" #include "structures/lsmtree/lsmtree_config.h" #include "structures/lsmtree/segments/segment_config.h" @@ -28,4 +30,24 @@ template auto make_shared(Args... args) return std::make_shared(std::forward(args)...); } +auto loadConfigJson(const std::string &configPath) -> nlohmann::json; + +void validateConfigJson(const nlohmann::json &configJson); + +void configureLogging(const std::string &loggingLevel); + +auto loadDatabaseConfig(const nlohmann::json &configJson) -> config::shared_ptr_t; + +void loadWALConfig(const nlohmann::json &walConfig, config::shared_ptr_t dbConfig); + +void loadLSMTreeConfig( + const nlohmann::json &lsmtreeConfig, + config::shared_ptr_t dbConfig, + const std::string &configPath +); + +auto loadServerConfig(const nlohmann::json &configJson, config::shared_ptr_t dbConfig); + +auto initializeDatabaseConfig(const nlohmann::json &configJson, const std::string &configPath) + -> config::shared_ptr_t; } // namespace config diff --git a/lib/db/db.h b/src/include/db/db.h similarity index 100% rename from lib/db/db.h rename to src/include/db/db.h diff --git a/lib/db/db_config.h b/src/include/db/db_config.h similarity index 100% rename from lib/db/db_config.h rename to src/include/db/db_config.h diff --git a/lib/db/manifest/manifest.h b/src/include/db/manifest/manifest.h similarity index 99% rename from lib/db/manifest/manifest.h rename to src/include/db/manifest/manifest.h index 403259c..cc430e3 100644 --- a/lib/db/manifest/manifest.h +++ b/src/include/db/manifest/manifest.h @@ -12,7 +12,7 @@ #include #include "fs/types.h" -#include "wal.h" +#include "wal/wal.h" namespace db::manifest { diff --git a/lib/fs/append_only_file.h b/src/include/fs/append_only_file.h similarity index 96% rename from lib/fs/append_only_file.h rename to src/include/fs/append_only_file.h index 6097d69..00c6a19 100644 --- a/lib/fs/append_only_file.h +++ b/src/include/fs/append_only_file.h @@ -1,13 +1,13 @@ #pragma once #include - #include #include + #include -#include "common.h" -#include "random_access_file.h" +#include "fs/common.h" +#include "fs/random_access_file.h" namespace fs { diff --git a/lib/fs/common.h b/src/include/fs/common.h similarity index 100% rename from lib/fs/common.h rename to src/include/fs/common.h diff --git a/lib/fs/random_access_file.h b/src/include/fs/random_access_file.h similarity index 100% rename from lib/fs/random_access_file.h rename to src/include/fs/random_access_file.h diff --git a/lib/fs/types.h b/src/include/fs/types.h similarity index 100% rename from lib/fs/types.h rename to src/include/fs/types.h diff --git a/lib/posix_wrapper/open_flag.h b/src/include/posix_wrapper/open_flag.h similarity index 100% rename from lib/posix_wrapper/open_flag.h rename to src/include/posix_wrapper/open_flag.h diff --git a/lib/raft/persistence.h b/src/include/raft/persistence.h similarity index 100% rename from lib/raft/persistence.h rename to src/include/raft/persistence.h diff --git a/lib/raft/raft.h b/src/include/raft/raft.h similarity index 100% rename from lib/raft/raft.h rename to src/include/raft/raft.h diff --git a/lib/server/grpc_server.h b/src/include/server/grpc_server.h similarity index 93% rename from lib/server/grpc_server.h rename to src/include/server/grpc_server.h index 71d1b2a..3c8ef3d 100644 --- a/lib/server/grpc_server.h +++ b/src/include/server/grpc_server.h @@ -1,9 +1,11 @@ #pragma once -#include "server_concept.h" -#include "grpcpp/server.h" +#include + +#include "server/server_concept.h" #include "db/db.h" -#include "server_kind.h" +#include "server/server_kind.h" + #include "tinykvpp/v1/tinykvpp_service.grpc.pb.h" #include "tinykvpp/v1/tinykvpp_service.pb.h" diff --git a/lib/server/server.h b/src/include/server/server.h similarity index 94% rename from lib/server/server.h rename to src/include/server/server.h index 312a8e0..9f802d1 100644 --- a/lib/server/server.h +++ b/src/include/server/server.h @@ -2,7 +2,7 @@ #include -#include "server_factory.h" +#include "server/server_factory.h" namespace server { diff --git a/lib/server/server_concept.h b/src/include/server/server_concept.h similarity index 100% rename from lib/server/server_concept.h rename to src/include/server/server_concept.h index 3d06359..38fe113 100644 --- a/lib/server/server_concept.h +++ b/src/include/server/server_concept.h @@ -1,9 +1,9 @@ #pragma once -#include "db/db.h" - #include +#include "db/db.h" + namespace server { diff --git a/lib/server/server_config.h b/src/include/server/server_config.h similarity index 100% rename from lib/server/server_config.h rename to src/include/server/server_config.h diff --git a/lib/server/server_factory.h b/src/include/server/server_factory.h similarity index 62% rename from lib/server/server_factory.h rename to src/include/server/server_factory.h index 544fe4e..85a9f01 100644 --- a/lib/server/server_factory.h +++ b/src/include/server/server_factory.h @@ -1,27 +1,20 @@ #pragma once -#include "server_kind.h" -#include "server_concept.h" -#include "tcp_server.h" -#include "grpc_server.h" #include +#include "server/server_kind.h" +#include "server/server_concept.h" +#include "server/grpc_server.h" + namespace server { -using namespace tcp_communication; using namespace grpc_communication; template - requires communication_strategy_t || - communication_strategy_t + requires communication_strategy_t struct CommunicationFactory; -template <> struct CommunicationFactory -{ - using type = tcp_communication_t; -}; - template <> struct CommunicationFactory { using type = grpc_communication_t; diff --git a/lib/server/server_kind.h b/src/include/server/server_kind.h similarity index 87% rename from lib/server/server_kind.h rename to src/include/server/server_kind.h index 00e72fb..e852102 100644 --- a/lib/server/server_kind.h +++ b/src/include/server/server_kind.h @@ -1,14 +1,13 @@ #pragma once #include -#include #include +#include namespace server { static constexpr const std::string_view GRPC_STR_VIEW = "grpc"; -static constexpr const std::string_view TCP_STR_VIEW = "tcp"; static constexpr const std::string_view UNDEFINED_STR_VIEW = "undefined"; enum class communication_strategy_kind_k : uint8_t diff --git a/lib/structures/hashindex/hashindex.h b/src/include/structures/hashindex/hashindex.h similarity index 100% rename from lib/structures/hashindex/hashindex.h rename to src/include/structures/hashindex/hashindex.h diff --git a/lib/structures/lsmtree/compaction/compactation.h b/src/include/structures/lsmtree/compaction/compactation.h similarity index 100% rename from lib/structures/lsmtree/compaction/compactation.h rename to src/include/structures/lsmtree/compaction/compactation.h diff --git a/lib/structures/lsmtree/compaction/compaction_trigger.h b/src/include/structures/lsmtree/compaction/compaction_trigger.h similarity index 100% rename from lib/structures/lsmtree/compaction/compaction_trigger.h rename to src/include/structures/lsmtree/compaction/compaction_trigger.h diff --git a/lib/structures/lsmtree/compaction/level_zero_compactation.h b/src/include/structures/lsmtree/compaction/level_zero_compactation.h similarity index 100% rename from lib/structures/lsmtree/compaction/level_zero_compactation.h rename to src/include/structures/lsmtree/compaction/level_zero_compactation.h diff --git a/lib/structures/lsmtree/levels/level.h b/src/include/structures/lsmtree/levels/level.h similarity index 100% rename from lib/structures/lsmtree/levels/level.h rename to src/include/structures/lsmtree/levels/level.h diff --git a/lib/structures/lsmtree/levels/levels.h b/src/include/structures/lsmtree/levels/levels.h similarity index 100% rename from lib/structures/lsmtree/levels/levels.h rename to src/include/structures/lsmtree/levels/levels.h diff --git a/lib/structures/lsmtree/lsmtree.h b/src/include/structures/lsmtree/lsmtree.h similarity index 98% rename from lib/structures/lsmtree/lsmtree.h rename to src/include/structures/lsmtree/lsmtree.h index 935ee75..27f4c55 100644 --- a/lib/structures/lsmtree/lsmtree.h +++ b/src/include/structures/lsmtree/lsmtree.h @@ -11,7 +11,7 @@ #include "wal/wal.h" #include "structures/lsmtree/levels/levels.h" #include "concurrency/thread_safe_queue.h" -#include "raft/v1/raft_service.grpc.pb.h" + #include "raft/v1/raft_service.pb.h" namespace structures::lsmtree diff --git a/lib/structures/lsmtree/lsmtree_config.h b/src/include/structures/lsmtree/lsmtree_config.h similarity index 100% rename from lib/structures/lsmtree/lsmtree_config.h rename to src/include/structures/lsmtree/lsmtree_config.h diff --git a/lib/structures/lsmtree/lsmtree_types.h b/src/include/structures/lsmtree/lsmtree_types.h similarity index 100% rename from lib/structures/lsmtree/lsmtree_types.h rename to src/include/structures/lsmtree/lsmtree_types.h diff --git a/lib/structures/lsmtree/segments/helpers.h b/src/include/structures/lsmtree/segments/helpers.h similarity index 100% rename from lib/structures/lsmtree/segments/helpers.h rename to src/include/structures/lsmtree/segments/helpers.h diff --git a/lib/structures/lsmtree/segments/lsmtree_regular_segment.h b/src/include/structures/lsmtree/segments/lsmtree_regular_segment.h similarity index 100% rename from lib/structures/lsmtree/segments/lsmtree_regular_segment.h rename to src/include/structures/lsmtree/segments/lsmtree_regular_segment.h diff --git a/lib/structures/lsmtree/segments/lsmtree_segment_factory.h b/src/include/structures/lsmtree/segments/lsmtree_segment_factory.h similarity index 72% rename from lib/structures/lsmtree/segments/lsmtree_segment_factory.h rename to src/include/structures/lsmtree/segments/lsmtree_segment_factory.h index 6adc4eb..617500d 100644 --- a/lib/structures/lsmtree/segments/lsmtree_segment_factory.h +++ b/src/include/structures/lsmtree/segments/lsmtree_segment_factory.h @@ -1,8 +1,7 @@ #pragma once -#include "structures/lsmtree/segments/lsmtree_regular_segment.h" +#include "lsmtree_regular_segment.h" #include "types.h" -#include namespace structures::lsmtree::segments::factories { diff --git a/lib/structures/lsmtree/segments/lsmtree_segment_index.h b/src/include/structures/lsmtree/segments/lsmtree_segment_index.h similarity index 86% rename from lib/structures/lsmtree/segments/lsmtree_segment_index.h rename to src/include/structures/lsmtree/segments/lsmtree_segment_index.h index c47d717..e7b6e35 100644 --- a/lib/structures/lsmtree/segments/lsmtree_segment_index.h +++ b/src/include/structures/lsmtree/segments/lsmtree_segment_index.h @@ -1,7 +1,7 @@ #pragma once +#include #include - namespace structures::lsmtree { @@ -26,8 +26,8 @@ class lsmtree_segment_index_t }; // TODO: Decide on the interface. - offset_type_t write(const key_t &key); - offset_type_t read(const key_t &key) const; + auto write(const key_t &key) -> offset_type_t; + [[nodiscard]] auto read(const key_t &key) const; private: offset_type_t m_lastOffset{0}; diff --git a/lib/structures/lsmtree/segments/segment_config.h b/src/include/structures/lsmtree/segments/segment_config.h similarity index 100% rename from lib/structures/lsmtree/segments/segment_config.h rename to src/include/structures/lsmtree/segments/segment_config.h diff --git a/lib/structures/lsmtree/segments/segment_storage.h b/src/include/structures/lsmtree/segments/segment_storage.h similarity index 100% rename from lib/structures/lsmtree/segments/segment_storage.h rename to src/include/structures/lsmtree/segments/segment_storage.h diff --git a/lib/structures/lsmtree/segments/types.h b/src/include/structures/lsmtree/segments/types.h similarity index 100% rename from lib/structures/lsmtree/segments/types.h rename to src/include/structures/lsmtree/segments/types.h diff --git a/lib/structures/lsmtree/segments/uuid.h b/src/include/structures/lsmtree/segments/uuid.h similarity index 100% rename from lib/structures/lsmtree/segments/uuid.h rename to src/include/structures/lsmtree/segments/uuid.h diff --git a/lib/structures/memtable/memtable.h b/src/include/structures/memtable/memtable.h similarity index 100% rename from lib/structures/memtable/memtable.h rename to src/include/structures/memtable/memtable.h diff --git a/lib/structures/memtable/memtable_reader_writer.h b/src/include/structures/memtable/memtable_reader_writer.h similarity index 100% rename from lib/structures/memtable/memtable_reader_writer.h rename to src/include/structures/memtable/memtable_reader_writer.h diff --git a/lib/structures/skiplist/skiplist.h b/src/include/structures/skiplist/skiplist.h similarity index 100% rename from lib/structures/skiplist/skiplist.h rename to src/include/structures/skiplist/skiplist.h diff --git a/lib/structures/sorted_vector/sorted_vector.h b/src/include/structures/sorted_vector/sorted_vector.h similarity index 100% rename from lib/structures/sorted_vector/sorted_vector.h rename to src/include/structures/sorted_vector/sorted_vector.h diff --git a/lib/wal/backend/append_only_file_storage_backend.h b/src/include/wal/backend/append_only_file_storage_backend.h similarity index 98% rename from lib/wal/backend/append_only_file_storage_backend.h rename to src/include/wal/backend/append_only_file_storage_backend.h index 79b2807..2bdf9f1 100644 --- a/lib/wal/backend/append_only_file_storage_backend.h +++ b/src/include/wal/backend/append_only_file_storage_backend.h @@ -5,8 +5,8 @@ #include #include -#include "backend.h" -#include "../concepts.h" +#include "wal/backend/backend.h" +#include "wal/concepts.h" #include "fs/append_only_file.h" namespace wal::backend diff --git a/lib/wal/backend/backend.h b/src/include/wal/backend/backend.h similarity index 99% rename from lib/wal/backend/backend.h rename to src/include/wal/backend/backend.h index a772267..78a885a 100644 --- a/lib/wal/backend/backend.h +++ b/src/include/wal/backend/backend.h @@ -2,7 +2,7 @@ #include -#include "../concepts.h" +#include "wal/concepts.h" #include "fs/types.h" namespace wal::backend diff --git a/lib/wal/common.h b/src/include/wal/common.h similarity index 100% rename from lib/wal/common.h rename to src/include/wal/common.h diff --git a/lib/wal/concepts.h b/src/include/wal/concepts.h similarity index 100% rename from lib/wal/concepts.h rename to src/include/wal/concepts.h diff --git a/lib/wal/config.h b/src/include/wal/config.h similarity index 100% rename from lib/wal/config.h rename to src/include/wal/config.h diff --git a/lib/wal/in_memory_log_storage.h b/src/include/wal/in_memory_log_storage.h similarity index 100% rename from lib/wal/in_memory_log_storage.h rename to src/include/wal/in_memory_log_storage.h diff --git a/lib/wal/persistent_log_storage.h b/src/include/wal/persistent_log_storage.h similarity index 98% rename from lib/wal/persistent_log_storage.h rename to src/include/wal/persistent_log_storage.h index c08727c..f81d3da 100644 --- a/lib/wal/persistent_log_storage.h +++ b/src/include/wal/persistent_log_storage.h @@ -6,9 +6,9 @@ #include #include -#include "concepts.h" -#include "backend/backend.h" -#include "backend/append_only_file_storage_backend.h" +#include "wal/concepts.h" +#include "wal/backend/backend.h" +#include "wal/backend/append_only_file_storage_backend.h" namespace wal { diff --git a/lib/wal/wal.h b/src/include/wal/wal.h similarity index 97% rename from lib/wal/wal.h rename to src/include/wal/wal.h index f9a2c0b..33a4fa9 100644 --- a/lib/wal/wal.h +++ b/src/include/wal/wal.h @@ -7,9 +7,9 @@ #include -#include "common.h" -#include "in_memory_log_storage.h" -#include "persistent_log_storage.h" +#include "wal/common.h" +#include "wal/in_memory_log_storage.h" +#include "wal/persistent_log_storage.h" namespace wal { diff --git a/src/main.cpp b/src/main.cpp index bcc0ec7..86e1756 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -12,15 +12,13 @@ #include #include -#include "config.h" -#include "db.h" -#include "db_config.h" -#include "lsmtree.h" -#include "manifest/manifest.h" -#include "wal/common.h" -#include "memtable.h" +#include "db/db.h" #include "raft/raft.h" +#include "structures/memtable/memtable.h" +#include "structures/lsmtree/lsmtree.h" +#include "wal/common.h" #include "server/grpc_server.h" +#include "config/config.h" using tk_key_t = structures::memtable::memtable_t::record_t::key_t; using tk_value_t = structures::memtable::memtable_t::record_t::value_t; @@ -166,14 +164,14 @@ auto main(int argc, char *argv[]) -> int // Load the config const auto &configPath = parsedOptions["config"].as(); - const auto &configJson = loadConfigJson(configPath); - validateConfigJson(configJson); + const auto &configJson = config::loadConfigJson(configPath); + config::validateConfigJson(configJson); // Setup the logging - configureLogging(configJson["logging"]["loggingLevel"].get()); + config::configureLogging(configJson["logging"]["loggingLevel"].get()); // Setup database config - auto pDbConfig = initializeDatabaseConfig(configJson, configPath); + auto pDbConfig = config::initializeDatabaseConfig(configJson, configPath); if (pDbConfig->WALConfig.storageType == wal::log_storage_type_k::undefined_k) { spdlog::error("Undefined WAL storage type"); @@ -259,7 +257,7 @@ auto main(int argc, char *argv[]) -> int }; // Start consensus module and gRPC server - auto serverThread = std::jthread([&pServer] { pServer->Wait(); }); + auto serverThread = std::jthread([&pServer] -> void { pServer->Wait(); }); // Start consensus module if (pConsensusModule) diff --git a/lib/posix_wrapper/CMakeLists.txt b/src/posix_wrapper/CMakeLists.txt similarity index 67% rename from lib/posix_wrapper/CMakeLists.txt rename to src/posix_wrapper/CMakeLists.txt index 0ffcefe..febc2fa 100644 --- a/lib/posix_wrapper/CMakeLists.txt +++ b/src/posix_wrapper/CMakeLists.txt @@ -2,11 +2,13 @@ cmake_minimum_required(VERSION 3.25) project(frankie) add_library(POSIXWrapper open_flag.cpp) - set_target_properties(POSIXWrapper PROPERTIES CXX_STANDARD 23) -target_include_directories(POSIXWrapper INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories( + POSIXWrapper + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include +) target_link_libraries(POSIXWrapper PUBLIC uring spdlog::spdlog) add_executable(POSIXWrapperTest posix_wrapper_test.cpp) set_target_properties(POSIXWrapperTest PROPERTIES CXX_STANDARD 23) -target_link_libraries(POSIXWrapperTest gtest::gtest) +target_link_libraries(POSIXWrapperTest gtest::gtest POSIXWrapper) diff --git a/src/posix_wrapper/open_flag.cpp b/src/posix_wrapper/open_flag.cpp new file mode 100644 index 0000000..b7cfe5f --- /dev/null +++ b/src/posix_wrapper/open_flag.cpp @@ -0,0 +1 @@ +#include "posix_wrapper/open_flag.h" diff --git a/lib/posix_wrapper/posix_wrapper_test.cpp b/src/posix_wrapper/posix_wrapper_test.cpp similarity index 96% rename from lib/posix_wrapper/posix_wrapper_test.cpp rename to src/posix_wrapper/posix_wrapper_test.cpp index 4ab053b..d38efad 100644 --- a/lib/posix_wrapper/posix_wrapper_test.cpp +++ b/src/posix_wrapper/posix_wrapper_test.cpp @@ -1,6 +1,6 @@ #include -#include "open_flag.h" +#include "posix_wrapper/open_flag.h" namespace pw = posix_wrapper; diff --git a/src/raft/CMakeLists.txt b/src/raft/CMakeLists.txt new file mode 100644 index 0000000..0abac3a --- /dev/null +++ b/src/raft/CMakeLists.txt @@ -0,0 +1,31 @@ +cmake_minimum_required(VERSION 3.25) +project(frankie) + +add_library(Raft raft.cpp) +set_target_properties(Raft PROPERTIES CXX_STANDARD 23) +target_include_directories(Raft PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include) +target_link_libraries( + Raft + RaftProtoObjects + Config + FS + WAL + libassert::assert + spdlog::spdlog + fmt::fmt + magic_enum::magic_enum +) + +add_executable(RaftTest raft_test.cpp) +set_target_properties(RaftTest PROPERTIES CXX_STANDARD 23) +target_link_libraries( + RaftTest + gtest::gtest + Raft + RaftProtoObjects + TKVProtoObjects + Concurrency +) + +include(GoogleTest) +gtest_discover_tests(RaftTest) diff --git a/lib/raft/persistence.cpp b/src/raft/persistence.cpp similarity index 56% rename from lib/raft/persistence.cpp rename to src/raft/persistence.cpp index 7826bf5..4cd53ca 100644 --- a/lib/raft/persistence.cpp +++ b/src/raft/persistence.cpp @@ -1,4 +1,4 @@ -#include "persistence.h" +#include "raft/persistence.h" namespace raft { diff --git a/lib/raft/raft.cpp b/src/raft/raft.cpp similarity index 99% rename from lib/raft/raft.cpp rename to src/raft/raft.cpp index 21d29eb..ea3696d 100644 --- a/lib/raft/raft.cpp +++ b/src/raft/raft.cpp @@ -4,9 +4,7 @@ #include #include #include -#include #include -#include #include #include @@ -15,9 +13,9 @@ #include #include #include +#include #include "raft/raft.h" -#include "concurrency/thread_pool.h" #include "config/config.h" #include "wal/wal.h" diff --git a/lib/raft/raft_test.cpp b/src/raft/raft_test.cpp similarity index 99% rename from lib/raft/raft_test.cpp rename to src/raft/raft_test.cpp index a85f74a..92f92cf 100644 --- a/lib/raft/raft_test.cpp +++ b/src/raft/raft_test.cpp @@ -9,7 +9,7 @@ #include #include "config/config.h" -#include "raft.h" +#include "raft/raft.h" #include "wal/wal.h" #include "raft/v1/raft_service_mock.grpc.pb.h" // Generated mock stubs diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt new file mode 100644 index 0000000..02bbdc5 --- /dev/null +++ b/src/server/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.25) +project(frankie) + +add_library(Server server_kind.cpp grpc_server.cpp) +target_include_directories(Server PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include) +target_link_libraries(Server DB Config fmt::fmt) diff --git a/lib/server/grpc_server.cpp b/src/server/grpc_server.cpp similarity index 99% rename from lib/server/grpc_server.cpp rename to src/server/grpc_server.cpp index 5fbd1e3..317fd86 100644 --- a/lib/server/grpc_server.cpp +++ b/src/server/grpc_server.cpp @@ -1,5 +1,5 @@ #include "server/grpc_server.h" -#include "db.h" +#include "db/db.h" #include #include diff --git a/lib/server/server_kind.cpp b/src/server/server_kind.cpp similarity index 77% rename from lib/server/server_kind.cpp rename to src/server/server_kind.cpp index 63a9d35..b9f5528 100644 --- a/lib/server/server_kind.cpp +++ b/src/server/server_kind.cpp @@ -10,8 +10,6 @@ auto to_string(const communication_strategy_kind_k kind) noexcept -> std::option { case communication_strategy_kind_k::grpc_k: return GRPC_STR_VIEW; - case communication_strategy_kind_k::tcp_k: - return TCP_STR_VIEW; default: return std::nullopt; } @@ -25,11 +23,6 @@ auto from_string(const std::string_view kind) noexcept return communication_strategy_kind_k::grpc_k; } - if (kind == TCP_STR_VIEW) - { - return communication_strategy_kind_k::tcp_k; - } - return std::nullopt; } diff --git a/lib/structures/CMakeLists.txt b/src/structures/CMakeLists.txt similarity index 100% rename from lib/structures/CMakeLists.txt rename to src/structures/CMakeLists.txt diff --git a/lib/structures/hashindex/CMakeLists.txt b/src/structures/hashindex/CMakeLists.txt similarity index 63% rename from lib/structures/hashindex/CMakeLists.txt rename to src/structures/hashindex/CMakeLists.txt index 830df9d..9aeaa82 100644 --- a/lib/structures/hashindex/CMakeLists.txt +++ b/src/structures/hashindex/CMakeLists.txt @@ -1,14 +1,14 @@ cmake_minimum_required(VERSION 3.25) project(frankie) -add_library( - HashIndex - "hashindex.cpp") - +add_library(HashIndex hashindex.cpp) set_target_properties(HashIndex PROPERTIES CXX_STANDARD 23) -target_include_directories(HashIndex INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories( + HashIndex + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../include +) -# add_executable(HashIndexTest "hashindex_test.cpp") +# add_executable(HashIndexTest hashindex_test.cpp) # set_target_properties(HashIndexTest PROPERTIES CXX_STANDARD 23) # target_link_libraries( # HashIndexTest diff --git a/lib/structures/hashindex/hashindex.cpp b/src/structures/hashindex/hashindex.cpp similarity index 100% rename from lib/structures/hashindex/hashindex.cpp rename to src/structures/hashindex/hashindex.cpp diff --git a/src/structures/lsmtree/CMakeLists.txt b/src/structures/lsmtree/CMakeLists.txt new file mode 100644 index 0000000..b00616f --- /dev/null +++ b/src/structures/lsmtree/CMakeLists.txt @@ -0,0 +1,53 @@ +cmake_minimum_required(VERSION 3.25) + +add_library( + LSMTree + lsmtree.cpp + segments/lsmtree_regular_segment.cpp + segments/lsmtree_segment_factory.cpp + segments/segment_storage.cpp + segments/helpers.cpp + levels/level.cpp + levels/levels.cpp +) + +set_target_properties(LSMTree PROPERTIES CXX_STANDARD 23) +target_include_directories( + LSMTree + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../include +) +target_link_libraries( + LSMTree + PUBLIC + spdlog::spdlog + abseil::abseil + libassert::assert + magic_enum::magic_enum + Concurrency + WAL + Raft + RaftProtoObjects +) + +add_executable(LSMTreeTest lsmtree_test.cpp) +set_target_properties(LSMTreeTest PROPERTIES CXX_STANDARD 23) +target_link_libraries( + LSMTreeTest + PRIVATE + Catch2::Catch2WithMain + spdlog::spdlog + fmt::fmt + LSMTree + MemTable + Config + HashIndex + WAL + Raft + FS + DB + Concurrency +) + +# Register Catch2 tests with CTest +include(Catch) +catch_discover_tests(LSMTreeTest) diff --git a/lib/structures/lsmtree/compaction/level_zero_compactation.cpp b/src/structures/lsmtree/compaction/level_zero_compactation.cpp similarity index 100% rename from lib/structures/lsmtree/compaction/level_zero_compactation.cpp rename to src/structures/lsmtree/compaction/level_zero_compactation.cpp diff --git a/lib/structures/lsmtree/levels/level.cpp b/src/structures/lsmtree/levels/level.cpp similarity index 99% rename from lib/structures/lsmtree/levels/level.cpp rename to src/structures/lsmtree/levels/level.cpp index cc2a4ef..5907f43 100644 --- a/lib/structures/lsmtree/levels/level.cpp +++ b/src/structures/lsmtree/levels/level.cpp @@ -11,7 +11,7 @@ #include #include -#include "level.h" +#include "structures/lsmtree/levels/level.h" #include "structures/lsmtree/lsmtree_types.h" #include "structures/lsmtree/segments/lsmtree_segment_factory.h" #include "structures/lsmtree/segments/segment_storage.h" @@ -181,7 +181,7 @@ auto level_t::compact() const noexcept -> segments::regular_segment::shared_ptr_ // The postfix "_compacted" signals that the segment is an intermediate result auto name{fmt::format("{}_{}_compacted", helpers::segment_name(), index())}; return segments::factories::lsmtree_segment_factory( - name, helpers::segment_path(m_pConfig->datadir_path(), name), mergedMemtable + name, helpers::segment_path(m_pConfig->datadir_path(), name), std::move(mergedMemtable) ); } diff --git a/lib/structures/lsmtree/levels/levels.cpp b/src/structures/lsmtree/levels/levels.cpp similarity index 97% rename from lib/structures/lsmtree/levels/levels.cpp rename to src/structures/lsmtree/levels/levels.cpp index 9aefbe8..487babb 100644 --- a/lib/structures/lsmtree/levels/levels.cpp +++ b/src/structures/lsmtree/levels/levels.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -7,8 +6,8 @@ #include #include -#include "levels.h" -#include "../segments/helpers.h" +#include "structures/lsmtree/segments/helpers.h" +#include "structures/lsmtree/levels/levels.h" #include "db/manifest/manifest.h" #include "concurrency/helpers.h" @@ -23,7 +22,7 @@ levels_t::levels_t(config::shared_ptr_t pConfig, db::manifest::shared_ptr_t pMan m_pManifest{std::move(std::move(pManifest))}, m_compaction_thread([this](std::stop_token stoken) { compaction_task(stoken); }) { - // TODO: Make number of levels configurable + // TODO(lnikon): Make number of levels configurable // const std::size_t levelCount{m_pConfig->LSMTreeConfig.LevelCount}; const std::size_t levelCount{7}; for (std::size_t idx{0}; idx < levelCount; idx++) @@ -240,7 +239,7 @@ void levels_t::compaction_task(std::stop_token stoken) noexcept { spdlog::error("Compaction failed."); } - ASSERT(ok); + // ASSERT(ok); } } diff --git a/lib/structures/lsmtree/lsmtree.cpp b/src/structures/lsmtree/lsmtree.cpp similarity index 100% rename from lib/structures/lsmtree/lsmtree.cpp rename to src/structures/lsmtree/lsmtree.cpp diff --git a/lib/structures/lsmtree/lsmtree_test.cpp b/src/structures/lsmtree/lsmtree_test.cpp similarity index 100% rename from lib/structures/lsmtree/lsmtree_test.cpp rename to src/structures/lsmtree/lsmtree_test.cpp diff --git a/lib/structures/lsmtree/segments/gsl/gsl b/src/structures/lsmtree/segments/gsl/gsl similarity index 100% rename from lib/structures/lsmtree/segments/gsl/gsl rename to src/structures/lsmtree/segments/gsl/gsl diff --git a/lib/structures/lsmtree/segments/gsl/gsl_algorithm b/src/structures/lsmtree/segments/gsl/gsl_algorithm similarity index 100% rename from lib/structures/lsmtree/segments/gsl/gsl_algorithm rename to src/structures/lsmtree/segments/gsl/gsl_algorithm diff --git a/lib/structures/lsmtree/segments/gsl/gsl_assert b/src/structures/lsmtree/segments/gsl/gsl_assert similarity index 100% rename from lib/structures/lsmtree/segments/gsl/gsl_assert rename to src/structures/lsmtree/segments/gsl/gsl_assert diff --git a/lib/structures/lsmtree/segments/gsl/gsl_byte b/src/structures/lsmtree/segments/gsl/gsl_byte similarity index 100% rename from lib/structures/lsmtree/segments/gsl/gsl_byte rename to src/structures/lsmtree/segments/gsl/gsl_byte diff --git a/lib/structures/lsmtree/segments/gsl/gsl_util b/src/structures/lsmtree/segments/gsl/gsl_util similarity index 100% rename from lib/structures/lsmtree/segments/gsl/gsl_util rename to src/structures/lsmtree/segments/gsl/gsl_util diff --git a/lib/structures/lsmtree/segments/gsl/multi_span b/src/structures/lsmtree/segments/gsl/multi_span similarity index 100% rename from lib/structures/lsmtree/segments/gsl/multi_span rename to src/structures/lsmtree/segments/gsl/multi_span diff --git a/lib/structures/lsmtree/segments/gsl/pointers b/src/structures/lsmtree/segments/gsl/pointers similarity index 100% rename from lib/structures/lsmtree/segments/gsl/pointers rename to src/structures/lsmtree/segments/gsl/pointers diff --git a/lib/structures/lsmtree/segments/gsl/span b/src/structures/lsmtree/segments/gsl/span similarity index 100% rename from lib/structures/lsmtree/segments/gsl/span rename to src/structures/lsmtree/segments/gsl/span diff --git a/lib/structures/lsmtree/segments/gsl/string_span b/src/structures/lsmtree/segments/gsl/string_span similarity index 100% rename from lib/structures/lsmtree/segments/gsl/string_span rename to src/structures/lsmtree/segments/gsl/string_span diff --git a/lib/structures/lsmtree/segments/helpers.cpp b/src/structures/lsmtree/segments/helpers.cpp similarity index 89% rename from lib/structures/lsmtree/segments/helpers.cpp rename to src/structures/lsmtree/segments/helpers.cpp index 74586c3..6f0a03c 100644 --- a/lib/structures/lsmtree/segments/helpers.cpp +++ b/src/structures/lsmtree/segments/helpers.cpp @@ -1,17 +1,13 @@ -// -// Created by nikon on 3/8/24. -// - #include #include -#include - #include #include -#include "uuid.h" #include +#include "structures/lsmtree/segments/helpers.h" +#include "structures/lsmtree/segments/uuid.h" + namespace structures::lsmtree::segments::helpers { diff --git a/lib/structures/lsmtree/segments/lsmtree_regular_segment.cpp b/src/structures/lsmtree/segments/lsmtree_regular_segment.cpp similarity index 99% rename from lib/structures/lsmtree/segments/lsmtree_regular_segment.cpp rename to src/structures/lsmtree/segments/lsmtree_regular_segment.cpp index 976cca3..4cb28d5 100644 --- a/lib/structures/lsmtree/segments/lsmtree_regular_segment.cpp +++ b/src/structures/lsmtree/segments/lsmtree_regular_segment.cpp @@ -14,6 +14,7 @@ namespace structures::lsmtree::segments::regular_segment { +// TODO(lnikon): This is horrible. const auto footerSize{128}; // bytes regular_segment_t::regular_segment_t( diff --git a/lib/structures/lsmtree/segments/lsmtree_segment_factory.cpp b/src/structures/lsmtree/segments/lsmtree_segment_factory.cpp similarity index 63% rename from lib/structures/lsmtree/segments/lsmtree_segment_factory.cpp rename to src/structures/lsmtree/segments/lsmtree_segment_factory.cpp index 98e56f1..9b3a094 100644 --- a/lib/structures/lsmtree/segments/lsmtree_segment_factory.cpp +++ b/src/structures/lsmtree/segments/lsmtree_segment_factory.cpp @@ -1,10 +1,4 @@ -// -// Created by nikon on 2/6/22. -// - -#include -#include -#include +#include "structures/lsmtree/segments/lsmtree_segment_factory.h" namespace structures::lsmtree::segments::factories { diff --git a/lib/structures/lsmtree/segments/segment_storage.cpp b/src/structures/lsmtree/segments/segment_storage.cpp similarity index 89% rename from lib/structures/lsmtree/segments/segment_storage.cpp rename to src/structures/lsmtree/segments/segment_storage.cpp index c984b05..0558351 100644 --- a/lib/structures/lsmtree/segments/segment_storage.cpp +++ b/src/structures/lsmtree/segments/segment_storage.cpp @@ -1,8 +1,9 @@ -#include "structures/lsmtree/segments/lsmtree_regular_segment.h" +#include + #include -#include -#include +#include "structures/lsmtree/segments/lsmtree_regular_segment.h" +#include "structures/lsmtree/segments/segment_storage.h" namespace structures::lsmtree::segments::storage { @@ -81,15 +82,18 @@ void segment_storage_t::remove(regular_segment::shared_ptr_t pSegment) const auto oldSize = m_segmentsVector.size(); m_segmentsVector.erase( std::remove(std::begin(m_segmentsVector), std::end(m_segmentsVector), pSegment), - std::end(m_segmentsVector)); + std::end(m_segmentsVector) + ); m_segmentsMap.erase(pSegment->get_name()); const auto newSize = m_segmentsVector.size(); - spdlog::debug("({}): Removed {}, old size {}, new size {}", - "segment_storage_t::remove", - pSegment->get_name(), - oldSize, - newSize); + spdlog::debug( + "({}): Removed {}, old size {}, new size {}", + "segment_storage_t::remove", + pSegment->get_name(), + oldSize, + newSize + ); } auto segment_storage_t::find(const std::string &name) const noexcept diff --git a/lib/structures/memtable/CMakeLists.txt b/src/structures/memtable/CMakeLists.txt similarity index 50% rename from lib/structures/memtable/CMakeLists.txt rename to src/structures/memtable/CMakeLists.txt index 1f7d9dc..29ef31b 100644 --- a/lib/structures/memtable/CMakeLists.txt +++ b/src/structures/memtable/CMakeLists.txt @@ -1,14 +1,20 @@ cmake_minimum_required(VERSION 3.25) project(frankie) -add_library(MemTable "memtable.cpp") +add_library(MemTable memtable.cpp) target_compile_features(MemTable PUBLIC cxx_std_23) +target_include_directories( + MemTable + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../include +) target_link_libraries(MemTable PRIVATE fmt::fmt) -target_include_directories(MemTable INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) -add_executable(MemTableTest "memtable_test.cpp") +add_executable(MemTableTest memtable_test.cpp) target_compile_features(MemTableTest PUBLIC cxx_std_23) -target_link_libraries(MemTableTest PRIVATE Catch2::Catch2WithMain spdlog::spdlog fmt::fmt LSMTree MemTable) +target_link_libraries( + MemTableTest + PRIVATE Catch2::Catch2WithMain spdlog::spdlog fmt::fmt LSMTree MemTable +) # Register Catch2 tests with CTest include(Catch) diff --git a/lib/structures/memtable/memtable.cpp b/src/structures/memtable/memtable.cpp similarity index 99% rename from lib/structures/memtable/memtable.cpp rename to src/structures/memtable/memtable.cpp index 6276f38..2df5f7c 100644 --- a/lib/structures/memtable/memtable.cpp +++ b/src/structures/memtable/memtable.cpp @@ -1,4 +1,4 @@ -#include "memtable.h" +#include "structures/memtable/memtable.h" #include #include diff --git a/lib/structures/memtable/memtable_test.cpp b/src/structures/memtable/memtable_test.cpp similarity index 100% rename from lib/structures/memtable/memtable_test.cpp rename to src/structures/memtable/memtable_test.cpp diff --git a/src/structures/skiplist/CMakeLists.txt b/src/structures/skiplist/CMakeLists.txt new file mode 100644 index 0000000..b6e3870 --- /dev/null +++ b/src/structures/skiplist/CMakeLists.txt @@ -0,0 +1,27 @@ +cmake_minimum_required(VERSION 3.25) +project(frankie) + +add_library(SkipList skiplist.cpp) + +set_target_properties(SkipList PROPERTIES CXX_STANDARD 23) +target_include_directories( + SkipList + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../include +) + +add_executable(SkipListTest skiplist_test.cpp) +set_target_properties(SkipListTest PROPERTIES CXX_STANDARD 23) +target_link_libraries( + SkipListTest + Catch2::Catch2WithMain + spdlog::spdlog + fmt::fmt + LSMTree + SkipList + MemTable + Config +) + +# Register Catch2 tests with CTest +include(Catch) +catch_discover_tests(SkipListTest) diff --git a/src/structures/skiplist/skiplist.cpp b/src/structures/skiplist/skiplist.cpp new file mode 100644 index 0000000..9e42d25 --- /dev/null +++ b/src/structures/skiplist/skiplist.cpp @@ -0,0 +1 @@ +#include "structures/skiplist/skiplist.h" diff --git a/lib/structures/skiplist/skiplist_test.cpp b/src/structures/skiplist/skiplist_test.cpp similarity index 100% rename from lib/structures/skiplist/skiplist_test.cpp rename to src/structures/skiplist/skiplist_test.cpp diff --git a/src/structures/sorted_vector/CMakeLists.txt b/src/structures/sorted_vector/CMakeLists.txt new file mode 100644 index 0000000..37efb94 --- /dev/null +++ b/src/structures/sorted_vector/CMakeLists.txt @@ -0,0 +1,27 @@ +cmake_minimum_required(VERSION 3.25) +project(frankie) + +add_library(SortedVector sorted_vector.cpp) +target_compile_features(SortedVector PUBLIC cxx_std_23) +target_include_directories( + SortedVector + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../include +) + +add_executable(SortedVectorTest sorted_vector_test.cpp) +set_target_properties(SortedVectorTest PROPERTIES CXX_STANDARD 23) +target_link_libraries( + SortedVectorTest + Catch2::Catch2WithMain + spdlog::spdlog + fmt::fmt + LSMTree + SkipList + MemTable + Config + SortedVector +) + +# Register Catch2 tests with CTest +include(Catch) +catch_discover_tests(SortedVectorTest) diff --git a/src/structures/sorted_vector/sorted_vector.cpp b/src/structures/sorted_vector/sorted_vector.cpp new file mode 100644 index 0000000..20f001b --- /dev/null +++ b/src/structures/sorted_vector/sorted_vector.cpp @@ -0,0 +1 @@ +#include "structures/sorted_vector/sorted_vector.h" diff --git a/lib/structures/sorted_vector/sorted_vector_test.cpp b/src/structures/sorted_vector/sorted_vector_test.cpp similarity index 100% rename from lib/structures/sorted_vector/sorted_vector_test.cpp rename to src/structures/sorted_vector/sorted_vector_test.cpp diff --git a/lib/wal/CMakeLists.txt b/src/wal/CMakeLists.txt similarity index 76% rename from lib/wal/CMakeLists.txt rename to src/wal/CMakeLists.txt index a5ad25c..5728a03 100644 --- a/lib/wal/CMakeLists.txt +++ b/src/wal/CMakeLists.txt @@ -4,8 +4,7 @@ project(frankie) add_library(WAL wal.cpp common.cpp) set_target_properties(WAL PROPERTIES CXX_STANDARD 23) -target_include_directories(WAL INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_SOURCE_DIR}/lib) +target_include_directories(WAL PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include) target_link_libraries(WAL PUBLIC abseil::abseil spdlog::spdlog FS RaftProtoObjects) diff --git a/lib/wal/common.cpp b/src/wal/common.cpp similarity index 96% rename from lib/wal/common.cpp rename to src/wal/common.cpp index b5dbb37..38a93dd 100644 --- a/lib/wal/common.cpp +++ b/src/wal/common.cpp @@ -1,4 +1,4 @@ -#include "common.h" +#include "wal/common.h" namespace wal { diff --git a/lib/wal/wal.cpp b/src/wal/wal.cpp similarity index 88% rename from lib/wal/wal.cpp rename to src/wal/wal.cpp index 35d68c2..120debd 100644 --- a/lib/wal/wal.cpp +++ b/src/wal/wal.cpp @@ -1,4 +1,4 @@ -#include "wal.h" +#include "wal/wal.h" namespace wal { diff --git a/lib/wal/wal_test.cpp b/src/wal/wal_test.cpp similarity index 100% rename from lib/wal/wal_test.cpp rename to src/wal/wal_test.cpp From 7b70eb4838a5174985337b0fd3e47ea812e607b1 Mon Sep 17 00:00:00 2001 From: lnikon Date: Mon, 27 Oct 2025 21:18:46 +0400 Subject: [PATCH 14/17] Testing: - Fix MemTableTest --- .gitignore | 3 ++ src/structures/memtable/CMakeLists.txt | 11 ++++--- src/structures/memtable/memtable_test.cpp | 39 ++++++++++------------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 08fbf23..4d25f84 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,6 @@ current # Compiled Protobufs *.pb.* + +# node +node_modules diff --git a/src/structures/memtable/CMakeLists.txt b/src/structures/memtable/CMakeLists.txt index 29ef31b..88892d3 100644 --- a/src/structures/memtable/CMakeLists.txt +++ b/src/structures/memtable/CMakeLists.txt @@ -13,9 +13,12 @@ add_executable(MemTableTest memtable_test.cpp) target_compile_features(MemTableTest PUBLIC cxx_std_23) target_link_libraries( MemTableTest - PRIVATE Catch2::Catch2WithMain spdlog::spdlog fmt::fmt LSMTree MemTable + gtest::gtest + spdlog::spdlog + fmt::fmt + LSMTree + MemTable ) -# Register Catch2 tests with CTest -include(Catch) -catch_discover_tests(MemTableTest) +include(GoogleTest) +gtest_discover_tests(MemTableTest) diff --git a/src/structures/memtable/memtable_test.cpp b/src/structures/memtable/memtable_test.cpp index c06b3c9..6f8ad6d 100644 --- a/src/structures/memtable/memtable_test.cpp +++ b/src/structures/memtable/memtable_test.cpp @@ -1,11 +1,6 @@ -// -// Created by nikon on 1/22/22. -// +#include -#include -#include - -#include "memtable.h" +#include "structures/memtable/memtable.h" using namespace structures; @@ -14,7 +9,7 @@ using record_t = memtable::memtable_t::record_t; using record_key_t = structures::memtable::memtable_t::record_t::key_t; using record_value_t = structures::memtable::memtable_t::record_t::value_t; -TEST_CASE("Emplace and Find", "[MemTable]") +TEST(MemTableTest, EmplaceAndFind) { memtable_t mt; mt.emplace(record_t{record_key_t{"B"}, record_value_t{"123"}}); @@ -23,14 +18,14 @@ TEST_CASE("Emplace and Find", "[MemTable]") mt.emplace(record_t{record_key_t{"C"}, record_value_t{"Hello"}}); auto record = mt.find(record_key_t{"C"}); - REQUIRE(record->m_key == record_key_t{"C"}); - REQUIRE(record->m_value == record_value_t{"Hello"}); + EXPECT_EQ(record->m_key, record_key_t{"C"}); + EXPECT_EQ(record->m_value, record_value_t{"Hello"}); record = mt.find(record_key_t{"V"}); - REQUIRE(record == std::nullopt); + EXPECT_EQ(record, std::nullopt); } -TEST_CASE("Check record size before and after insertion", "[MemTable]") +TEST(MemTableTest, CheckRecordSize) { { memtable_t mt; @@ -40,11 +35,11 @@ TEST_CASE("Check record size before and after insertion", "[MemTable]") mt.emplace(record_t{k, v}); auto record = mt.find(record_key_t{"B"}); - REQUIRE(record != std::nullopt); + EXPECT_NE(record, std::nullopt); size_t actualSize = record->size(); size_t expectedSize = k.size() + v.size(); - REQUIRE(actualSize == expectedSize); + EXPECT_EQ(actualSize, expectedSize); } { @@ -55,11 +50,11 @@ TEST_CASE("Check record size before and after insertion", "[MemTable]") mt.emplace(record_t{k, v}); auto record = mt.find(record_key_t{"B"}); - REQUIRE(record != std::nullopt); + EXPECT_NE(record, std::nullopt); size_t actualSize = record->size(); size_t expectedSize = k.size() + v.size(); - REQUIRE(actualSize == expectedSize); + EXPECT_EQ(actualSize, expectedSize); } { @@ -71,16 +66,16 @@ TEST_CASE("Check record size before and after insertion", "[MemTable]") mt.emplace(record); auto recordOpt = mt.find(k); - REQUIRE(recordOpt != std::nullopt); + EXPECT_NE(recordOpt, std::nullopt); record = *recordOpt; size_t actualSize = record.size(); size_t expectedSize = k.size() + v.size(); - REQUIRE(actualSize == expectedSize); + EXPECT_EQ(actualSize, expectedSize); } } -TEST_CASE("Check size", "[MemTable]") +TEST(MemTableTest, CheckSize) { memtable_t mt; auto k1 = record_key_t{"B"}, k2 = record_key_t{"A"}, k3 = record_key_t{"Z"}; @@ -90,10 +85,10 @@ TEST_CASE("Check size", "[MemTable]") mt.emplace(record_t{k3, v3}); // TODO: Not sure if this a good/correct way to check MemTable::Size() :) - REQUIRE(mt.size() == k1.size() + v1.size() + k2.size() + v2.size() + k3.size() + v3.size()); + EXPECT_EQ(mt.size(), k1.size() + v1.size() + k2.size() + v2.size() + k3.size() + v3.size()); } -TEST_CASE("Check count", "[MemTable]") +TEST(MemTableTest, CheckCount) { memtable_t mt; mt.emplace(record_t{record_key_t{"B"}, record_value_t{"123"}}); @@ -101,5 +96,5 @@ TEST_CASE("Check count", "[MemTable]") mt.emplace(record_t{record_key_t{"Z"}, record_value_t{"z`34.44"}}); mt.emplace(record_t{record_key_t{"C"}, record_value_t{"Hello"}}); - REQUIRE(mt.count() == 4); + EXPECT_EQ(mt.count(), 4); } From fd18bf7359b5bc81c80ca5e816ede738474e04ad Mon Sep 17 00:00:00 2001 From: lnikon Date: Sun, 2 Nov 2025 21:24:09 +0400 Subject: [PATCH 15/17] Testing: - Upgrade project structure - Fix vanishing records bug in lsmtree - Fix lsmtree tests --- CMakeLists.txt | 2 + conanfile.txt | 2 +- src/db/db.cpp | 3 +- src/db/manifest/manifest.cpp | 16 +- src/include/common/helpers.cpp | 28 + src/include/common/helpers.h | 21 + src/include/common/uuid.h | 967 +++++++ src/include/concurrency/thread_safe_queue.h | 76 +- src/include/db/manifest/manifest.h | 10 +- src/include/structures/lsmtree/levels/level.h | 2 +- .../structures/lsmtree/levels/levels.h | 10 +- src/include/structures/lsmtree/lsmtree.h | 25 +- .../structures/lsmtree/segments/helpers.h | 1 - .../structures/lsmtree/segments/uuid.h | 1038 -------- src/include/structures/skiplist/skiplist.h | 12 + src/include/wal/wal.h | 14 +- src/main.cpp | 8 +- src/structures/lsmtree/CMakeLists.txt | 9 +- src/structures/lsmtree/levels/level.cpp | 21 +- src/structures/lsmtree/levels/levels.cpp | 54 +- src/structures/lsmtree/lsmtree.cpp | 124 +- src/structures/lsmtree/lsmtree_test.cpp | 156 +- src/structures/lsmtree/segments/gsl/gsl | 29 - .../lsmtree/segments/gsl/gsl_algorithm | 63 - .../lsmtree/segments/gsl/gsl_assert | 145 -- src/structures/lsmtree/segments/gsl/gsl_byte | 181 -- src/structures/lsmtree/segments/gsl/gsl_util | 158 -- .../lsmtree/segments/gsl/multi_span | 2242 ----------------- src/structures/lsmtree/segments/gsl/pointers | 193 -- src/structures/lsmtree/segments/gsl/span | 766 ------ .../lsmtree/segments/gsl/string_span | 730 ------ src/structures/lsmtree/segments/helpers.cpp | 2 +- src/structures/memtable/memtable.cpp | 2 + src/structures/skiplist/CMakeLists.txt | 7 +- src/structures/skiplist/skiplist_test.cpp | 78 +- src/wal/wal_test.cpp | 127 +- 36 files changed, 1421 insertions(+), 5901 deletions(-) create mode 100644 src/include/common/helpers.cpp create mode 100644 src/include/common/helpers.h create mode 100644 src/include/common/uuid.h delete mode 100644 src/include/structures/lsmtree/segments/uuid.h delete mode 100644 src/structures/lsmtree/segments/gsl/gsl delete mode 100644 src/structures/lsmtree/segments/gsl/gsl_algorithm delete mode 100644 src/structures/lsmtree/segments/gsl/gsl_assert delete mode 100644 src/structures/lsmtree/segments/gsl/gsl_byte delete mode 100644 src/structures/lsmtree/segments/gsl/gsl_util delete mode 100644 src/structures/lsmtree/segments/gsl/multi_span delete mode 100644 src/structures/lsmtree/segments/gsl/pointers delete mode 100644 src/structures/lsmtree/segments/gsl/span delete mode 100644 src/structures/lsmtree/segments/gsl/string_span diff --git a/CMakeLists.txt b/CMakeLists.txt index 779f998..bc61e4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,8 @@ find_package(prometheus-cpp REQUIRED) find_package(libassert REQUIRED) find_package(magic_enum REQUIRED) find_package(liburing REQUIRED) +find_package(Microsoft.GSL REQUIRED) +find_package(libuuid REQUIRED) add_subdirectory(proto) add_subdirectory(src) diff --git a/conanfile.txt b/conanfile.txt index 38e12a1..b3b9d84 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -5,7 +5,6 @@ fmt/11.2.0 cxxopts/3.2.0 nlohmann_json/3.11.3 json-schema-validator/2.3.0 -stduuid/1.2.3 benchmark/1.8.4 abseil/20240116.2 celero/2.9.0 @@ -16,6 +15,7 @@ prometheus-cpp/1.3.0 libassert/2.1.4 magic_enum/0.9.7 liburing/2.11 +ms-gsl/4.2.0 [tool_requires] protobuf/5.27.0 diff --git a/src/db/db.cpp b/src/db/db.cpp index 8a6c922..2b24bd7 100644 --- a/src/db/db.cpp +++ b/src/db/db.cpp @@ -375,8 +375,7 @@ auto db_t::onRaftCommit(const raft::v1::LogEntry &entry) -> bool { case tinykvpp::v1::DatabaseOperation::TYPE_PUT: { - if (m_pLSMtree->put(std::move(record)) == - structures::lsmtree::lsmtree_status_k::put_failed_k) + if (!m_pLSMtree->put(std::move(record))) { spdlog::error("Failed to update the lsmtree"); return false; diff --git a/src/db/manifest/manifest.cpp b/src/db/manifest/manifest.cpp index 91f28e3..1063ea6 100644 --- a/src/db/manifest/manifest.cpp +++ b/src/db/manifest/manifest.cpp @@ -7,9 +7,9 @@ namespace db::manifest { -manifest_t::manifest_t(fs::path_t path, wal::wal_t wal) noexcept +manifest_t::manifest_t(fs::path_t path, wal::shared_ptr_t wal) noexcept : m_path{std::move(path)}, - m_wal{std::move(wal)} + m_pWal{std::move(wal)} { } @@ -20,7 +20,7 @@ auto manifest_t::path() const noexcept -> fs::path_t auto manifest_t::add(record_t info) -> bool { - return m_enabled ? m_wal.add(std::move(info)) : [this]() + return m_enabled ? m_pWal->add(std::move(info)) : [this]() -> bool { spdlog::info("Manifest at {} is disabled - skipping record addition", m_path.c_str()); return false; @@ -29,27 +29,27 @@ auto manifest_t::add(record_t info) -> bool auto manifest_t::records() const noexcept -> std::vector { - return m_wal.records(); + return m_pWal->records(); } void manifest_t::enable() { m_enabled = true; spdlog::info("Manifest at {} enabled - ready to record changes", m_path.c_str()); - spdlog::debug("Manifest enable triggered with {} pending records", m_wal.size()); + spdlog::debug("Manifest enable triggered with {} pending records", m_pWal->size()); } void manifest_t::disable() { m_enabled = false; spdlog::info("Manifest at {} disabled - changes will not be recorded", m_path.c_str()); - spdlog::debug("Manifest disable triggered with {} pending records", m_wal.size()); + spdlog::debug("Manifest disable triggered with {} pending records", m_pWal->size()); } -auto manifest_builder_t::build(fs::path_t path, wal::wal_t wal) +auto manifest_builder_t::build(fs::path_t path, wal::shared_ptr_t pWal) -> std::optional { - return std::make_optional(manifest_t{std::move(path), std::move(wal)}); + return std::make_optional(manifest_t{std::move(path), std::move(pWal)}); } } // namespace db::manifest diff --git a/src/include/common/helpers.cpp b/src/include/common/helpers.cpp new file mode 100644 index 0000000..2db2db3 --- /dev/null +++ b/src/include/common/helpers.cpp @@ -0,0 +1,28 @@ +#include "common/helpers.h" + +namespace common +{ + +auto uuid() -> std::string +{ + std::random_device rnd; + auto seed_data = std::array{}; + std::ranges::generate(seed_data, std::ref(rnd)); + + std::seed_seq seq(std::begin(seed_data), std::end(seed_data)); + std::mt19937 generator(seq); + uuids::uuid_random_generator gen{generator}; + + return to_string(uuids::basic_uuid_random_generator{gen}()); +} + +auto segment_name() -> std::string +{ + return fmt::format("segment_{}", uuid()); +} + +auto segment_path(const std::filesystem::path &datadir, const std::string &name) + -> std::filesystem::path +{ + return datadir / name; +} diff --git a/src/include/common/helpers.h b/src/include/common/helpers.h new file mode 100644 index 0000000..557ce74 --- /dev/null +++ b/src/include/common/helpers.h @@ -0,0 +1,21 @@ +#include +#include +#include +#include +#include + +#include + +#include "common/uuid.h" + +namespace common +{ + +auto uuid() -> std::string; + +auto segment_name() -> std::string; + +auto segment_path(const std::filesystem::path &datadir, const std::string &name) + -> std::filesystem::path; + +} // namespace common diff --git a/src/include/common/uuid.h b/src/include/common/uuid.h new file mode 100644 index 0000000..d48059d --- /dev/null +++ b/src/include/common/uuid.h @@ -0,0 +1,967 @@ +#ifndef STDUUID_H +#define STDUUID_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus + +# if (__cplusplus >= 202002L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) +# define LIBUUID_CPP20_OR_GREATER +# endif + +#endif + + +#ifdef LIBUUID_CPP20_OR_GREATER +#include +#else +#include +#endif + +#ifdef _WIN32 + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif + +#ifdef UUID_SYSTEM_GENERATOR +#include +#endif + +#ifdef UUID_TIME_GENERATOR +#include +#pragma comment(lib, "IPHLPAPI.lib") +#endif + +#elif defined(__linux__) || defined(__unix__) + +#ifdef UUID_SYSTEM_GENERATOR +#include +#endif + +#elif defined(__APPLE__) + +#ifdef UUID_SYSTEM_GENERATOR +#include +#endif + +#endif + +namespace uuids +{ +#ifdef __cpp_lib_span + template + using span = std::span; +#else + template + using span = gsl::span; +#endif + + namespace detail + { + template + [[nodiscard]] constexpr inline unsigned char hex2char(TChar const ch) noexcept + { + if (ch >= static_cast('0') && ch <= static_cast('9')) + return static_cast(ch - static_cast('0')); + if (ch >= static_cast('a') && ch <= static_cast('f')) + return static_cast(10 + ch - static_cast('a')); + if (ch >= static_cast('A') && ch <= static_cast('F')) + return static_cast(10 + ch - static_cast('A')); + return 0; + } + + template + [[nodiscard]] constexpr inline bool is_hex(TChar const ch) noexcept + { + return + (ch >= static_cast('0') && ch <= static_cast('9')) || + (ch >= static_cast('a') && ch <= static_cast('f')) || + (ch >= static_cast('A') && ch <= static_cast('F')); + } + + template + [[nodiscard]] constexpr std::basic_string_view to_string_view(TChar const * str) noexcept + { + if (str) return str; + return {}; + } + + template + [[nodiscard]] + constexpr std::basic_string_view< + typename StringType::value_type, + typename StringType::traits_type> + to_string_view(StringType const & str) noexcept + { + return str; + } + + class sha1 + { + public: + using digest32_t = uint32_t[5]; + using digest8_t = uint8_t[20]; + + static constexpr unsigned int block_bytes = 64; + + [[nodiscard]] inline static uint32_t left_rotate(uint32_t value, size_t const count) noexcept + { + return (value << count) ^ (value >> (32 - count)); + } + + sha1() { reset(); } + + void reset() noexcept + { + m_digest[0] = 0x67452301; + m_digest[1] = 0xEFCDAB89; + m_digest[2] = 0x98BADCFE; + m_digest[3] = 0x10325476; + m_digest[4] = 0xC3D2E1F0; + m_blockByteIndex = 0; + m_byteCount = 0; + } + + void process_byte(uint8_t octet) + { + this->m_block[this->m_blockByteIndex++] = octet; + ++this->m_byteCount; + if (m_blockByteIndex == block_bytes) + { + this->m_blockByteIndex = 0; + process_block(); + } + } + + void process_block(void const * const start, void const * const end) + { + const uint8_t* begin = static_cast(start); + const uint8_t* finish = static_cast(end); + while (begin != finish) + { + process_byte(*begin); + begin++; + } + } + + void process_bytes(void const * const data, size_t const len) + { + const uint8_t* block = static_cast(data); + process_block(block, block + len); + } + + uint32_t const * get_digest(digest32_t digest) + { + size_t const bitCount = this->m_byteCount * 8; + process_byte(0x80); + if (this->m_blockByteIndex > 56) { + while (m_blockByteIndex != 0) { + process_byte(0); + } + while (m_blockByteIndex < 56) { + process_byte(0); + } + } + else { + while (m_blockByteIndex < 56) { + process_byte(0); + } + } + process_byte(0); + process_byte(0); + process_byte(0); + process_byte(0); + process_byte(static_cast((bitCount >> 24) & 0xFF)); + process_byte(static_cast((bitCount >> 16) & 0xFF)); + process_byte(static_cast((bitCount >> 8) & 0xFF)); + process_byte(static_cast((bitCount) & 0xFF)); + + memcpy(digest, m_digest, 5 * sizeof(uint32_t)); + return digest; + } + + uint8_t const * get_digest_bytes(digest8_t digest) + { + digest32_t d32; + get_digest(d32); + size_t di = 0; + digest[di++] = static_cast(d32[0] >> 24); + digest[di++] = static_cast(d32[0] >> 16); + digest[di++] = static_cast(d32[0] >> 8); + digest[di++] = static_cast(d32[0] >> 0); + + digest[di++] = static_cast(d32[1] >> 24); + digest[di++] = static_cast(d32[1] >> 16); + digest[di++] = static_cast(d32[1] >> 8); + digest[di++] = static_cast(d32[1] >> 0); + + digest[di++] = static_cast(d32[2] >> 24); + digest[di++] = static_cast(d32[2] >> 16); + digest[di++] = static_cast(d32[2] >> 8); + digest[di++] = static_cast(d32[2] >> 0); + + digest[di++] = static_cast(d32[3] >> 24); + digest[di++] = static_cast(d32[3] >> 16); + digest[di++] = static_cast(d32[3] >> 8); + digest[di++] = static_cast(d32[3] >> 0); + + digest[di++] = static_cast(d32[4] >> 24); + digest[di++] = static_cast(d32[4] >> 16); + digest[di++] = static_cast(d32[4] >> 8); + digest[di++] = static_cast(d32[4] >> 0); + + return digest; + } + + private: + void process_block() + { + uint32_t w[80]; + for (size_t i = 0; i < 16; i++) { + w[i] = static_cast(m_block[i * 4 + 0] << 24); + w[i] |= static_cast(m_block[i * 4 + 1] << 16); + w[i] |= static_cast(m_block[i * 4 + 2] << 8); + w[i] |= static_cast(m_block[i * 4 + 3]); + } + for (size_t i = 16; i < 80; i++) { + w[i] = left_rotate((w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16]), 1); + } + + uint32_t a = m_digest[0]; + uint32_t b = m_digest[1]; + uint32_t c = m_digest[2]; + uint32_t d = m_digest[3]; + uint32_t e = m_digest[4]; + + for (std::size_t i = 0; i < 80; ++i) + { + uint32_t f = 0; + uint32_t k = 0; + + if (i < 20) { + f = (b & c) | (~b & d); + k = 0x5A827999; + } + else if (i < 40) { + f = b ^ c ^ d; + k = 0x6ED9EBA1; + } + else if (i < 60) { + f = (b & c) | (b & d) | (c & d); + k = 0x8F1BBCDC; + } + else { + f = b ^ c ^ d; + k = 0xCA62C1D6; + } + uint32_t temp = left_rotate(a, 5) + f + e + k + w[i]; + e = d; + d = c; + c = left_rotate(b, 30); + b = a; + a = temp; + } + + m_digest[0] += a; + m_digest[1] += b; + m_digest[2] += c; + m_digest[3] += d; + m_digest[4] += e; + } + + private: + digest32_t m_digest; + uint8_t m_block[64]; + size_t m_blockByteIndex; + size_t m_byteCount; + }; + + template + inline constexpr CharT empty_guid[37] = "00000000-0000-0000-0000-000000000000"; + + template <> + inline constexpr wchar_t empty_guid[37] = L"00000000-0000-0000-0000-000000000000"; + + template + inline constexpr CharT guid_encoder[17] = "0123456789abcdef"; + + template <> + inline constexpr wchar_t guid_encoder[17] = L"0123456789abcdef"; + } + + // -------------------------------------------------------------------------------------------------------------------------- + // UUID format https://tools.ietf.org/html/rfc4122 + // -------------------------------------------------------------------------------------------------------------------------- + + // -------------------------------------------------------------------------------------------------------------------------- + // Field NDR Data Type Octet # Note + // -------------------------------------------------------------------------------------------------------------------------- + // time_low unsigned long 0 - 3 The low field of the timestamp. + // time_mid unsigned short 4 - 5 The middle field of the timestamp. + // time_hi_and_version unsigned short 6 - 7 The high field of the timestamp multiplexed with the version number. + // clock_seq_hi_and_reserved unsigned small 8 The high field of the clock sequence multiplexed with the variant. + // clock_seq_low unsigned small 9 The low field of the clock sequence. + // node character 10 - 15 The spatially unique node identifier. + // -------------------------------------------------------------------------------------------------------------------------- + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | time_low | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | time_mid | time_hi_and_version | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |clk_seq_hi_res | clk_seq_low | node (0-1) | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | node (2-5) | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + // -------------------------------------------------------------------------------------------------------------------------- + // enumerations + // -------------------------------------------------------------------------------------------------------------------------- + + // indicated by a bit pattern in octet 8, marked with N in xxxxxxxx-xxxx-xxxx-Nxxx-xxxxxxxxxxxx + enum class uuid_variant + { + // NCS backward compatibility (with the obsolete Apollo Network Computing System 1.5 UUID format) + // N bit pattern: 0xxx + // > the first 6 octets of the UUID are a 48-bit timestamp (the number of 4 microsecond units of time since 1 Jan 1980 UTC); + // > the next 2 octets are reserved; + // > the next octet is the "address family"; + // > the final 7 octets are a 56-bit host ID in the form specified by the address family + ncs, + + // RFC 4122/DCE 1.1 + // N bit pattern: 10xx + // > big-endian byte order + rfc, + + // Microsoft Corporation backward compatibility + // N bit pattern: 110x + // > little endian byte order + // > formely used in the Component Object Model (COM) library + microsoft, + + // reserved for possible future definition + // N bit pattern: 111x + reserved + }; + + // indicated by a bit pattern in octet 6, marked with M in xxxxxxxx-xxxx-Mxxx-xxxx-xxxxxxxxxxxx + enum class uuid_version + { + none = 0, // only possible for nil or invalid uuids + time_based = 1, // The time-based version specified in RFC 4122 + dce_security = 2, // DCE Security version, with embedded POSIX UIDs. + name_based_md5 = 3, // The name-based version specified in RFS 4122 with MD5 hashing + random_number_based = 4, // The randomly or pseudo-randomly generated version specified in RFS 4122 + name_based_sha1 = 5 // The name-based version specified in RFS 4122 with SHA1 hashing + }; + + // Forward declare uuid & to_string so that we can declare to_string as a friend later. + class uuid; + template , + class Allocator = std::allocator> + std::basic_string to_string(uuid const &id); + + // -------------------------------------------------------------------------------------------------------------------------- + // uuid class + // -------------------------------------------------------------------------------------------------------------------------- + class uuid + { + public: + using value_type = uint8_t; + + constexpr uuid() noexcept = default; + + uuid(value_type(&arr)[16]) noexcept + { + std::copy(std::cbegin(arr), std::cend(arr), std::begin(data)); + } + + constexpr uuid(std::array const & arr) noexcept : data{arr} {} + + explicit uuid(span bytes) + { + std::copy(std::cbegin(bytes), std::cend(bytes), std::begin(data)); + } + + template + explicit uuid(ForwardIterator first, ForwardIterator last) + { + if (std::distance(first, last) == 16) + std::copy(first, last, std::begin(data)); + } + + [[nodiscard]] constexpr uuid_variant variant() const noexcept + { + if ((data[8] & 0x80) == 0x00) + return uuid_variant::ncs; + else if ((data[8] & 0xC0) == 0x80) + return uuid_variant::rfc; + else if ((data[8] & 0xE0) == 0xC0) + return uuid_variant::microsoft; + else + return uuid_variant::reserved; + } + + [[nodiscard]] constexpr uuid_version version() const noexcept + { + if ((data[6] & 0xF0) == 0x10) + return uuid_version::time_based; + else if ((data[6] & 0xF0) == 0x20) + return uuid_version::dce_security; + else if ((data[6] & 0xF0) == 0x30) + return uuid_version::name_based_md5; + else if ((data[6] & 0xF0) == 0x40) + return uuid_version::random_number_based; + else if ((data[6] & 0xF0) == 0x50) + return uuid_version::name_based_sha1; + else + return uuid_version::none; + } + + [[nodiscard]] constexpr bool is_nil() const noexcept + { + for (size_t i = 0; i < data.size(); ++i) if (data[i] != 0) return false; + return true; + } + + void swap(uuid & other) noexcept + { + data.swap(other.data); + } + + [[nodiscard]] inline span as_bytes() const + { + return span(reinterpret_cast(data.data()), 16); + } + + template + [[nodiscard]] constexpr static bool is_valid_uuid(StringType const & in_str) noexcept + { + auto str = detail::to_string_view(in_str); + bool firstDigit = true; + size_t hasBraces = 0; + size_t index = 0; + + if (str.empty()) + return false; + + if (str.front() == '{') + hasBraces = 1; + if (hasBraces && str.back() != '}') + return false; + + for (size_t i = hasBraces; i < str.size() - hasBraces; ++i) + { + if (str[i] == '-') continue; + + if (index >= 16 || !detail::is_hex(str[i])) + { + return false; + } + + if (firstDigit) + { + firstDigit = false; + } + else + { + index++; + firstDigit = true; + } + } + + if (index < 16) + { + return false; + } + + return true; + } + + template + [[nodiscard]] constexpr static std::optional from_string(StringType const & in_str) noexcept + { + auto str = detail::to_string_view(in_str); + bool firstDigit = true; + size_t hasBraces = 0; + size_t index = 0; + + std::array data{ { 0 } }; + + if (str.empty()) return {}; + + if (str.front() == '{') + hasBraces = 1; + if (hasBraces && str.back() != '}') + return {}; + + for (size_t i = hasBraces; i < str.size() - hasBraces; ++i) + { + if (str[i] == '-') continue; + + if (index >= 16 || !detail::is_hex(str[i])) + { + return {}; + } + + if (firstDigit) + { + data[index] = static_cast(detail::hex2char(str[i]) << 4); + firstDigit = false; + } + else + { + data[index] = static_cast(data[index] | detail::hex2char(str[i])); + index++; + firstDigit = true; + } + } + + if (index < 16) + { + return {}; + } + + return uuid{ data }; + } + + private: + std::array data{ { 0 } }; + + friend bool operator==(uuid const & lhs, uuid const & rhs) noexcept; + friend bool operator<(uuid const & lhs, uuid const & rhs) noexcept; + + template + friend std::basic_ostream & operator<<(std::basic_ostream &s, uuid const & id); + + template + friend std::basic_string to_string(uuid const& id); + + friend std::hash; + }; + + // -------------------------------------------------------------------------------------------------------------------------- + // operators and non-member functions + // -------------------------------------------------------------------------------------------------------------------------- + + [[nodiscard]] inline bool operator== (uuid const& lhs, uuid const& rhs) noexcept + { + return lhs.data == rhs.data; + } + + [[nodiscard]] inline bool operator!= (uuid const& lhs, uuid const& rhs) noexcept + { + return !(lhs == rhs); + } + + [[nodiscard]] inline bool operator< (uuid const& lhs, uuid const& rhs) noexcept + { + return lhs.data < rhs.data; + } + + template + [[nodiscard]] inline std::basic_string to_string(uuid const & id) + { + std::basic_string uustr{detail::empty_guid}; + + for (size_t i = 0, index = 0; i < 36; ++i) + { + if (i == 8 || i == 13 || i == 18 || i == 23) + { + continue; + } + uustr[i] = detail::guid_encoder[id.data[index] >> 4 & 0x0f]; + uustr[++i] = detail::guid_encoder[id.data[index] & 0x0f]; + index++; + } + + return uustr; + } + + template + std::basic_ostream& operator<<(std::basic_ostream& s, uuid const& id) + { + s << to_string(id); + return s; + } + + inline void swap(uuids::uuid & lhs, uuids::uuid & rhs) noexcept + { + lhs.swap(rhs); + } + + // -------------------------------------------------------------------------------------------------------------------------- + // namespace IDs that could be used for generating name-based uuids + // -------------------------------------------------------------------------------------------------------------------------- + + // Name string is a fully-qualified domain name + static uuid uuid_namespace_dns{ {0x6b, 0xa7, 0xb8, 0x10, 0x9d, 0xad, 0x11, 0xd1, 0x80, 0xb4, 0x00, 0xc0, 0x4f, 0xd4, 0x30, 0xc8} }; + + // Name string is a URL + static uuid uuid_namespace_url{ {0x6b, 0xa7, 0xb8, 0x11, 0x9d, 0xad, 0x11, 0xd1, 0x80, 0xb4, 0x00, 0xc0, 0x4f, 0xd4, 0x30, 0xc8} }; + + // Name string is an ISO OID (See https://oidref.com/, https://en.wikipedia.org/wiki/Object_identifier) + static uuid uuid_namespace_oid{ {0x6b, 0xa7, 0xb8, 0x12, 0x9d, 0xad, 0x11, 0xd1, 0x80, 0xb4, 0x00, 0xc0, 0x4f, 0xd4, 0x30, 0xc8} }; + + // Name string is an X.500 DN, in DER or a text output format (See https://en.wikipedia.org/wiki/X.500, https://en.wikipedia.org/wiki/Abstract_Syntax_Notation_One) + static uuid uuid_namespace_x500{ {0x6b, 0xa7, 0xb8, 0x14, 0x9d, 0xad, 0x11, 0xd1, 0x80, 0xb4, 0x00, 0xc0, 0x4f, 0xd4, 0x30, 0xc8} }; + + // -------------------------------------------------------------------------------------------------------------------------- + // uuid generators + // -------------------------------------------------------------------------------------------------------------------------- + +#ifdef UUID_SYSTEM_GENERATOR + class uuid_system_generator + { + public: + using result_type = uuid; + + uuid operator()() + { +#ifdef _WIN32 + + GUID newId; + HRESULT hr = ::CoCreateGuid(&newId); + + if (FAILED(hr)) + { + throw std::system_error(hr, std::system_category(), "CoCreateGuid failed"); + } + + std::array bytes = + { { + static_cast((newId.Data1 >> 24) & 0xFF), + static_cast((newId.Data1 >> 16) & 0xFF), + static_cast((newId.Data1 >> 8) & 0xFF), + static_cast((newId.Data1) & 0xFF), + + (unsigned char)((newId.Data2 >> 8) & 0xFF), + (unsigned char)((newId.Data2) & 0xFF), + + (unsigned char)((newId.Data3 >> 8) & 0xFF), + (unsigned char)((newId.Data3) & 0xFF), + + newId.Data4[0], + newId.Data4[1], + newId.Data4[2], + newId.Data4[3], + newId.Data4[4], + newId.Data4[5], + newId.Data4[6], + newId.Data4[7] + } }; + + return uuid{ std::begin(bytes), std::end(bytes) }; + +#elif defined(__linux__) || defined(__unix__) + + uuid_t id; + uuid_generate(id); + + std::array bytes = + { { + id[0], + id[1], + id[2], + id[3], + id[4], + id[5], + id[6], + id[7], + id[8], + id[9], + id[10], + id[11], + id[12], + id[13], + id[14], + id[15] + } }; + + return uuid{ std::begin(bytes), std::end(bytes) }; + +#elif defined(__APPLE__) + auto newId = CFUUIDCreate(NULL); + auto bytes = CFUUIDGetUUIDBytes(newId); + CFRelease(newId); + + std::array arrbytes = + { { + bytes.byte0, + bytes.byte1, + bytes.byte2, + bytes.byte3, + bytes.byte4, + bytes.byte5, + bytes.byte6, + bytes.byte7, + bytes.byte8, + bytes.byte9, + bytes.byte10, + bytes.byte11, + bytes.byte12, + bytes.byte13, + bytes.byte14, + bytes.byte15 + } }; + return uuid{ std::begin(arrbytes), std::end(arrbytes) }; +#else + return uuid{}; +#endif + } + }; +#endif + + template + class basic_uuid_random_generator + { + public: + using engine_type = UniformRandomNumberGenerator; + + explicit basic_uuid_random_generator(engine_type& gen) : + generator(&gen, [](auto) {}) {} + explicit basic_uuid_random_generator(engine_type* gen) : + generator(gen, [](auto) {}) {} + + [[nodiscard]] uuid operator()() + { + alignas(uint32_t) uint8_t bytes[16]; + for (int i = 0; i < 16; i += 4) + *reinterpret_cast(bytes + i) = distribution(*generator); + + // variant must be 10xxxxxx + bytes[8] &= 0xBF; + bytes[8] |= 0x80; + + // version must be 0100xxxx + bytes[6] &= 0x4F; + bytes[6] |= 0x40; + + return uuid{std::begin(bytes), std::end(bytes)}; + } + + private: + std::uniform_int_distribution distribution; + std::shared_ptr generator; + }; + + using uuid_random_generator = basic_uuid_random_generator; + + class uuid_name_generator + { + public: + explicit uuid_name_generator(uuid const& namespace_uuid) noexcept + : nsuuid(namespace_uuid) + {} + + template + [[nodiscard]] uuid operator()(StringType const & name) + { + reset(); + process_characters(detail::to_string_view(name)); + return make_uuid(); + } + + private: + void reset() + { + hasher.reset(); + std::byte bytes[16]; + auto nsbytes = nsuuid.as_bytes(); + std::copy(std::cbegin(nsbytes), std::cend(nsbytes), bytes); + hasher.process_bytes(bytes, 16); + } + + template + void process_characters(std::basic_string_view const str) + { + for (uint32_t c : str) + { + hasher.process_byte(static_cast(c & 0xFF)); + if constexpr (!std::is_same_v) + { + hasher.process_byte(static_cast((c >> 8) & 0xFF)); + hasher.process_byte(static_cast((c >> 16) & 0xFF)); + hasher.process_byte(static_cast((c >> 24) & 0xFF)); + } + } + } + + [[nodiscard]] uuid make_uuid() + { + detail::sha1::digest8_t digest; + hasher.get_digest_bytes(digest); + + // variant must be 0b10xxxxxx + digest[8] &= 0xBF; + digest[8] |= 0x80; + + // version must be 0b0101xxxx + digest[6] &= 0x5F; + digest[6] |= 0x50; + + return uuid{ digest, digest + 16 }; + } + + private: + uuid nsuuid; + detail::sha1 hasher; + }; + +#ifdef UUID_TIME_GENERATOR + // !!! DO NOT USE THIS IN PRODUCTION + // this implementation is unreliable for good uuids + class uuid_time_generator + { + using mac_address = std::array; + + std::optional device_address; + + [[nodiscard]] bool get_mac_address() + { + if (device_address.has_value()) + { + return true; + } + +#ifdef _WIN32 + DWORD len = 0; + auto ret = GetAdaptersInfo(nullptr, &len); + if (ret != ERROR_BUFFER_OVERFLOW) return false; + std::vector buf(len); + auto pips = reinterpret_cast(&buf.front()); + ret = GetAdaptersInfo(pips, &len); + if (ret != ERROR_SUCCESS) return false; + mac_address addr; + std::copy(pips->Address, pips->Address + 6, std::begin(addr)); + device_address = addr; +#endif + + return device_address.has_value(); + } + + [[nodiscard]] long long get_time_intervals() + { + auto start = std::chrono::system_clock::from_time_t(time_t(-12219292800)); + auto diff = std::chrono::system_clock::now() - start; + auto ns = std::chrono::duration_cast(diff).count(); + return ns / 100; + } + + [[nodiscard]] static unsigned short get_clock_sequence() + { + static std::mt19937 clock_gen(std::random_device{}()); + static std::uniform_int_distribution clock_dis; + static std::atomic_ushort clock_sequence = clock_dis(clock_gen); + return clock_sequence++; + } + + public: + [[nodiscard]] uuid operator()() + { + if (get_mac_address()) + { + std::array data; + + auto tm = get_time_intervals(); + + auto clock_seq = get_clock_sequence(); + + auto ptm = reinterpret_cast(&tm); + + memcpy(&data[0], ptm + 4, 4); + memcpy(&data[4], ptm + 2, 2); + memcpy(&data[6], ptm, 2); + + memcpy(&data[8], &clock_seq, 2); + + // variant must be 0b10xxxxxx + data[8] &= 0xBF; + data[8] |= 0x80; + + // version must be 0b0001xxxx + data[6] &= 0x1F; + data[6] |= 0x10; + + memcpy(&data[10], &device_address.value()[0], 6); + + return uuids::uuid{std::cbegin(data), std::cend(data)}; + } + + return {}; + } + }; +#endif +} + +namespace std +{ + template <> + struct hash + { + using argument_type = uuids::uuid; + using result_type = std::size_t; + + [[nodiscard]] result_type operator()(argument_type const &uuid) const + { +#ifdef UUID_HASH_STRING_BASED + std::hash hasher; + return static_cast(hasher(uuids::to_string(uuid))); +#else + uint64_t l = + static_cast(uuid.data[0]) << 56 | + static_cast(uuid.data[1]) << 48 | + static_cast(uuid.data[2]) << 40 | + static_cast(uuid.data[3]) << 32 | + static_cast(uuid.data[4]) << 24 | + static_cast(uuid.data[5]) << 16 | + static_cast(uuid.data[6]) << 8 | + static_cast(uuid.data[7]); + uint64_t h = + static_cast(uuid.data[8]) << 56 | + static_cast(uuid.data[9]) << 48 | + static_cast(uuid.data[10]) << 40 | + static_cast(uuid.data[11]) << 32 | + static_cast(uuid.data[12]) << 24 | + static_cast(uuid.data[13]) << 16 | + static_cast(uuid.data[14]) << 8 | + static_cast(uuid.data[15]); + + if constexpr (sizeof(result_type) > 4) + { + return result_type(l ^ h); + } + else + { + uint64_t hash64 = l ^ h; + return result_type(uint32_t(hash64 >> 32) ^ uint32_t(hash64)); + } +#endif + } + }; +} + +#endif /* STDUUID_H */ diff --git a/src/include/concurrency/thread_safe_queue.h b/src/include/concurrency/thread_safe_queue.h index b3e9fed..b330661 100644 --- a/src/include/concurrency/thread_safe_queue.h +++ b/src/include/concurrency/thread_safe_queue.h @@ -1,12 +1,16 @@ #pragma once +#include #include #include +#include #include #include #include +#include +#include "db/db_config.h" #include "helpers.h" namespace concurrency @@ -17,7 +21,7 @@ template class thread_safe_queue_t public: using queue_t = std::deque; - thread_safe_queue_t() = default; + thread_safe_queue_t() noexcept; thread_safe_queue_t(const thread_safe_queue_t &) = delete; auto operator=(const thread_safe_queue_t &) -> thread_safe_queue_t & = delete; @@ -32,6 +36,16 @@ template class thread_safe_queue_t auto pop() -> std::optional; auto pop_all() -> queue_t; + // Transactional interface. + // NOLINTBEGIN(modernize-use-trailing-return-type) + /** + * Returns stored items in oldest-to-newest order (queue in a reversed order). + */ + std::optional> reserve() noexcept ABSL_LOCKS_EXCLUDED(m_mutex); + bool consume() noexcept ABSL_LOCKS_EXCLUDED(m_mutex); + + // NOLINTEND(modernize-use-trailing-return-type) + auto size() -> std::size_t; template @@ -40,12 +54,20 @@ template class thread_safe_queue_t void shutdown(); private: + std::atomic m_shutdown{false}; + mutable absl::Mutex m_mutex; - queue_t m_queue ABSL_GUARDED_BY(m_mutex); - std::atomic m_shutdown{false}; + queue_t m_queue ABSL_GUARDED_BY(m_mutex); + std::optional m_reservedIndex ABSL_GUARDED_BY(m_mutex); }; +template +inline thread_safe_queue_t::thread_safe_queue_t() noexcept + : m_reservedIndex{std::nullopt} +{ +} + template inline thread_safe_queue_t::thread_safe_queue_t(thread_safe_queue_t &&other) noexcept : m_queue{ @@ -102,9 +124,52 @@ template inline auto thread_safe_queue_t::pop() -> std:: template inline auto thread_safe_queue_t::pop_all() -> queue_t { absl::WriterMutexLock lock(&m_mutex); + spdlog::info("thread_safe_queue_t::pop_all()"); return m_queue.empty() ? queue_t{} : std::move(m_queue); } +template +inline auto thread_safe_queue_t::reserve() noexcept -> std::optional> +{ + absl::WriterMutexLock lock{&m_mutex}; + + if (m_queue.empty()) + { + spdlog::debug("thread_safe_queue: Queue is empty. Nothing to reserve"); + return std::nullopt; + } + + if (m_reservedIndex.has_value()) + { + spdlog::error("thread_safe_queue: Another reservation is in progress"); + return std::nullopt; + } + m_reservedIndex = m_queue.size(); + + return std::make_optional>(std::rbegin(m_queue), std::rend(m_queue)); +} + +template inline auto thread_safe_queue_t::consume() noexcept -> bool +{ + absl::WriterMutexLock lock{&m_mutex}; + + if (!m_reservedIndex.has_value()) + { + spdlog::error("thread_safe_queue: No reservation to consume"); + return false; + } + + const auto consumeUpTo{m_reservedIndex.value()}; + ASSERT(consumeUpTo != 0); + for (std::uint64_t idx{0}; idx < consumeUpTo; idx++) + { + m_queue.pop_front(); + } + m_reservedIndex = std::nullopt; + + return true; +} + template inline auto thread_safe_queue_t::size() -> std::size_t { absl::ReaderMutexLock lock(&m_mutex); @@ -118,6 +183,11 @@ inline auto thread_safe_queue_t::find(const TKey &recordKey) const noexce { absl::ReaderMutexLock lock(&m_mutex); + if (m_queue.empty()) + { + spdlog::info("MEMTABLES ARE EMPTY"); + } + for (const auto &memtable : m_queue) { if (auto record = memtable.find(recordKey); record.has_value()) diff --git a/src/include/db/manifest/manifest.h b/src/include/db/manifest/manifest.h index cc430e3..e66e1b9 100644 --- a/src/include/db/manifest/manifest.h +++ b/src/include/db/manifest/manifest.h @@ -157,7 +157,7 @@ struct manifest_t using record_t = std::variant; using storage_t = std::vector; - manifest_t(fs::path_t path, wal::wal_t wal) noexcept; + manifest_t(fs::path_t path, wal::shared_ptr_t pWal) noexcept; [[nodiscard]] auto path() const noexcept -> fs::path_t; @@ -168,9 +168,9 @@ struct manifest_t void disable(); private: - bool m_enabled{false}; - fs::path_t m_path; - wal::wal_t m_wal; + bool m_enabled{false}; + fs::path_t m_path; + wal::shared_ptr_t m_pWal; }; using shared_ptr_t = std::shared_ptr; @@ -233,7 +233,7 @@ auto operator>>(TStream &stream, manifest_t::record_t &rRecord) -> TStream & struct manifest_builder_t final { - [[nodiscard]] auto build(fs::path_t path, wal::wal_t wal) + [[nodiscard]] auto build(fs::path_t path, wal::shared_ptr_t wal) -> std::optional; }; diff --git a/src/include/structures/lsmtree/levels/level.h b/src/include/structures/lsmtree/levels/level.h index 06c5bd3..53950b1 100644 --- a/src/include/structures/lsmtree/levels/level.h +++ b/src/include/structures/lsmtree/levels/level.h @@ -61,7 +61,7 @@ class level_t [[__nodiscard__]] auto bytes_used() const noexcept -> std::size_t; private: - void purge(const segments::regular_segment::shared_ptr_t &pSegment) noexcept; + void purge(segments::regular_segment::shared_ptr_t pSegment) noexcept; mutable absl::Mutex m_mutex; diff --git a/src/include/structures/lsmtree/levels/levels.h b/src/include/structures/lsmtree/levels/levels.h index 341f473..7caa39f 100644 --- a/src/include/structures/lsmtree/levels/levels.h +++ b/src/include/structures/lsmtree/levels/levels.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -13,6 +14,8 @@ namespace structures::lsmtree::levels { +static constexpr const auto gWaitForSegmentFlushNotificationTimeout{absl::Milliseconds(50)}; + class levels_t { public: @@ -25,8 +28,8 @@ class levels_t levels_t(const levels_t &) = delete; auto operator=(const levels_t &) -> levels_t & = delete; - levels_t(levels_t &&other) noexcept; - auto operator=(levels_t &&other) noexcept -> levels_t &; + levels_t(levels_t &&other) noexcept = delete; + auto operator=(levels_t &&other) noexcept -> levels_t & = delete; ~levels_t() noexcept; @@ -34,16 +37,13 @@ class levels_t [[nodiscard]] auto compact() -> segments::regular_segment::shared_ptr_t; [[nodiscard]] auto level() noexcept -> level::shared_ptr_t; [[nodiscard]] auto level(std::size_t idx) noexcept -> level::shared_ptr_t; - [[nodiscard]] auto size() const noexcept -> levels_storage_t::size_type; - [[nodiscard]] auto flush_to_level0(memtable::memtable_t memtable) const noexcept -> segments::regular_segment::shared_ptr_t; auto restore() noexcept -> void; private: - void move_from(levels_t &&other) noexcept; void compaction_task(std::stop_token stoken) noexcept; config::shared_ptr_t m_pConfig; diff --git a/src/include/structures/lsmtree/lsmtree.h b/src/include/structures/lsmtree/lsmtree.h index 27f4c55..f6cd4eb 100644 --- a/src/include/structures/lsmtree/lsmtree.h +++ b/src/include/structures/lsmtree/lsmtree.h @@ -1,5 +1,7 @@ #pragma once +#include +#include #include #include #include @@ -17,10 +19,16 @@ namespace structures::lsmtree { -enum class lsmtree_status_k : uint8_t +enum class lsmtree_success_k : std::int8_t { + undefined_k = -1, ok_k, memtable_reset_k, +}; + +enum class lsmtree_error_k : std::int8_t +{ + undefined_k = -1, put_failed_k, get_failed_k, recover_failed_k, @@ -34,7 +42,6 @@ enum class lsmtree_status_k : uint8_t level_add_failed_k, level_compact_failed_k, level_purge_failed_k, - unknown_error_k }; class lsmtree_t final @@ -55,21 +62,21 @@ class lsmtree_t final lsmtree_t(const lsmtree_t &) = delete; auto operator=(const lsmtree_t &) -> lsmtree_t & = delete; - lsmtree_t(lsmtree_t &&other) noexcept; - auto operator=(lsmtree_t &&other) noexcept -> lsmtree_t &; + lsmtree_t(lsmtree_t &&other) noexcept = delete; + auto operator=(lsmtree_t &&other) noexcept -> lsmtree_t & = delete; ~lsmtree_t() noexcept; - [[nodiscard]] auto put(record_t record) noexcept -> lsmtree_status_k; + [[nodiscard]] auto put(record_t record) noexcept + -> std::expected; + [[nodiscard]] auto get(const key_t &key) noexcept -> std::optional; private: void memtable_flush_task(std::stop_token stoken) noexcept; - void move_from(lsmtree_t &&other) noexcept; config::shared_ptr_t m_pConfig; - // TODO(lnikon): Add absl:: thread guards! absl::Mutex m_mutex; std::optional m_table ABSL_GUARDED_BY(m_mutex); db::manifest::shared_ptr_t m_pManifest ABSL_GUARDED_BY(m_mutex); @@ -82,7 +89,9 @@ using shared_ptr_t = std::shared_ptr; struct lsmtree_builder_t final { - using wal_t = wal::shared_ptr_t; + using entry_t = raft::v1::LogEntry; + using wal_t = wal::shared_ptr_t; + [[nodiscard]] auto build(config::shared_ptr_t pConfig, db::manifest::shared_ptr_t pManifest, wal_t pWal) const -> std::shared_ptr; diff --git a/src/include/structures/lsmtree/segments/helpers.h b/src/include/structures/lsmtree/segments/helpers.h index 596dfd2..4d7f302 100644 --- a/src/include/structures/lsmtree/segments/helpers.h +++ b/src/include/structures/lsmtree/segments/helpers.h @@ -1,6 +1,5 @@ #pragma once -#include "config/config.h" #include "structures/lsmtree/segments/types.h" #include diff --git a/src/include/structures/lsmtree/segments/uuid.h b/src/include/structures/lsmtree/segments/uuid.h deleted file mode 100644 index 8c56bf9..0000000 --- a/src/include/structures/lsmtree/segments/uuid.h +++ /dev/null @@ -1,1038 +0,0 @@ -#ifndef STDUUID_H -#define STDUUID_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __cplusplus - -#if (__cplusplus >= 202002L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) -#define LIBUUID_CPP20_OR_GREATER -#endif - -#endif - -#ifdef LIBUUID_CPP20_OR_GREATER -#include -#else -#include -#endif - -#ifdef _WIN32 - -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#ifndef NOMINMAX -#define NOMINMAX -#endif - -#ifdef UUID_SYSTEM_GENERATOR -#include -#endif - -#ifdef UUID_TIME_GENERATOR -#include -#pragma comment(lib, "IPHLPAPI.lib") -#endif - -#elif defined(__linux__) || defined(__unix__) - -#ifdef UUID_SYSTEM_GENERATOR -#include -#endif - -#elif defined(__APPLE__) - -#ifdef UUID_SYSTEM_GENERATOR -#include -#endif - -#endif - -namespace uuids -{ -#ifdef __cpp_lib_span -template using span = std::span; -#else -template using span = gsl::span; -#endif - -namespace detail -{ -template -[[nodiscard]] constexpr inline unsigned char hex2char(TChar const ch) noexcept -{ - if (ch >= static_cast('0') && ch <= static_cast('9')) - return static_cast(ch - static_cast('0')); - if (ch >= static_cast('a') && ch <= static_cast('f')) - return static_cast(10 + ch - static_cast('a')); - if (ch >= static_cast('A') && ch <= static_cast('F')) - return static_cast(10 + ch - static_cast('A')); - return 0; -} - -template [[nodiscard]] constexpr inline bool is_hex(TChar const ch) noexcept -{ - return (ch >= static_cast('0') && ch <= static_cast('9')) || - (ch >= static_cast('a') && ch <= static_cast('f')) || - (ch >= static_cast('A') && ch <= static_cast('F')); -} - -template -[[nodiscard]] constexpr std::basic_string_view to_string_view(TChar const *str) noexcept -{ - if (str) - return str; - return {}; -} - -template -[[nodiscard]] -constexpr std::basic_string_view -to_string_view(StringType const &str) noexcept -{ - return str; -} - -class sha1 -{ - public: - using digest32_t = uint32_t[5]; - using digest8_t = uint8_t[20]; - - static constexpr unsigned int block_bytes = 64; - - [[nodiscard]] inline static uint32_t left_rotate(uint32_t value, size_t const count) noexcept - { - return (value << count) ^ (value >> (32 - count)); - } - - sha1() - { - reset(); - } - - void reset() noexcept - { - m_digest[0] = 0x67452301; - m_digest[1] = 0xEFCDAB89; - m_digest[2] = 0x98BADCFE; - m_digest[3] = 0x10325476; - m_digest[4] = 0xC3D2E1F0; - m_blockByteIndex = 0; - m_byteCount = 0; - } - - void process_byte(uint8_t octet) - { - this->m_block[this->m_blockByteIndex++] = octet; - ++this->m_byteCount; - if (m_blockByteIndex == block_bytes) - { - this->m_blockByteIndex = 0; - process_block(); - } - } - - void process_block(void const *const start, void const *const end) - { - const uint8_t *begin = static_cast(start); - const uint8_t *finish = static_cast(end); - while (begin != finish) - { - process_byte(*begin); - begin++; - } - } - - void process_bytes(void const *const data, size_t const len) - { - const uint8_t *block = static_cast(data); - process_block(block, block + len); - } - - uint32_t const *get_digest(digest32_t digest) - { - size_t const bitCount = this->m_byteCount * 8; - process_byte(0x80); - if (this->m_blockByteIndex > 56) - { - while (m_blockByteIndex != 0) - { - process_byte(0); - } - while (m_blockByteIndex < 56) - { - process_byte(0); - } - } - else - { - while (m_blockByteIndex < 56) - { - process_byte(0); - } - } - process_byte(0); - process_byte(0); - process_byte(0); - process_byte(0); - process_byte(static_cast((bitCount >> 24) & 0xFF)); - process_byte(static_cast((bitCount >> 16) & 0xFF)); - process_byte(static_cast((bitCount >> 8) & 0xFF)); - process_byte(static_cast((bitCount) & 0xFF)); - - memcpy(digest, m_digest, 5 * sizeof(uint32_t)); - return digest; - } - - uint8_t const *get_digest_bytes(digest8_t digest) - { - digest32_t d32; - get_digest(d32); - size_t di = 0; - digest[di++] = static_cast(d32[0] >> 24); - digest[di++] = static_cast(d32[0] >> 16); - digest[di++] = static_cast(d32[0] >> 8); - digest[di++] = static_cast(d32[0] >> 0); - - digest[di++] = static_cast(d32[1] >> 24); - digest[di++] = static_cast(d32[1] >> 16); - digest[di++] = static_cast(d32[1] >> 8); - digest[di++] = static_cast(d32[1] >> 0); - - digest[di++] = static_cast(d32[2] >> 24); - digest[di++] = static_cast(d32[2] >> 16); - digest[di++] = static_cast(d32[2] >> 8); - digest[di++] = static_cast(d32[2] >> 0); - - digest[di++] = static_cast(d32[3] >> 24); - digest[di++] = static_cast(d32[3] >> 16); - digest[di++] = static_cast(d32[3] >> 8); - digest[di++] = static_cast(d32[3] >> 0); - - digest[di++] = static_cast(d32[4] >> 24); - digest[di++] = static_cast(d32[4] >> 16); - digest[di++] = static_cast(d32[4] >> 8); - digest[di++] = static_cast(d32[4] >> 0); - - return digest; - } - - private: - void process_block() - { - uint32_t w[80]; - for (size_t i = 0; i < 16; i++) - { - w[i] = static_cast(m_block[i * 4 + 0] << 24); - w[i] |= static_cast(m_block[i * 4 + 1] << 16); - w[i] |= static_cast(m_block[i * 4 + 2] << 8); - w[i] |= static_cast(m_block[i * 4 + 3]); - } - for (size_t i = 16; i < 80; i++) - { - w[i] = left_rotate((w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16]), 1); - } - - uint32_t a = m_digest[0]; - uint32_t b = m_digest[1]; - uint32_t c = m_digest[2]; - uint32_t d = m_digest[3]; - uint32_t e = m_digest[4]; - - for (std::size_t i = 0; i < 80; ++i) - { - uint32_t f = 0; - uint32_t k = 0; - - if (i < 20) - { - f = (b & c) | (~b & d); - k = 0x5A827999; - } - else if (i < 40) - { - f = b ^ c ^ d; - k = 0x6ED9EBA1; - } - else if (i < 60) - { - f = (b & c) | (b & d) | (c & d); - k = 0x8F1BBCDC; - } - else - { - f = b ^ c ^ d; - k = 0xCA62C1D6; - } - uint32_t temp = left_rotate(a, 5) + f + e + k + w[i]; - e = d; - d = c; - c = left_rotate(b, 30); - b = a; - a = temp; - } - - m_digest[0] += a; - m_digest[1] += b; - m_digest[2] += c; - m_digest[3] += d; - m_digest[4] += e; - } - - private: - digest32_t m_digest; - uint8_t m_block[64]; - size_t m_blockByteIndex; - size_t m_byteCount; -}; - -template -inline constexpr CharT empty_guid[37] = "00000000-0000-0000-0000-000000000000"; - -template <> -inline constexpr wchar_t empty_guid[37] = L"00000000-0000-0000-0000-000000000000"; - -template inline constexpr CharT guid_encoder[17] = "0123456789abcdef"; - -template <> inline constexpr wchar_t guid_encoder[17] = L"0123456789abcdef"; -} // namespace detail - -// -------------------------------------------------------------------------------------------------------------------------- -// UUID format https://tools.ietf.org/html/rfc4122 -// -------------------------------------------------------------------------------------------------------------------------- - -// -------------------------------------------------------------------------------------------------------------------------- -// Field NDR Data Type Octet # Note -// -------------------------------------------------------------------------------------------------------------------------- -// time_low unsigned long 0 - 3 The low field of the -// timestamp. time_mid unsigned short 4 - 5 The -// middle field of the timestamp. time_hi_and_version unsigned short -// 6 - 7 The high field of the timestamp multiplexed with the version -// number. clock_seq_hi_and_reserved unsigned small 8 The high -// field of the clock sequence multiplexed with the variant. clock_seq_low -// unsigned small 9 The low field of the clock sequence. node -// character 10 - 15 The spatially unique node identifier. -// -------------------------------------------------------------------------------------------------------------------------- -// 0 1 2 3 -// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | time_low | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | time_mid | time_hi_and_version | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// |clk_seq_hi_res | clk_seq_low | node (0-1) | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | node (2-5) | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -// -------------------------------------------------------------------------------------------------------------------------- -// enumerations -// -------------------------------------------------------------------------------------------------------------------------- - -// indicated by a bit pattern in octet 8, marked with N in -// xxxxxxxx-xxxx-xxxx-Nxxx-xxxxxxxxxxxx -enum class uuid_variant -{ - // NCS backward compatibility (with the obsolete Apollo Network Computing - // System 1.5 UUID format) N bit pattern: 0xxx > the first 6 octets of the - // UUID are a 48-bit timestamp (the number of 4 microsecond units of time - // since 1 Jan 1980 UTC); > the next 2 octets are reserved; > the next octet - // is the "address family"; > the final 7 octets are a 56-bit host ID in the - // form specified by the address family - ncs, - - // RFC 4122/DCE 1.1 - // N bit pattern: 10xx - // > big-endian byte order - rfc, - - // Microsoft Corporation backward compatibility - // N bit pattern: 110x - // > little endian byte order - // > formely used in the Component Object Model (COM) library - microsoft, - - // reserved for possible future definition - // N bit pattern: 111x - reserved -}; - -// indicated by a bit pattern in octet 6, marked with M in -// xxxxxxxx-xxxx-Mxxx-xxxx-xxxxxxxxxxxx -enum class uuid_version -{ - none = 0, // only possible for nil or invalid uuids - time_based = 1, // The time-based version specified in RFC 4122 - dce_security = 2, // DCE Security version, with embedded POSIX UIDs. - name_based_md5 = 3, // The name-based version specified in RFS 4122 with MD5 hashing - random_number_based = 4, // The randomly or pseudo-randomly generated - // version specified in RFS 4122 - name_based_sha1 = 5 // The name-based version specified in RFS 4122 with SHA1 hashing -}; - -// Forward declare uuid & to_string so that we can declare to_string as a friend -// later. -class uuid; -template , - class Allocator = std::allocator> -std::basic_string to_string(uuid const &id); - -// -------------------------------------------------------------------------------------------------------------------------- -// uuid class -// -------------------------------------------------------------------------------------------------------------------------- -class uuid -{ - public: - using value_type = uint8_t; - - constexpr uuid() noexcept = default; - - uuid(value_type (&arr)[16]) noexcept - { - std::copy(std::cbegin(arr), std::cend(arr), std::begin(data)); - } - - constexpr uuid(std::array const &arr) noexcept - : data{arr} - { - } - - explicit uuid(span bytes) - { - std::copy(std::cbegin(bytes), std::cend(bytes), std::begin(data)); - } - - template explicit uuid(ForwardIterator first, ForwardIterator last) - { - if (std::distance(first, last) == 16) - std::copy(first, last, std::begin(data)); - } - - [[nodiscard]] constexpr uuid_variant variant() const noexcept - { - if ((data[8] & 0x80) == 0x00) - return uuid_variant::ncs; - else if ((data[8] & 0xC0) == 0x80) - return uuid_variant::rfc; - else if ((data[8] & 0xE0) == 0xC0) - return uuid_variant::microsoft; - else - return uuid_variant::reserved; - } - - [[nodiscard]] constexpr uuid_version version() const noexcept - { - if ((data[6] & 0xF0) == 0x10) - return uuid_version::time_based; - else if ((data[6] & 0xF0) == 0x20) - return uuid_version::dce_security; - else if ((data[6] & 0xF0) == 0x30) - return uuid_version::name_based_md5; - else if ((data[6] & 0xF0) == 0x40) - return uuid_version::random_number_based; - else if ((data[6] & 0xF0) == 0x50) - return uuid_version::name_based_sha1; - else - return uuid_version::none; - } - - [[nodiscard]] constexpr bool is_nil() const noexcept - { - for (size_t i = 0; i < data.size(); ++i) - if (data[i] != 0) - return false; - return true; - } - - void swap(uuid &other) noexcept - { - data.swap(other.data); - } - - [[nodiscard]] inline span as_bytes() const - { - return span(reinterpret_cast(data.data()), 16); - } - - template - [[nodiscard]] constexpr static bool is_valid_uuid(StringType const &in_str) noexcept - { - auto str = detail::to_string_view(in_str); - bool firstDigit = true; - size_t hasBraces = 0; - size_t index = 0; - - if (str.empty()) - return false; - - if (str.front() == '{') - hasBraces = 1; - if (hasBraces && str.back() != '}') - return false; - - for (size_t i = hasBraces; i < str.size() - hasBraces; ++i) - { - if (str[i] == '-') - continue; - - if (index >= 16 || !detail::is_hex(str[i])) - { - return false; - } - - if (firstDigit) - { - firstDigit = false; - } - else - { - index++; - firstDigit = true; - } - } - - if (index < 16) - { - return false; - } - - return true; - } - - template - [[nodiscard]] constexpr static std::optional - from_string(StringType const &in_str) noexcept - { - auto str = detail::to_string_view(in_str); - bool firstDigit = true; - size_t hasBraces = 0; - size_t index = 0; - - std::array data{{0}}; - - if (str.empty()) - return {}; - - if (str.front() == '{') - hasBraces = 1; - if (hasBraces && str.back() != '}') - return {}; - - for (size_t i = hasBraces; i < str.size() - hasBraces; ++i) - { - if (str[i] == '-') - continue; - - if (index >= 16 || !detail::is_hex(str[i])) - { - return {}; - } - - if (firstDigit) - { - data[index] = static_cast(detail::hex2char(str[i]) << 4); - firstDigit = false; - } - else - { - data[index] = static_cast(data[index] | detail::hex2char(str[i])); - index++; - firstDigit = true; - } - } - - if (index < 16) - { - return {}; - } - - return uuid{data}; - } - - private: - std::array data{{0}}; - - friend bool operator==(uuid const &lhs, uuid const &rhs) noexcept; - friend bool operator<(uuid const &lhs, uuid const &rhs) noexcept; - - template - friend std::basic_ostream &operator<<(std::basic_ostream &s, - uuid const &id); - - template - friend std::basic_string to_string(uuid const &id); - - friend std::hash; -}; - -// -------------------------------------------------------------------------------------------------------------------------- -// operators and non-member functions -// -------------------------------------------------------------------------------------------------------------------------- - -[[nodiscard]] inline bool operator==(uuid const &lhs, uuid const &rhs) noexcept -{ - return lhs.data == rhs.data; -} - -[[nodiscard]] inline bool operator!=(uuid const &lhs, uuid const &rhs) noexcept -{ - return !(lhs == rhs); -} - -[[nodiscard]] inline bool operator<(uuid const &lhs, uuid const &rhs) noexcept -{ - return lhs.data < rhs.data; -} - -template -[[nodiscard]] inline std::basic_string to_string(uuid const &id) -{ - std::basic_string uustr{detail::empty_guid}; - - for (size_t i = 0, index = 0; i < 36; ++i) - { - if (i == 8 || i == 13 || i == 18 || i == 23) - { - continue; - } - uustr[i] = detail::guid_encoder[id.data[index] >> 4 & 0x0f]; - uustr[++i] = detail::guid_encoder[id.data[index] & 0x0f]; - index++; - } - - return uustr; -} - -template -std::basic_ostream &operator<<(std::basic_ostream &s, uuid const &id) -{ - s << to_string(id); - return s; -} - -inline void swap(uuids::uuid &lhs, uuids::uuid &rhs) noexcept -{ - lhs.swap(rhs); -} - -// -------------------------------------------------------------------------------------------------------------------------- -// namespace IDs that could be used for generating name-based uuids -// -------------------------------------------------------------------------------------------------------------------------- - -// Name string is a fully-qualified domain name -static uuid uuid_namespace_dns{{0x6b, - 0xa7, - 0xb8, - 0x10, - 0x9d, - 0xad, - 0x11, - 0xd1, - 0x80, - 0xb4, - 0x00, - 0xc0, - 0x4f, - 0xd4, - 0x30, - 0xc8}}; - -// Name string is a URL -static uuid uuid_namespace_url{{0x6b, - 0xa7, - 0xb8, - 0x11, - 0x9d, - 0xad, - 0x11, - 0xd1, - 0x80, - 0xb4, - 0x00, - 0xc0, - 0x4f, - 0xd4, - 0x30, - 0xc8}}; - -// Name string is an ISO OID (See https://oidref.com/, -// https://en.wikipedia.org/wiki/Object_identifier) -static uuid uuid_namespace_oid{{0x6b, - 0xa7, - 0xb8, - 0x12, - 0x9d, - 0xad, - 0x11, - 0xd1, - 0x80, - 0xb4, - 0x00, - 0xc0, - 0x4f, - 0xd4, - 0x30, - 0xc8}}; - -// Name string is an X.500 DN, in DER or a text output format (See -// https://en.wikipedia.org/wiki/X.500, -// https://en.wikipedia.org/wiki/Abstract_Syntax_Notation_One) -static uuid uuid_namespace_x500{{0x6b, - 0xa7, - 0xb8, - 0x14, - 0x9d, - 0xad, - 0x11, - 0xd1, - 0x80, - 0xb4, - 0x00, - 0xc0, - 0x4f, - 0xd4, - 0x30, - 0xc8}}; - -// -------------------------------------------------------------------------------------------------------------------------- -// uuid generators -// -------------------------------------------------------------------------------------------------------------------------- - -#ifdef UUID_SYSTEM_GENERATOR -class uuid_system_generator -{ - public: - using result_type = uuid; - - uuid operator()() - { -#ifdef _WIN32 - - GUID newId; - HRESULT hr = ::CoCreateGuid(&newId); - - if (FAILED(hr)) - { - throw std::system_error(hr, std::system_category(), "CoCreateGuid failed"); - } - - std::array bytes = {{static_cast((newId.Data1 >> 24) & 0xFF), - static_cast((newId.Data1 >> 16) & 0xFF), - static_cast((newId.Data1 >> 8) & 0xFF), - static_cast((newId.Data1) & 0xFF), - - (unsigned char)((newId.Data2 >> 8) & 0xFF), - (unsigned char)((newId.Data2) & 0xFF), - - (unsigned char)((newId.Data3 >> 8) & 0xFF), - (unsigned char)((newId.Data3) & 0xFF), - - newId.Data4[0], - newId.Data4[1], - newId.Data4[2], - newId.Data4[3], - newId.Data4[4], - newId.Data4[5], - newId.Data4[6], - newId.Data4[7]}}; - - return uuid{std::begin(bytes), std::end(bytes)}; - -#elif defined(__linux__) || defined(__unix__) - - uuid_t id; - uuid_generate(id); - - std::array bytes = {{id[0], - id[1], - id[2], - id[3], - id[4], - id[5], - id[6], - id[7], - id[8], - id[9], - id[10], - id[11], - id[12], - id[13], - id[14], - id[15]}}; - - return uuid{std::begin(bytes), std::end(bytes)}; - -#elif defined(__APPLE__) - auto newId = CFUUIDCreate(NULL); - auto bytes = CFUUIDGetUUIDBytes(newId); - CFRelease(newId); - - std::array arrbytes = {{bytes.byte0, - bytes.byte1, - bytes.byte2, - bytes.byte3, - bytes.byte4, - bytes.byte5, - bytes.byte6, - bytes.byte7, - bytes.byte8, - bytes.byte9, - bytes.byte10, - bytes.byte11, - bytes.byte12, - bytes.byte13, - bytes.byte14, - bytes.byte15}}; - return uuid{std::begin(arrbytes), std::end(arrbytes)}; -#else - return uuid{}; -#endif - } -}; -#endif - -template class basic_uuid_random_generator -{ - public: - using engine_type = UniformRandomNumberGenerator; - - explicit basic_uuid_random_generator(engine_type &gen) - : generator(&gen, [](auto) {}) - { - } - explicit basic_uuid_random_generator(engine_type *gen) - : generator(gen, [](auto) {}) - { - } - - [[nodiscard]] uuid operator()() - { - alignas(uint32_t) uint8_t bytes[16]; - for (int i = 0; i < 16; i += 4) - *reinterpret_cast(bytes + i) = distribution(*generator); - - // variant must be 10xxxxxx - bytes[8] &= 0xBF; - bytes[8] |= 0x80; - - // version must be 0100xxxx - bytes[6] &= 0x4F; - bytes[6] |= 0x40; - - return uuid{std::begin(bytes), std::end(bytes)}; - } - - private: - std::uniform_int_distribution distribution; - std::shared_ptr generator; -}; - -using uuid_random_generator = basic_uuid_random_generator; - -class uuid_name_generator -{ - public: - explicit uuid_name_generator(uuid const &namespace_uuid) noexcept - : nsuuid(namespace_uuid) - { - } - - template [[nodiscard]] uuid operator()(StringType const &name) - { - reset(); - process_characters(detail::to_string_view(name)); - return make_uuid(); - } - - private: - void reset() - { - hasher.reset(); - std::byte bytes[16]; - auto nsbytes = nsuuid.as_bytes(); - std::copy(std::cbegin(nsbytes), std::cend(nsbytes), bytes); - hasher.process_bytes(bytes, 16); - } - - template - void process_characters(std::basic_string_view const str) - { - for (uint32_t c : str) - { - hasher.process_byte(static_cast(c & 0xFF)); - if constexpr (!std::is_same_v) - { - hasher.process_byte(static_cast((c >> 8) & 0xFF)); - hasher.process_byte(static_cast((c >> 16) & 0xFF)); - hasher.process_byte(static_cast((c >> 24) & 0xFF)); - } - } - } - - [[nodiscard]] uuid make_uuid() - { - detail::sha1::digest8_t digest; - hasher.get_digest_bytes(digest); - - // variant must be 0b10xxxxxx - digest[8] &= 0xBF; - digest[8] |= 0x80; - - // version must be 0b0101xxxx - digest[6] &= 0x5F; - digest[6] |= 0x50; - - return uuid{digest, digest + 16}; - } - - private: - uuid nsuuid; - detail::sha1 hasher; -}; - -#ifdef UUID_TIME_GENERATOR -// !!! DO NOT USE THIS IN PRODUCTION -// this implementation is unreliable for good uuids -class uuid_time_generator -{ - using mac_address = std::array; - - std::optional device_address; - - [[nodiscard]] bool get_mac_address() - { - if (device_address.has_value()) - { - return true; - } - -#ifdef _WIN32 - DWORD len = 0; - auto ret = GetAdaptersInfo(nullptr, &len); - if (ret != ERROR_BUFFER_OVERFLOW) - return false; - std::vector buf(len); - auto pips = reinterpret_cast(&buf.front()); - ret = GetAdaptersInfo(pips, &len); - if (ret != ERROR_SUCCESS) - return false; - mac_address addr; - std::copy(pips->Address, pips->Address + 6, std::begin(addr)); - device_address = addr; -#endif - - return device_address.has_value(); - } - - [[nodiscard]] long long get_time_intervals() - { - auto start = std::chrono::system_clock::from_time_t(time_t(-12219292800)); - auto diff = std::chrono::system_clock::now() - start; - auto ns = std::chrono::duration_cast(diff).count(); - return ns / 100; - } - - [[nodiscard]] static unsigned short get_clock_sequence() - { - static std::mt19937 clock_gen(std::random_device{}()); - static std::uniform_int_distribution clock_dis; - static std::atomic_ushort clock_sequence = clock_dis(clock_gen); - return clock_sequence++; - } - - public: - [[nodiscard]] uuid operator()() - { - if (get_mac_address()) - { - std::array data; - - auto tm = get_time_intervals(); - - auto clock_seq = get_clock_sequence(); - - auto ptm = reinterpret_cast(&tm); - - memcpy(&data[0], ptm + 4, 4); - memcpy(&data[4], ptm + 2, 2); - memcpy(&data[6], ptm, 2); - - memcpy(&data[8], &clock_seq, 2); - - // variant must be 0b10xxxxxx - data[8] &= 0xBF; - data[8] |= 0x80; - - // version must be 0b0001xxxx - data[6] &= 0x1F; - data[6] |= 0x10; - - memcpy(&data[10], &device_address.value()[0], 6); - - return uuids::uuid{std::cbegin(data), std::cend(data)}; - } - - return {}; - } -}; -#endif -} // namespace uuids - -namespace std -{ -template <> struct hash -{ - using argument_type = uuids::uuid; - using result_type = std::size_t; - - [[nodiscard]] result_type operator()(argument_type const &uuid) const - { -#ifdef UUID_HASH_STRING_BASED - std::hash hasher; - return static_cast(hasher(uuids::to_string(uuid))); -#else - uint64_t l = - static_cast(uuid.data[0]) << 56 | static_cast(uuid.data[1]) << 48 | - static_cast(uuid.data[2]) << 40 | static_cast(uuid.data[3]) << 32 | - static_cast(uuid.data[4]) << 24 | static_cast(uuid.data[5]) << 16 | - static_cast(uuid.data[6]) << 8 | static_cast(uuid.data[7]); - uint64_t h = - static_cast(uuid.data[8]) << 56 | static_cast(uuid.data[9]) << 48 | - static_cast(uuid.data[10]) << 40 | - static_cast(uuid.data[11]) << 32 | - static_cast(uuid.data[12]) << 24 | - static_cast(uuid.data[13]) << 16 | static_cast(uuid.data[14]) << 8 | - static_cast(uuid.data[15]); - - if constexpr (sizeof(result_type) > 4) - { - return result_type(l ^ h); - } - else - { - uint64_t hash64 = l ^ h; - return result_type(uint32_t(hash64 >> 32) ^ uint32_t(hash64)); - } -#endif - } -}; -} // namespace std - -#endif /* STDUUID_H */ diff --git a/src/include/structures/skiplist/skiplist.h b/src/include/structures/skiplist/skiplist.h index e8c1043..4a52588 100644 --- a/src/include/structures/skiplist/skiplist.h +++ b/src/include/structures/skiplist/skiplist.h @@ -50,6 +50,8 @@ template class skiplist_t using pointer = Pointer; using reference = Reference; + iterator_base() = default; + explicit iterator_base(node_shared_ptr_t node) : m_node{node} { @@ -122,6 +124,16 @@ template class skiplist_t return const_iterator(nullptr); } + [[nodiscard]] auto begin() const -> const_iterator + { + return const_iterator(m_head->forward[0]); + } + + [[nodiscard]] auto end() const -> const_iterator + { + return const_iterator(nullptr); + } + auto find(const typename record_gt::key_t &key) const noexcept -> std::optional { node_shared_ptr_t current{m_head}; diff --git a/src/include/wal/wal.h b/src/include/wal/wal.h index 33a4fa9..dbb04a3 100644 --- a/src/include/wal/wal.h +++ b/src/include/wal/wal.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -74,7 +75,7 @@ inline auto wal_t::read(size_t index) const -> std::optional inline auto wal_t::add(entry_type_t entry) -> bool { - return std::visit([&](auto &storage) { return storage.append(entry); }, m_log); + return std::visit([&](auto &storage) -> auto { return storage.append(entry); }, m_log); } template @@ -145,11 +146,14 @@ class wal_builder_t final auto set_file_path(fs::path_t path) -> wal_builder_t &; template - [[nodiscard]] auto build(log_storage_type_k type) noexcept -> std::optional> + [[nodiscard]] auto build(log_storage_type_k type) noexcept + -> std::optional> { if (type == log_storage_type_k::in_memory_k) { - return wal_t{variant_t{in_memory_log_storage_t{}}}; + return std::make_shared>( + variant_t{in_memory_log_storage_t{}} + ); } if (type == log_storage_type_k::file_based_persistent_k) @@ -166,7 +170,9 @@ class wal_builder_t final .build(); storage.has_value()) { - return wal_t{variant_t{std::move(storage.value())}}; + return std::make_shared>( + variant_t{std::move(storage.value())} + ); } return std::nullopt; diff --git a/src/main.cpp b/src/main.cpp index 86e1756..e3ef409 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -191,7 +191,6 @@ auto main(int argc, char *argv[]) -> int spdlog::error("Unable to build WAL"); return EXIT_FAILURE; } - auto pWAL = wal::make_shared(std::move(maybeWal.value())); // Build consensus module consensus::node_config_t nodeConfig{ @@ -205,7 +204,9 @@ auto main(int argc, char *argv[]) -> int grpcBuilder.AddListeningPort(nodeConfig.m_ip, grpc::InsecureServerCredentials()); std::shared_ptr pConsensusModule{nullptr}; - if (auto maybeConsensusModule{maybe_create_consensus_module(pDbConfig, nodeConfig, pWAL)}; + if (auto maybeConsensusModule{ + maybe_create_consensus_module(pDbConfig, nodeConfig, maybeWal.value()) + }; maybeConsensusModule.has_value()) { pConsensusModule = std::move(maybeConsensusModule.value()); @@ -231,7 +232,8 @@ auto main(int argc, char *argv[]) -> int }; // Build LSMTree - auto pLSMTree = structures::lsmtree::lsmtree_builder_t{}.build(pDbConfig, pManifest, pWAL); + auto pLSMTree = + structures::lsmtree::lsmtree_builder_t{}.build(pDbConfig, pManifest, maybeWal.value()); if (!pLSMTree) { spdlog::error("Main: Unable to build LSMTree"); diff --git a/src/structures/lsmtree/CMakeLists.txt b/src/structures/lsmtree/CMakeLists.txt index b00616f..772853f 100644 --- a/src/structures/lsmtree/CMakeLists.txt +++ b/src/structures/lsmtree/CMakeLists.txt @@ -27,6 +27,8 @@ target_link_libraries( WAL Raft RaftProtoObjects + Microsoft.GSL::GSL + libuuid::libuuid ) add_executable(LSMTreeTest lsmtree_test.cpp) @@ -34,7 +36,7 @@ set_target_properties(LSMTreeTest PROPERTIES CXX_STANDARD 23) target_link_libraries( LSMTreeTest PRIVATE - Catch2::Catch2WithMain + gtest::gtest spdlog::spdlog fmt::fmt LSMTree @@ -48,6 +50,5 @@ target_link_libraries( Concurrency ) -# Register Catch2 tests with CTest -include(Catch) -catch_discover_tests(LSMTreeTest) +include(GoogleTest) +gtest_discover_tests(LSMTreeTest) diff --git a/src/structures/lsmtree/levels/level.cpp b/src/structures/lsmtree/levels/level.cpp index 5907f43..c83dc67 100644 --- a/src/structures/lsmtree/levels/level.cpp +++ b/src/structures/lsmtree/levels/level.cpp @@ -127,9 +127,9 @@ auto level_t::compact() const noexcept -> segments::regular_segment::shared_ptr_ }; if (used < threshold) { - spdlog::info( - "Skipping level={} compaction. Used={}, threshold={}", index(), used, threshold - ); + // spdlog::info( + // "Skipping level={} compaction. Used={}, threshold={}", index(), used, threshold + // ); return nullptr; } @@ -196,7 +196,7 @@ void level_t::merge(const segments::regular_segment::shared_ptr_t &pSegment) noe // Segments overlapping with input memtable auto overlappingSegmentsView = m_storage | std::views::filter( - [pSegment](const auto ¤tSegment) + [pSegment](const auto ¤tSegment) -> auto { return !( currentSegment->max().value() < pSegment->min().value() || @@ -272,7 +272,7 @@ void level_t::merge(const segments::regular_segment::shared_ptr_t &pSegment) noe // Delete overlapping segments after the merging process is complete std::ranges::for_each( - overlappingSegmentsView, [this](auto pSegment) { this->purge(pSegment); } + overlappingSegmentsView, [this](auto pSegment) -> auto { this->purge(std::move(pSegment)); } ); // Flush new segments @@ -292,14 +292,14 @@ void level_t::purge() noexcept { pSegment->remove_from_disk(); - auto ok{m_manifest->add( + auto success{m_manifest->add( db::manifest::manifest_t::segment_record_t{ .op = segment_operation_k::remove_segment_k, .name = pSegment->get_name(), .level = idx } )}; - ASSERT(ok); + ASSERT(success); } m_storage.clear(); } @@ -318,20 +318,21 @@ auto level_t::purge(const segments::types::name_t &segmentName) noexcept -> void } } -void level_t::purge(const segments::regular_segment::shared_ptr_t &pSegment) noexcept +void level_t::purge(segments::regular_segment::shared_ptr_t pSegment) noexcept { absl::WriterMutexLock lock{&m_mutex}; assert(pSegment); spdlog::debug("Removing segment {} from level {}", pSegment->get_name(), index()); - ASSERT(m_manifest->add( + auto success{m_manifest->add( db::manifest::manifest_t::segment_record_t{ .op = segment_operation_k::remove_segment_k, .name = pSegment->get_name(), .level = index() } - )); + )}; + ASSERT(success); pSegment->remove_from_disk(); m_storage.remove(pSegment); diff --git a/src/structures/lsmtree/levels/levels.cpp b/src/structures/lsmtree/levels/levels.cpp index 487babb..8e2b3b4 100644 --- a/src/structures/lsmtree/levels/levels.cpp +++ b/src/structures/lsmtree/levels/levels.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -31,22 +32,6 @@ levels_t::levels_t(config::shared_ptr_t pConfig, db::manifest::shared_ptr_t pMan } } -levels_t::levels_t(levels_t &&other) noexcept -{ - absl::WriterMutexLock otherLock{&other.m_mutex}; - move_from(std::move(other)); -} - -auto levels_t::operator=(levels_t &&other) noexcept -> levels_t & -{ - if (this != &other) - { - concurrency::absl_dual_mutex_lock_guard lock{m_mutex, other.m_mutex}; - move_from(std::move(other)); - } - return *this; -} - levels_t::~levels_t() noexcept { m_compaction_thread.request_stop(); @@ -54,6 +39,7 @@ levels_t::~levels_t() noexcept { spdlog::debug("Waiting for compaction thread to finish"); m_compaction_thread.join(); + spdlog::debug("Waiting for compaction thread to finish... Done"); } else { @@ -232,34 +218,20 @@ void levels_t::compaction_task(std::stop_token stoken) noexcept { while (!stoken.stop_requested()) { - m_level0_segment_flushed_notification.WaitForNotification(); - - auto ok{compact()}; - if (!ok) + if (m_level0_segment_flushed_notification.WaitForNotificationWithTimeout( + gWaitForSegmentFlushNotificationTimeout + )) { - spdlog::error("Compaction failed."); - } - // ASSERT(ok); - } -} -void levels_t::move_from(levels_t &&other) noexcept -{ - m_pConfig = std::move(other.m_pConfig); - m_pManifest = std::move(other.m_pManifest); - m_levels = std::move(other.m_levels); - - other.m_compaction_thread.request_stop(); - if (other.m_compaction_thread.joinable()) - { - spdlog::debug("Waiting for compaction thread to finish"); - other.m_compaction_thread.join(); - } - else - { - spdlog::debug("Compaction thread is not joinable, skipping join"); + auto ok{compact()}; + if (!ok) + { + // spdlog::error("Compaction failed."); + } + // TODO(lnikon): compact returns null to indicate that compaction didn't happened or on + // error, which causes false-positives on this ASSERT ASSERT(ok); + } } - m_compaction_thread = std::jthread([this](std::stop_token stoken) { compaction_task(stoken); }); } } // namespace structures::lsmtree::levels diff --git a/src/structures/lsmtree/lsmtree.cpp b/src/structures/lsmtree/lsmtree.cpp index 12ff417..81b0582 100644 --- a/src/structures/lsmtree/lsmtree.cpp +++ b/src/structures/lsmtree/lsmtree.cpp @@ -1,3 +1,6 @@ +#include +#include +#include #include #include #include @@ -34,22 +37,6 @@ lsmtree_t::lsmtree_t( { } -lsmtree_t::lsmtree_t(lsmtree_t &&other) noexcept -{ - absl::WriterMutexLock otherLock{&other.m_mutex}; - move_from(std::move(other)); -} - -auto lsmtree_t::operator=(lsmtree_t &&other) noexcept -> lsmtree_t & -{ - if (this != &other) - { - concurrency::absl_dual_mutex_lock_guard lock{m_mutex, other.m_mutex}; - move_from(std::move(other)); - } - return *this; -} - lsmtree_t::~lsmtree_t() noexcept { m_flushing_thread.request_stop(); @@ -57,6 +44,7 @@ lsmtree_t::~lsmtree_t() noexcept { spdlog::debug("Waiting for flushing thread to finish"); m_flushing_thread.join(); + spdlog::debug("Waiting for flushing thread to finish... Done"); } else { @@ -64,7 +52,7 @@ lsmtree_t::~lsmtree_t() noexcept } } -auto lsmtree_t::put(record_t record) noexcept -> lsmtree_status_k +auto lsmtree_t::put(record_t record) noexcept -> std::expected { assert(m_pConfig); @@ -79,10 +67,10 @@ auto lsmtree_t::put(record_t record) noexcept -> lsmtree_status_k { m_flushing_queue.push(std::move(m_table.value())); m_table = std::make_optional(); - return lsmtree_status_k::memtable_reset_k; + return lsmtree_success_k::memtable_reset_k; } - return lsmtree_status_k::ok_k; + return lsmtree_success_k::ok_k; } auto lsmtree_t::get(const key_t &key) noexcept -> std::optional @@ -104,14 +92,36 @@ auto lsmtree_t::get(const key_t &key) noexcept -> std::optional // Lookup in immutable memtables if (!result.has_value()) { + spdlog::info( + "cant find record {} in memtable... searching in flush queues size={}", + key.m_key, + m_flushing_queue.size() + ); result = m_flushing_queue.find(key); + if (result.has_value()) + { + spdlog::info("found record {} in flushing queues", key.m_key); + } + } + else + { + spdlog::info("found record {} in memtable", key.m_key); } // If key isn't in in-memory table, then it probably was flushed. // Lookup for the key in on-disk segments if (!result.has_value()) { + spdlog::info("cant find record {} in flush queues... searching in levels", key.m_key); result = m_pLevels->record(key); + if (!result.has_value()) + { + spdlog::info("cant find record {} in levels :(((", key.m_key); + } + else + { + spdlog::info("found record {} in levels", key.m_key); + } } return result; @@ -134,65 +144,49 @@ void lsmtree_t::memtable_flush_task(std::stop_token stoken) noexcept m_flushing_queue.size() ); - auto memtables = m_flushing_queue.pop_all(); - while (!memtables.empty()) + // auto memtables = m_flushing_queue.pop_all(); + auto memtables = m_flushing_queue.reserve(); + if (!memtables.has_value()) { - auto memtable = memtables.front(); - memtables.pop_front(); + return; + } + for (auto &memtable : memtables.value()) + { // TODO: Assert will crash the program, maybe we // should return an error code? absl::WriterMutexLock lock{&m_mutex}; bool ok{m_pLevels->flush_to_level0(std::move(memtable))}; assert(ok); } + + m_flushing_queue.consume(); return; } - if (std::optional memtable = m_flushing_queue.pop(); - memtable.has_value() && !memtable->empty()) - { - spdlog::debug( - "Flushing memtable to level0. " - "memtable.size={}, flushing_queue.size={}", - memtable.value().size(), - m_flushing_queue.size() - ); - - // TODO: Assert will crash the program, maybe we should - // return an error code? - absl::WriterMutexLock lock{&m_mutex}; - bool ok{m_pLevels->flush_to_level0(std::move(memtable.value()))}; - assert(ok); - } + // auto memtables = m_flushing_queue.reserve(); + // if (!memtables.has_value()) + // { + // absl::SleepFor(absl::Milliseconds(50)); + // continue; + // } + // + // auto idx{1}; + // for (auto &memtable : memtables.value()) + // { + // spdlog::debug("LSMTree: Flushing {}/{} memtable into disk", idx++, + // memtables->size()); + // + // // TODO: Assert will crash the program, maybe we should + // // return an error code? + // absl::WriterMutexLock lock{&m_mutex}; + // bool ok{m_pLevels->flush_to_level0(std::move(memtable))}; + // assert(ok); + // } + // m_flushing_queue.consume(); } } -void lsmtree_t::move_from(lsmtree_t &&other) noexcept -{ - m_pConfig = std::move(other.m_pConfig); - m_table = std::move(other.m_table); - m_pManifest = std::move(other.m_pManifest); - m_pLevels = std::move(other.m_pLevels); - - other.m_flushing_thread.request_stop(); - if (other.m_flushing_thread.joinable()) - { - spdlog::debug("Waiting for flushing thread to finish"); - other.m_flushing_thread.join(); - } - else - { - spdlog::debug("Flushing thread is not joinable, skipping join"); - } - - // Start a new flushing thread for the moved object - m_flushing_queue = std::move(other.m_flushing_queue); - m_flushing_thread = - std::jthread([this](std::stop_token stoken) { memtable_flush_task(stoken); }); - spdlog::debug("Flushing thread started for moved lsmtree_t object"); -} - auto lsmtree_builder_t::build( config::shared_ptr_t pConfig, db::manifest::shared_ptr_t pManifest, wal_t pWal ) const -> std::shared_ptr @@ -275,7 +269,7 @@ auto lsmtree_builder_t::build_levels_from_manifest( for (const auto &record : records) { std::visit( - [&](auto record) + [&](auto record) -> auto { using T = std::decay_t; if constexpr (std::is_same_v) diff --git a/src/structures/lsmtree/lsmtree_test.cpp b/src/structures/lsmtree/lsmtree_test.cpp index b708191..b893a09 100644 --- a/src/structures/lsmtree/lsmtree_test.cpp +++ b/src/structures/lsmtree/lsmtree_test.cpp @@ -1,53 +1,21 @@ -#include - -#include -#include -#include - #include #include -#include - -#include -#include - -class RemoveArtefactsListener : public Catch::EventListenerBase -{ - public: - using Catch::EventListenerBase::EventListenerBase; - - void prepare() - { - std::filesystem::remove_all("segments"); - std::filesystem::remove("segments"); - std::filesystem::create_directory("segments"); - std::filesystem::remove("manifest"); - std::filesystem::remove("wal"); - } - - /// cppcheck-suppress unusedFunction - void testCaseStarting(Catch::TestCaseInfo const &testInfo) override - { - (void)testInfo; - prepare(); - } +#include - /// cppcheck-suppress unusedFunction - void testCaseEnded(Catch::TestCaseStats const &testCaseStats) override - { - (void)testCaseStats; - prepare(); - } -}; +#include +#include +#include -CATCH_REGISTER_LISTENER(RemoveArtefactsListener) +#include "structures/lsmtree/lsmtree.h" +#include "config/config.h" namespace { template -auto generateRandomNumber(const TNumber min = std::numeric_limits::min(), - const TNumber max = std::numeric_limits::max()) noexcept - -> TNumber +auto generateRandomNumber( + const TNumber min = std::numeric_limits::min(), + const TNumber max = std::numeric_limits::max() +) noexcept -> TNumber { std::mt19937 rng{std::random_device{}()}; if constexpr (std::is_same_v) @@ -96,60 +64,80 @@ auto generateRandomStringPairVector(const std::size_t length) noexcept result.reserve(length); for (std::string::size_type size = 0; size < length; size++) { - result.emplace_back(generateRandomString(generateRandomNumber(64, 64)), - generateRandomString(generateRandomNumber(64, 64))); + result.emplace_back( + generateRandomString(generateRandomNumber(64, 64)), + generateRandomString(generateRandomNumber(64, 64)) + ); } return result; } -inline constexpr std::string_view componentName = "[LSMTree]"; +auto generateConfig( + std::uint64_t memtableThreshold, + std::uint64_t levelZeroThreshold, + std::int64_t levelNonZeroThreshold +) +{ + auto pResult{config::make_shared()}; + pResult->LSMTreeConfig.DiskFlushThresholdSize = memtableThreshold; + pResult->LSMTreeConfig.LevelZeroCompactionThreshold = levelZeroThreshold; + pResult->LSMTreeConfig.LevelNonZeroCompactionThreshold = levelNonZeroThreshold; + return pResult; +} + +using record_t = structures::memtable::memtable_t::record_t; + } // namespace -TEST_CASE("Flush regular segment", std::string(componentName)) +namespace raft::v1 +{ +template auto operator>>(TStream &stream, LogEntry &record) -> TStream & +{ + record.ParseFromString(stream.str()); + return stream; +} +} // namespace raft::v1 + +TEST(LSMTreeTest, FlushRegularSegment) { using namespace structures; - // Disable logging to save on text execution time - spdlog::set_level(spdlog::level::info); + spdlog::set_level(spdlog::level::debug); + + const auto &randomKeys = generateRandomStringPairVector(16); - auto randomKeys = generateRandomStringPairVector(16); + auto pConfig{generateConfig(16, 64, 128)}; - SECTION("Put and Get") + auto pManifest{db::manifest::make_shared( + pConfig->manifest_path(), + wal::wal_builder_t{} + .build(wal::log_storage_type_k::in_memory_k) + .value() + )}; + auto pWAL{ + wal::wal_builder_t{}.build(wal::log_storage_type_k::in_memory_k).value() + }; + auto pLsm{structures::lsmtree::lsmtree_builder_t{}.build(pConfig, pManifest, pWAL)}; + + for (const auto &kv : randomKeys) { - auto pConfig{config::make_shared()}; - pConfig->LSMTreeConfig.DiskFlushThresholdSize = 1; // 64mb = 64000000 - - auto manifest{db::manifest::make_shared(pConfig)}; - auto wal{wal::make_shared("wal")}; - auto lsmTree{structures::lsmtree::lsmtree_t{pConfig, manifest, wal}}; - lsmtree::lsmtree_t lsmt(pConfig, manifest, wal); - for (const auto &kv : randomKeys) - { - lsmt.put(lsmtree::key_t{kv.first}, lsmtree::value_t{kv.second}); - } - - for (const auto &kv : randomKeys) - { - REQUIRE(lsmt.get(lsmtree::key_t{kv.first}).value().m_key == lsmtree::key_t{kv.first}); - REQUIRE(lsmt.get(lsmtree::key_t{kv.first}).value().m_value == - lsmtree::value_t{kv.second}); - } + auto rec{record_t{}}; + rec.m_key.m_key = kv.first; + rec.m_value.m_value = kv.second; + auto status{pLsm->put(rec)}; + EXPECT_TRUE(status.has_value()); } - // SECTION("Flush segment when memtable is full") - // { - // config::shared_ptr_t pConfig{config::make_shared()}; - // pConfig->LSMTreeConfig.DiskFlushThresholdSize = 128; - // - // auto manifest{db::manifest::make_shared(pConfig)}; - // auto wal{wal::make_shared("wal")}; - // lsmtree::lsmtree_t lsmt(pConfig, manifest, wal); - // for (const auto &kv : randomKeys) - // { - // lsmt.put(lsmtree::key_t{kv.first}, lsmtree::value_t{kv.second}); - // } - // - // REQUIRE(std::filesystem::exists("segments")); - // REQUIRE(!std::filesystem::is_empty("segments")); - // } + for (const auto &kv : randomKeys) + { + auto rec{record_t{}}; + rec.m_key.m_key = kv.first; + rec.m_value.m_value = kv.second; + + auto expected = pLsm->get(rec.m_key); + + EXPECT_TRUE(expected.has_value()); + EXPECT_EQ(expected.value().m_key, rec.m_key); + EXPECT_EQ(expected.value().m_value, rec.m_value); + } } diff --git a/src/structures/lsmtree/segments/gsl/gsl b/src/structures/lsmtree/segments/gsl/gsl deleted file mode 100644 index b1f23c2..0000000 --- a/src/structures/lsmtree/segments/gsl/gsl +++ /dev/null @@ -1,29 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -// Copyright (c) 2015 Microsoft Corporation. All rights reserved. -// -// This code is licensed under the MIT License (MIT). -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef GSL_GSL_H -#define GSL_GSL_H - -#include "gsl_algorithm" // copy -#include "gsl_assert" // Ensures/Expects -#include "gsl_byte" // byte -#include "gsl_util" // finally()/narrow()/narrow_cast()... -#include "multi_span" // multi_span, strided_span... -#include "pointers" // owner, not_null -#include "span" // span -#include "string_span" // zstring, string_span, zstring_builder... - -#endif // GSL_GSL_H diff --git a/src/structures/lsmtree/segments/gsl/gsl_algorithm b/src/structures/lsmtree/segments/gsl/gsl_algorithm deleted file mode 100644 index c5c0f8f..0000000 --- a/src/structures/lsmtree/segments/gsl/gsl_algorithm +++ /dev/null @@ -1,63 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -// Copyright (c) 2015 Microsoft Corporation. All rights reserved. -// -// This code is licensed under the MIT License (MIT). -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef GSL_ALGORITHM_H -#define GSL_ALGORITHM_H - -#include "gsl_assert" // for Expects -#include "span" // for dynamic_extent, span - -#include // for copy_n -#include // for ptrdiff_t -#include // for is_assignable - -#ifdef _MSC_VER -#pragma warning(push) - -// turn off some warnings that are noisy about our Expects statements -#pragma warning(disable : 4127) // conditional expression is constant -#pragma warning(disable : 4996) // unsafe use of std::copy_n - -// blanket turn off warnings from CppCoreCheck for now -// so people aren't annoyed by them when running the tool. -// more targeted suppressions will be added in a future update to the GSL -#pragma warning(disable : 26481 26482 26483 26485 26490 26491 26492 26493 26495) -#endif // _MSC_VER - -namespace gsl -{ - -template -void copy(span src, span dest) -{ - static_assert(std::is_assignable::value, - "Elements of source span can not be assigned to elements of destination span"); - static_assert(SrcExtent == dynamic_extent || DestExtent == dynamic_extent || - (SrcExtent <= DestExtent), - "Source range is longer than target range"); - - Expects(dest.size() >= src.size()); - std::copy_n(src.data(), src.size(), dest.data()); -} - -} // namespace gsl - -#ifdef _MSC_VER -#pragma warning(pop) -#endif // _MSC_VER - -#endif // GSL_ALGORITHM_H diff --git a/src/structures/lsmtree/segments/gsl/gsl_assert b/src/structures/lsmtree/segments/gsl/gsl_assert deleted file mode 100644 index 131fa8b..0000000 --- a/src/structures/lsmtree/segments/gsl/gsl_assert +++ /dev/null @@ -1,145 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -// Copyright (c) 2015 Microsoft Corporation. All rights reserved. -// -// This code is licensed under the MIT License (MIT). -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef GSL_CONTRACTS_H -#define GSL_CONTRACTS_H - -#include -#include // for logic_error - -// -// Temporary until MSVC STL supports no-exceptions mode. -// Currently terminate is a no-op in this mode, so we add termination behavior back -// -#if defined(_MSC_VER) && defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS -#define GSL_MSVC_USE_STL_NOEXCEPTION_WORKAROUND -#endif - -// -// There are three configuration options for this GSL implementation's behavior -// when pre/post conditions on the GSL types are violated: -// -// 1. GSL_TERMINATE_ON_CONTRACT_VIOLATION: std::terminate will be called (default) -// 2. GSL_THROW_ON_CONTRACT_VIOLATION: a gsl::fail_fast exception will be thrown -// 3. GSL_UNENFORCED_ON_CONTRACT_VIOLATION: nothing happens -// -#if !(defined(GSL_THROW_ON_CONTRACT_VIOLATION) || defined(GSL_TERMINATE_ON_CONTRACT_VIOLATION) || \ - defined(GSL_UNENFORCED_ON_CONTRACT_VIOLATION)) -#define GSL_TERMINATE_ON_CONTRACT_VIOLATION -#endif - -#define GSL_STRINGIFY_DETAIL(x) #x -#define GSL_STRINGIFY(x) GSL_STRINGIFY_DETAIL(x) - -#if defined(__clang__) || defined(__GNUC__) -#define GSL_LIKELY(x) __builtin_expect(!!(x), 1) -#define GSL_UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define GSL_LIKELY(x) (!!(x)) -#define GSL_UNLIKELY(x) (!!(x)) -#endif - -// -// GSL_ASSUME(cond) -// -// Tell the optimizer that the predicate cond must hold. It is unspecified -// whether or not cond is actually evaluated. -// -#ifdef _MSC_VER -#define GSL_ASSUME(cond) __assume(cond) -#elif defined(__GNUC__) -#define GSL_ASSUME(cond) ((cond) ? static_cast(0) : __builtin_unreachable()) -#else -#define GSL_ASSUME(cond) static_cast((cond) ? 0 : 0) -#endif - -// -// GSL.assert: assertions -// - -namespace gsl -{ -struct fail_fast : public std::logic_error -{ - explicit fail_fast(char const* const message) : std::logic_error(message) {} -}; - -namespace details -{ -#if defined(GSL_MSVC_USE_STL_NOEXCEPTION_WORKAROUND) - - typedef void (__cdecl *terminate_handler)(); - - inline gsl::details::terminate_handler& get_terminate_handler() noexcept - { - static terminate_handler handler = &abort; - return handler; - } - -#endif - - [[noreturn]] inline void terminate() noexcept - { -#if defined(GSL_MSVC_USE_STL_NOEXCEPTION_WORKAROUND) - (*gsl::details::get_terminate_handler())(); -#else - std::terminate(); -#endif - } - -#if defined(GSL_TERMINATE_ON_CONTRACT_VIOLATION) - - template - [[noreturn]] void throw_exception(Exception&&) - { - gsl::details::terminate(); - } - -#else - - template - [[noreturn]] void throw_exception(Exception&& exception) - { - throw std::forward(exception); - } - -#endif - -} // namespace details -} // namespace gsl - -#if defined(GSL_THROW_ON_CONTRACT_VIOLATION) - -#define GSL_CONTRACT_CHECK(type, cond) \ - (GSL_LIKELY(cond) ? static_cast(0) \ - : gsl::details::throw_exception(gsl::fail_fast( \ - "GSL: " type " failure at " __FILE__ ": " GSL_STRINGIFY(__LINE__)))) - -#elif defined(GSL_TERMINATE_ON_CONTRACT_VIOLATION) - -#define GSL_CONTRACT_CHECK(type, cond) \ - (GSL_LIKELY(cond) ? static_cast(0) : gsl::details::terminate()) - -#elif defined(GSL_UNENFORCED_ON_CONTRACT_VIOLATION) - -#define GSL_CONTRACT_CHECK(type, cond) GSL_ASSUME(cond) - -#endif - -#define Expects(cond) GSL_CONTRACT_CHECK("Precondition", cond) -#define Ensures(cond) GSL_CONTRACT_CHECK("Postcondition", cond) - -#endif // GSL_CONTRACTS_H diff --git a/src/structures/lsmtree/segments/gsl/gsl_byte b/src/structures/lsmtree/segments/gsl/gsl_byte deleted file mode 100644 index e861173..0000000 --- a/src/structures/lsmtree/segments/gsl/gsl_byte +++ /dev/null @@ -1,181 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -// Copyright (c) 2015 Microsoft Corporation. All rights reserved. -// -// This code is licensed under the MIT License (MIT). -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef GSL_BYTE_H -#define GSL_BYTE_H - -#include - -#ifdef _MSC_VER - -#pragma warning(push) - -// don't warn about function style casts in byte related operators -#pragma warning(disable : 26493) - -#ifndef GSL_USE_STD_BYTE -// this tests if we are under MSVC and the standard lib has std::byte and it is enabled -#if defined(_HAS_STD_BYTE) && _HAS_STD_BYTE - -#define GSL_USE_STD_BYTE 1 - -#else // defined(_HAS_STD_BYTE) && _HAS_STD_BYTE - -#define GSL_USE_STD_BYTE 0 - -#endif // defined(_HAS_STD_BYTE) && _HAS_STD_BYTE -#endif // GSL_USE_STD_BYTE - -#else // _MSC_VER - -#ifndef GSL_USE_STD_BYTE -// this tests if we are under GCC or Clang with enough -std:c++1z power to get us std::byte -#if defined(__cplusplus) && (__cplusplus >= 201703L) - -#define GSL_USE_STD_BYTE 1 -#include - -#else // defined(__cplusplus) && (__cplusplus >= 201703L) - -#define GSL_USE_STD_BYTE 0 - -#endif //defined(__cplusplus) && (__cplusplus >= 201703L) -#endif // GSL_USE_STD_BYTE - -#endif // _MSC_VER - -// Use __may_alias__ attribute on gcc and clang -#if defined __clang__ || (__GNUC__ > 5) -#define byte_may_alias __attribute__((__may_alias__)) -#else // defined __clang__ || defined __GNUC__ -#define byte_may_alias -#endif // defined __clang__ || defined __GNUC__ - -namespace gsl -{ -#if GSL_USE_STD_BYTE - - -using std::byte; -using std::to_integer; - -#else // GSL_USE_STD_BYTE - -// This is a simple definition for now that allows -// use of byte within span<> to be standards-compliant -enum class byte_may_alias byte : unsigned char -{ -}; - -template ::value>> -constexpr byte& operator<<=(byte& b, IntegerType shift) noexcept -{ - return b = byte(static_cast(b) << shift); -} - -template ::value>> -constexpr byte operator<<(byte b, IntegerType shift) noexcept -{ - return byte(static_cast(b) << shift); -} - -template ::value>> -constexpr byte& operator>>=(byte& b, IntegerType shift) noexcept -{ - return b = byte(static_cast(b) >> shift); -} - -template ::value>> -constexpr byte operator>>(byte b, IntegerType shift) noexcept -{ - return byte(static_cast(b) >> shift); -} - -constexpr byte& operator|=(byte& l, byte r) noexcept -{ - return l = byte(static_cast(l) | static_cast(r)); -} - -constexpr byte operator|(byte l, byte r) noexcept -{ - return byte(static_cast(l) | static_cast(r)); -} - -constexpr byte& operator&=(byte& l, byte r) noexcept -{ - return l = byte(static_cast(l) & static_cast(r)); -} - -constexpr byte operator&(byte l, byte r) noexcept -{ - return byte(static_cast(l) & static_cast(r)); -} - -constexpr byte& operator^=(byte& l, byte r) noexcept -{ - return l = byte(static_cast(l) ^ static_cast(r)); -} - -constexpr byte operator^(byte l, byte r) noexcept -{ - return byte(static_cast(l) ^ static_cast(r)); -} - -constexpr byte operator~(byte b) noexcept { return byte(~static_cast(b)); } - -template ::value>> -constexpr IntegerType to_integer(byte b) noexcept -{ - return static_cast(b); -} - -#endif // GSL_USE_STD_BYTE - -template -constexpr byte to_byte_impl(T t) noexcept -{ - static_assert( - E, "gsl::to_byte(t) must be provided an unsigned char, otherwise data loss may occur. " - "If you are calling to_byte with an integer contant use: gsl::to_byte() version."); - return static_cast(t); -} -template <> -constexpr byte to_byte_impl(unsigned char t) noexcept -{ - return byte(t); -} - -template -constexpr byte to_byte(T t) noexcept -{ - return to_byte_impl::value, T>(t); -} - -template -constexpr byte to_byte() noexcept -{ - static_assert(I >= 0 && I <= 255, - "gsl::byte only has 8 bits of storage, values must be in range 0-255"); - return static_cast(I); -} - -} // namespace gsl - -#ifdef _MSC_VER -#pragma warning(pop) -#endif // _MSC_VER - -#endif // GSL_BYTE_H diff --git a/src/structures/lsmtree/segments/gsl/gsl_util b/src/structures/lsmtree/segments/gsl/gsl_util deleted file mode 100644 index 373ee79..0000000 --- a/src/structures/lsmtree/segments/gsl/gsl_util +++ /dev/null @@ -1,158 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -// Copyright (c) 2015 Microsoft Corporation. All rights reserved. -// -// This code is licensed under the MIT License (MIT). -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef GSL_UTIL_H -#define GSL_UTIL_H - -#include "gsl_assert" // for Expects - -#include -#include // for ptrdiff_t, size_t -#include // for exception -#include // for initializer_list -#include // for is_signed, integral_constant -#include // for forward - -#if defined(_MSC_VER) - -#pragma warning(push) -#pragma warning(disable : 4127) // conditional expression is constant - -#if _MSC_VER < 1910 -#pragma push_macro("constexpr") -#define constexpr /*constexpr*/ -#endif // _MSC_VER < 1910 -#endif // _MSC_VER - -namespace gsl -{ -// -// GSL.util: utilities -// - -// index type for all container indexes/subscripts/sizes -using index = std::ptrdiff_t; - -// final_action allows you to ensure something gets run at the end of a scope -template -class final_action -{ -public: - explicit final_action(F f) noexcept : f_(std::move(f)) {} - - final_action(final_action&& other) noexcept : f_(std::move(other.f_)), invoke_(other.invoke_) - { - other.invoke_ = false; - } - - final_action(const final_action&) = delete; - final_action& operator=(const final_action&) = delete; - final_action& operator=(final_action&&) = delete; - - ~final_action() noexcept - { - if (invoke_) f_(); - } - -private: - F f_; - bool invoke_ {true}; -}; - -// finally() - convenience function to generate a final_action -template - -final_action finally(const F& f) noexcept -{ - return final_action(f); -} - -template -final_action finally(F&& f) noexcept -{ - return final_action(std::forward(f)); -} - -// narrow_cast(): a searchable way to do narrowing casts of values -template -constexpr T narrow_cast(U&& u) noexcept -{ - return static_cast(std::forward(u)); -} - -struct narrowing_error : public std::exception -{ -}; - -namespace details -{ - template - struct is_same_signedness - : public std::integral_constant::value == std::is_signed::value> - { - }; -} - -// narrow() : a checked version of narrow_cast() that throws if the cast changed the value -template -T narrow(U u) -{ - T t = narrow_cast(u); - if (static_cast(t) != u) gsl::details::throw_exception(narrowing_error()); - if (!details::is_same_signedness::value && ((t < T{}) != (u < U{}))) - gsl::details::throw_exception(narrowing_error()); - return t; -} - -// -// at() - Bounds-checked way of accessing builtin arrays, std::array, std::vector -// -template -constexpr T& at(T (&arr)[N], const index i) -{ - Expects(i >= 0 && i < narrow_cast(N)); - return arr[static_cast(i)]; -} - -template -constexpr auto at(Cont& cont, const index i) -> decltype(cont[cont.size()]) -{ - Expects(i >= 0 && i < narrow_cast(cont.size())); - using size_type = decltype(cont.size()); - return cont[static_cast(i)]; -} - -template -constexpr T at(const std::initializer_list cont, const index i) -{ - Expects(i >= 0 && i < narrow_cast(cont.size())); - return *(cont.begin() + i); -} - -} // namespace gsl - -#if defined(_MSC_VER) -#if _MSC_VER < 1910 -#undef constexpr -#pragma pop_macro("constexpr") - -#endif // _MSC_VER < 1910 - -#pragma warning(pop) - -#endif // _MSC_VER - -#endif // GSL_UTIL_H diff --git a/src/structures/lsmtree/segments/gsl/multi_span b/src/structures/lsmtree/segments/gsl/multi_span deleted file mode 100644 index 316da1f..0000000 --- a/src/structures/lsmtree/segments/gsl/multi_span +++ /dev/null @@ -1,2242 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -// Copyright (c) 2015 Microsoft Corporation. All rights reserved. -// -// This code is licensed under the MIT License (MIT). -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef GSL_MULTI_SPAN_H -#define GSL_MULTI_SPAN_H - -#include "gsl_assert" // for Expects -#include "gsl_byte" // for byte -#include "gsl_util" // for narrow_cast - -#include // for transform, lexicographical_compare -#include // for array -#include -#include // for ptrdiff_t, size_t, nullptr_t -#include // for PTRDIFF_MAX -#include // for divides, multiplies, minus, negate, plus -#include // for initializer_list -#include // for iterator, random_access_iterator_tag -#include // for numeric_limits -#include -#include -#include -#include // for basic_string -#include // for enable_if_t, remove_cv_t, is_same, is_co... -#include - -#ifdef _MSC_VER - -// turn off some warnings that are noisy about our Expects statements -#pragma warning(push) -#pragma warning(disable : 4127) // conditional expression is constant -#pragma warning(disable : 4702) // unreachable code - -#if _MSC_VER < 1910 -#pragma push_macro("constexpr") -#define constexpr /*constexpr*/ - -#endif // _MSC_VER < 1910 -#endif // _MSC_VER - -// GCC 7 does not like the signed unsigned missmatch (size_t ptrdiff_t) -// While there is a conversion from signed to unsigned, it happens at -// compiletime, so the compiler wouldn't have to warn indiscriminently, but -// could check if the source value actually doesn't fit into the target type -// and only warn in those cases. -#if __GNUC__ > 6 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wsign-conversion" -#endif - -#ifdef GSL_THROW_ON_CONTRACT_VIOLATION -#define GSL_NOEXCEPT /*noexcept*/ -#else -#define GSL_NOEXCEPT noexcept -#endif // GSL_THROW_ON_CONTRACT_VIOLATION - -namespace gsl -{ - -/* -** begin definitions of index and bounds -*/ -namespace details -{ - template - struct SizeTypeTraits - { - static const SizeType max_value = std::numeric_limits::max(); - }; - - template - class are_integral : public std::integral_constant - { - }; - - template - class are_integral - : public std::integral_constant::value && are_integral::value> - { - }; -} - -template -class multi_span_index final -{ - static_assert(Rank > 0, "Rank must be greater than 0!"); - - template - friend class multi_span_index; - -public: - static const std::size_t rank = Rank; - using value_type = std::ptrdiff_t; - using size_type = value_type; - using reference = std::add_lvalue_reference_t; - using const_reference = std::add_lvalue_reference_t>; - - constexpr multi_span_index() GSL_NOEXCEPT {} - - constexpr multi_span_index(const value_type (&values)[Rank]) GSL_NOEXCEPT - { - std::copy(values, values + Rank, elems); - } - - template ::value>> - constexpr multi_span_index(Ts... ds) GSL_NOEXCEPT : elems{narrow_cast(ds)...} - { - } - - constexpr multi_span_index(const multi_span_index& other) GSL_NOEXCEPT = default; - - constexpr multi_span_index& operator=(const multi_span_index& rhs) GSL_NOEXCEPT = default; - - // Preconditions: component_idx < rank - constexpr reference operator[](std::size_t component_idx) - { - Expects(component_idx < Rank); // Component index must be less than rank - return elems[component_idx]; - } - - // Preconditions: component_idx < rank - constexpr const_reference operator[](std::size_t component_idx) const GSL_NOEXCEPT - { - Expects(component_idx < Rank); // Component index must be less than rank - return elems[component_idx]; - } - - constexpr bool operator==(const multi_span_index& rhs) const GSL_NOEXCEPT - { - return std::equal(elems, elems + rank, rhs.elems); - } - - constexpr bool operator!=(const multi_span_index& rhs) const GSL_NOEXCEPT { return !(*this == rhs); } - - constexpr multi_span_index operator+() const GSL_NOEXCEPT { return *this; } - - constexpr multi_span_index operator-() const GSL_NOEXCEPT - { - multi_span_index ret = *this; - std::transform(ret, ret + rank, ret, std::negate{}); - return ret; - } - - constexpr multi_span_index operator+(const multi_span_index& rhs) const GSL_NOEXCEPT - { - multi_span_index ret = *this; - ret += rhs; - return ret; - } - - constexpr multi_span_index operator-(const multi_span_index& rhs) const GSL_NOEXCEPT - { - multi_span_index ret = *this; - ret -= rhs; - return ret; - } - - constexpr multi_span_index& operator+=(const multi_span_index& rhs) GSL_NOEXCEPT - { - std::transform(elems, elems + rank, rhs.elems, elems, std::plus{}); - return *this; - } - - constexpr multi_span_index& operator-=(const multi_span_index& rhs) GSL_NOEXCEPT - { - std::transform(elems, elems + rank, rhs.elems, elems, std::minus{}); - return *this; - } - - constexpr multi_span_index operator*(value_type v) const GSL_NOEXCEPT - { - multi_span_index ret = *this; - ret *= v; - return ret; - } - - constexpr multi_span_index operator/(value_type v) const GSL_NOEXCEPT - { - multi_span_index ret = *this; - ret /= v; - return ret; - } - - friend constexpr multi_span_index operator*(value_type v, const multi_span_index& rhs) GSL_NOEXCEPT - { - return rhs * v; - } - - constexpr multi_span_index& operator*=(value_type v) GSL_NOEXCEPT - { - std::transform(elems, elems + rank, elems, - [v](value_type x) { return std::multiplies{}(x, v); }); - return *this; - } - - constexpr multi_span_index& operator/=(value_type v) GSL_NOEXCEPT - { - std::transform(elems, elems + rank, elems, - [v](value_type x) { return std::divides{}(x, v); }); - return *this; - } - -private: - value_type elems[Rank] = {}; -}; - -#if !defined(_MSC_VER) || _MSC_VER >= 1910 - -struct static_bounds_dynamic_range_t -{ - template ::value>> - constexpr operator T() const GSL_NOEXCEPT - { - return narrow_cast(-1); - } -}; - -constexpr bool operator==(static_bounds_dynamic_range_t, static_bounds_dynamic_range_t) GSL_NOEXCEPT -{ - return true; -} - -constexpr bool operator!=(static_bounds_dynamic_range_t, static_bounds_dynamic_range_t) GSL_NOEXCEPT -{ - return false; -} - -template ::value>> -constexpr bool operator==(static_bounds_dynamic_range_t, T other) GSL_NOEXCEPT -{ - return narrow_cast(-1) == other; -} - -template ::value>> -constexpr bool operator==(T left, static_bounds_dynamic_range_t right) GSL_NOEXCEPT -{ - return right == left; -} - -template ::value>> -constexpr bool operator!=(static_bounds_dynamic_range_t, T other) GSL_NOEXCEPT -{ - return narrow_cast(-1) != other; -} - -template ::value>> -constexpr bool operator!=(T left, static_bounds_dynamic_range_t right) GSL_NOEXCEPT -{ - return right != left; -} - -constexpr static_bounds_dynamic_range_t dynamic_range{}; -#else -const std::ptrdiff_t dynamic_range = -1; -#endif - -struct generalized_mapping_tag -{ -}; -struct contiguous_mapping_tag : generalized_mapping_tag -{ -}; - -namespace details -{ - - template - struct LessThan - { - static const bool value = Left < Right; - }; - - template - struct BoundsRanges - { - using size_type = std::ptrdiff_t; - static const size_type Depth = 0; - static const size_type DynamicNum = 0; - static const size_type CurrentRange = 1; - static const size_type TotalSize = 1; - - // TODO : following signature is for work around VS bug - template - BoundsRanges(const OtherRange&, bool /* firstLevel */) - { - } - - BoundsRanges(const std::ptrdiff_t* const) {} - BoundsRanges() = default; - - template - void serialize(T&) const - { - } - - template - size_type linearize(const T&) const - { - return 0; - } - - template - size_type contains(const T&) const - { - return -1; - } - - size_type elementNum(std::size_t) const GSL_NOEXCEPT { return 0; } - - size_type totalSize() const GSL_NOEXCEPT { return TotalSize; } - - bool operator==(const BoundsRanges&) const GSL_NOEXCEPT { return true; } - }; - - template - struct BoundsRanges : BoundsRanges - { - using Base = BoundsRanges; - using size_type = std::ptrdiff_t; - static const std::size_t Depth = Base::Depth + 1; - static const std::size_t DynamicNum = Base::DynamicNum + 1; - static const size_type CurrentRange = dynamic_range; - static const size_type TotalSize = dynamic_range; - - private: - size_type m_bound; - - public: - BoundsRanges(const std::ptrdiff_t* const arr) - : Base(arr + 1), m_bound(*arr * this->Base::totalSize()) - { - Expects(0 <= *arr); - } - - BoundsRanges() : m_bound(0) {} - - template - BoundsRanges(const BoundsRanges& other, - bool /* firstLevel */ = true) - : Base(static_cast&>(other), false) - , m_bound(other.totalSize()) - { - } - - template - void serialize(T& arr) const - { - arr[Dim] = elementNum(); - this->Base::template serialize(arr); - } - - template - size_type linearize(const T& arr) const - { - const size_type index = this->Base::totalSize() * arr[Dim]; - Expects(index < m_bound); - return index + this->Base::template linearize(arr); - } - - template - size_type contains(const T& arr) const - { - const ptrdiff_t last = this->Base::template contains(arr); - if (last == -1) return -1; - const ptrdiff_t cur = this->Base::totalSize() * arr[Dim]; - return cur < m_bound ? cur + last : -1; - } - - size_type totalSize() const GSL_NOEXCEPT { return m_bound; } - - size_type elementNum() const GSL_NOEXCEPT { return totalSize() / this->Base::totalSize(); } - - size_type elementNum(std::size_t dim) const GSL_NOEXCEPT - { - if (dim > 0) - return this->Base::elementNum(dim - 1); - else - return elementNum(); - } - - bool operator==(const BoundsRanges& rhs) const GSL_NOEXCEPT - { - return m_bound == rhs.m_bound && - static_cast(*this) == static_cast(rhs); - } - }; - - template - struct BoundsRanges : BoundsRanges - { - using Base = BoundsRanges; - using size_type = std::ptrdiff_t; - static const std::size_t Depth = Base::Depth + 1; - static const std::size_t DynamicNum = Base::DynamicNum; - static const size_type CurrentRange = CurRange; - static const size_type TotalSize = - Base::TotalSize == dynamic_range ? dynamic_range : CurrentRange * Base::TotalSize; - - BoundsRanges(const std::ptrdiff_t* const arr) : Base(arr) {} - BoundsRanges() = default; - - template - BoundsRanges(const BoundsRanges& other, - bool firstLevel = true) - : Base(static_cast&>(other), false) - { - (void) firstLevel; - } - - template - void serialize(T& arr) const - { - arr[Dim] = elementNum(); - this->Base::template serialize(arr); - } - - template - size_type linearize(const T& arr) const - { - Expects(arr[Dim] >= 0 && arr[Dim] < CurrentRange); // Index is out of range - return this->Base::totalSize() * arr[Dim] + - this->Base::template linearize(arr); - } - - template - size_type contains(const T& arr) const - { - if (arr[Dim] >= CurrentRange) return -1; - const size_type last = this->Base::template contains(arr); - if (last == -1) return -1; - return this->Base::totalSize() * arr[Dim] + last; - } - - size_type totalSize() const GSL_NOEXCEPT { return CurrentRange * this->Base::totalSize(); } - - size_type elementNum() const GSL_NOEXCEPT { return CurrentRange; } - - size_type elementNum(std::size_t dim) const GSL_NOEXCEPT - { - if (dim > 0) - return this->Base::elementNum(dim - 1); - else - return elementNum(); - } - - bool operator==(const BoundsRanges& rhs) const GSL_NOEXCEPT - { - return static_cast(*this) == static_cast(rhs); - } - }; - - template - struct BoundsRangeConvertible - : public std::integral_constant= TargetType::TotalSize || - TargetType::TotalSize == dynamic_range || - SourceType::TotalSize == dynamic_range || - TargetType::TotalSize == 0)> - { - }; - - template - struct TypeListIndexer - { - const TypeChain& obj_; - TypeListIndexer(const TypeChain& obj) : obj_(obj) {} - - template - const TypeChain& getObj(std::true_type) - { - return obj_; - } - - template - auto getObj(std::false_type) - -> decltype(TypeListIndexer(static_cast(obj_)).template get()) - { - return TypeListIndexer(static_cast(obj_)).template get(); - } - - template - auto get() -> decltype(getObj(std::integral_constant())) - { - return getObj(std::integral_constant()); - } - }; - - template - TypeListIndexer createTypeListIndexer(const TypeChain& obj) - { - return TypeListIndexer(obj); - } - - template 1), - typename Ret = std::enable_if_t>> - constexpr Ret shift_left(const multi_span_index& other) GSL_NOEXCEPT - { - Ret ret{}; - for (std::size_t i = 0; i < Rank - 1; ++i) { - ret[i] = other[i + 1]; - } - return ret; - } -} - -template -class bounds_iterator; - -template -class static_bounds -{ -public: - static_bounds(const details::BoundsRanges&) {} -}; - -template -class static_bounds -{ - using MyRanges = details::BoundsRanges; - - MyRanges m_ranges; - constexpr static_bounds(const MyRanges& range) : m_ranges(range) {} - - template - friend class static_bounds; - -public: - static const std::size_t rank = MyRanges::Depth; - static const std::size_t dynamic_rank = MyRanges::DynamicNum; - static const std::ptrdiff_t static_size = MyRanges::TotalSize; - - using size_type = std::ptrdiff_t; - using index_type = multi_span_index; - using const_index_type = std::add_const_t; - using iterator = bounds_iterator; - using const_iterator = bounds_iterator; - using difference_type = std::ptrdiff_t; - using sliced_type = static_bounds; - using mapping_type = contiguous_mapping_tag; - - constexpr static_bounds(const static_bounds&) = default; - - template - struct BoundsRangeConvertible2; - - template > - static auto helpBoundsRangeConvertible(SourceType, TargetType, std::true_type) -> Ret; - - template - static auto helpBoundsRangeConvertible(SourceType, TargetType, ...) -> std::false_type; - - template - struct BoundsRangeConvertible2 - : decltype(helpBoundsRangeConvertible( - SourceType(), TargetType(), - std::integral_constant())) - { - }; - - template - struct BoundsRangeConvertible2 : std::true_type - { - }; - - template - struct BoundsRangeConvertible - : decltype(helpBoundsRangeConvertible( - SourceType(), TargetType(), - std::integral_constant::value || - TargetType::CurrentRange == dynamic_range || - SourceType::CurrentRange == dynamic_range)>())) - { - }; - - template - struct BoundsRangeConvertible : std::true_type - { - }; - - template , - details::BoundsRanges>::value>> - constexpr static_bounds(const static_bounds& other) : m_ranges(other.m_ranges) - { - Expects((MyRanges::DynamicNum == 0 && details::BoundsRanges::DynamicNum == 0) || - MyRanges::DynamicNum > 0 || other.m_ranges.totalSize() >= m_ranges.totalSize()); - } - - constexpr static_bounds(std::initializer_list il) - : m_ranges(il.begin()) - { - // Size of the initializer list must match the rank of the array - Expects((MyRanges::DynamicNum == 0 && il.size() == 1 && *il.begin() == static_size) || - MyRanges::DynamicNum == il.size()); - // Size of the range must be less than the max element of the size type - Expects(m_ranges.totalSize() <= PTRDIFF_MAX); - } - - constexpr static_bounds() = default; - - constexpr sliced_type slice() const GSL_NOEXCEPT - { - return sliced_type{static_cast&>(m_ranges)}; - } - - constexpr size_type stride() const GSL_NOEXCEPT { return rank > 1 ? slice().size() : 1; } - - constexpr size_type size() const GSL_NOEXCEPT { return m_ranges.totalSize(); } - - constexpr size_type total_size() const GSL_NOEXCEPT { return m_ranges.totalSize(); } - - constexpr size_type linearize(const index_type& idx) const { return m_ranges.linearize(idx); } - - constexpr bool contains(const index_type& idx) const GSL_NOEXCEPT - { - return m_ranges.contains(idx) != -1; - } - - constexpr size_type operator[](std::size_t idx) const GSL_NOEXCEPT - { - return m_ranges.elementNum(idx); - } - - template - constexpr size_type extent() const GSL_NOEXCEPT - { - static_assert(Dim < rank, - "dimension should be less than rank (dimension count starts from 0)"); - return details::createTypeListIndexer(m_ranges).template get().elementNum(); - } - - template - constexpr size_type extent(IntType dim) const GSL_NOEXCEPT - { - static_assert(std::is_integral::value, - "Dimension parameter must be supplied as an integral type."); - auto real_dim = narrow_cast(dim); - Expects(real_dim < rank); - - return m_ranges.elementNum(real_dim); - } - - constexpr index_type index_bounds() const GSL_NOEXCEPT - { - size_type extents[rank] = {}; - m_ranges.serialize(extents); - return {extents}; - } - - template - constexpr bool operator==(const static_bounds& rhs) const GSL_NOEXCEPT - { - return this->size() == rhs.size(); - } - - template - constexpr bool operator!=(const static_bounds& rhs) const GSL_NOEXCEPT - { - return !(*this == rhs); - } - - constexpr const_iterator begin() const GSL_NOEXCEPT - { - return const_iterator(*this, index_type{}); - } - - constexpr const_iterator end() const GSL_NOEXCEPT - { - return const_iterator(*this, this->index_bounds()); - } -}; - -template -class strided_bounds -{ - template - friend class strided_bounds; - -public: - static const std::size_t rank = Rank; - using value_type = std::ptrdiff_t; - using reference = std::add_lvalue_reference_t; - using const_reference = std::add_const_t; - using size_type = value_type; - using difference_type = value_type; - using index_type = multi_span_index; - using const_index_type = std::add_const_t; - using iterator = bounds_iterator; - using const_iterator = bounds_iterator; - static const value_type dynamic_rank = rank; - static const value_type static_size = dynamic_range; - using sliced_type = std::conditional_t, void>; - using mapping_type = generalized_mapping_tag; - - constexpr strided_bounds(const strided_bounds&) GSL_NOEXCEPT = default; - - constexpr strided_bounds& operator=(const strided_bounds&) GSL_NOEXCEPT = default; - - constexpr strided_bounds(const value_type (&values)[rank], index_type strides) - : m_extents(values), m_strides(std::move(strides)) - { - } - - constexpr strided_bounds(const index_type& extents, const index_type& strides) GSL_NOEXCEPT - : m_extents(extents), - m_strides(strides) - { - } - - constexpr index_type strides() const GSL_NOEXCEPT { return m_strides; } - - constexpr size_type total_size() const GSL_NOEXCEPT - { - size_type ret = 0; - for (std::size_t i = 0; i < rank; ++i) { - ret += (m_extents[i] - 1) * m_strides[i]; - } - return ret + 1; - } - - constexpr size_type size() const GSL_NOEXCEPT - { - size_type ret = 1; - for (std::size_t i = 0; i < rank; ++i) { - ret *= m_extents[i]; - } - return ret; - } - - constexpr bool contains(const index_type& idx) const GSL_NOEXCEPT - { - for (std::size_t i = 0; i < rank; ++i) { - if (idx[i] < 0 || idx[i] >= m_extents[i]) return false; - } - return true; - } - - constexpr size_type linearize(const index_type& idx) const GSL_NOEXCEPT - { - size_type ret = 0; - for (std::size_t i = 0; i < rank; i++) { - Expects(idx[i] < m_extents[i]); // index is out of bounds of the array - ret += idx[i] * m_strides[i]; - } - return ret; - } - - constexpr size_type stride() const GSL_NOEXCEPT { return m_strides[0]; } - - template 1), typename Ret = std::enable_if_t> - constexpr sliced_type slice() const - { - return {details::shift_left(m_extents), details::shift_left(m_strides)}; - } - - template - constexpr size_type extent() const GSL_NOEXCEPT - { - static_assert(Dim < Rank, - "dimension should be less than rank (dimension count starts from 0)"); - return m_extents[Dim]; - } - - constexpr index_type index_bounds() const GSL_NOEXCEPT { return m_extents; } - constexpr const_iterator begin() const GSL_NOEXCEPT - { - return const_iterator{*this, index_type{}}; - } - - constexpr const_iterator end() const GSL_NOEXCEPT - { - return const_iterator{*this, index_bounds()}; - } - -private: - index_type m_extents; - index_type m_strides; -}; - -template -struct is_bounds : std::integral_constant -{ -}; -template -struct is_bounds> : std::integral_constant -{ -}; -template -struct is_bounds> : std::integral_constant -{ -}; - -template -class bounds_iterator -{ -public: - static const std::size_t rank = IndexType::rank; - using iterator_category = std::random_access_iterator_tag; - using value_type = IndexType; - using difference_type = std::ptrdiff_t; - using pointer = value_type*; - using reference = value_type&; - using index_type = value_type; - using index_size_type = typename IndexType::value_type; - template - explicit bounds_iterator(const Bounds& bnd, value_type curr) GSL_NOEXCEPT - : boundary_(bnd.index_bounds()), - curr_(std::move(curr)) - { - static_assert(is_bounds::value, "Bounds type must be provided"); - } - - constexpr reference operator*() const GSL_NOEXCEPT { return curr_; } - - constexpr pointer operator->() const GSL_NOEXCEPT { return &curr_; } - - constexpr bounds_iterator& operator++() GSL_NOEXCEPT - { - for (std::size_t i = rank; i-- > 0;) { - if (curr_[i] < boundary_[i] - 1) { - curr_[i]++; - return *this; - } - curr_[i] = 0; - } - // If we're here we've wrapped over - set to past-the-end. - curr_ = boundary_; - return *this; - } - - constexpr bounds_iterator operator++(int) GSL_NOEXCEPT - { - auto ret = *this; - ++(*this); - return ret; - } - - constexpr bounds_iterator& operator--() GSL_NOEXCEPT - { - if (!less(curr_, boundary_)) { - // if at the past-the-end, set to last element - for (std::size_t i = 0; i < rank; ++i) { - curr_[i] = boundary_[i] - 1; - } - return *this; - } - for (std::size_t i = rank; i-- > 0;) { - if (curr_[i] >= 1) { - curr_[i]--; - return *this; - } - curr_[i] = boundary_[i] - 1; - } - // If we're here the preconditions were violated - // "pre: there exists s such that r == ++s" - Expects(false); - return *this; - } - - constexpr bounds_iterator operator--(int) GSL_NOEXCEPT - { - auto ret = *this; - --(*this); - return ret; - } - - constexpr bounds_iterator operator+(difference_type n) const GSL_NOEXCEPT - { - bounds_iterator ret{*this}; - return ret += n; - } - - constexpr bounds_iterator& operator+=(difference_type n) GSL_NOEXCEPT - { - auto linear_idx = linearize(curr_) + n; - std::remove_const_t stride = 0; - stride[rank - 1] = 1; - for (std::size_t i = rank - 1; i-- > 0;) { - stride[i] = stride[i + 1] * boundary_[i + 1]; - } - for (std::size_t i = 0; i < rank; ++i) { - curr_[i] = linear_idx / stride[i]; - linear_idx = linear_idx % stride[i]; - } - // index is out of bounds of the array - Expects(!less(curr_, index_type{}) && !less(boundary_, curr_)); - return *this; - } - - constexpr bounds_iterator operator-(difference_type n) const GSL_NOEXCEPT - { - bounds_iterator ret{*this}; - return ret -= n; - } - - constexpr bounds_iterator& operator-=(difference_type n) GSL_NOEXCEPT { return *this += -n; } - - constexpr difference_type operator-(const bounds_iterator& rhs) const GSL_NOEXCEPT - { - return linearize(curr_) - linearize(rhs.curr_); - } - - constexpr value_type operator[](difference_type n) const GSL_NOEXCEPT { return *(*this + n); } - - constexpr bool operator==(const bounds_iterator& rhs) const GSL_NOEXCEPT - { - return curr_ == rhs.curr_; - } - - constexpr bool operator!=(const bounds_iterator& rhs) const GSL_NOEXCEPT - { - return !(*this == rhs); - } - - constexpr bool operator<(const bounds_iterator& rhs) const GSL_NOEXCEPT - { - return less(curr_, rhs.curr_); - } - - constexpr bool operator<=(const bounds_iterator& rhs) const GSL_NOEXCEPT - { - return !(rhs < *this); - } - - constexpr bool operator>(const bounds_iterator& rhs) const GSL_NOEXCEPT { return rhs < *this; } - - constexpr bool operator>=(const bounds_iterator& rhs) const GSL_NOEXCEPT - { - return !(rhs > *this); - } - - void swap(bounds_iterator& rhs) GSL_NOEXCEPT - { - std::swap(boundary_, rhs.boundary_); - std::swap(curr_, rhs.curr_); - } - -private: - constexpr bool less(index_type& one, index_type& other) const GSL_NOEXCEPT - { - for (std::size_t i = 0; i < rank; ++i) { - if (one[i] < other[i]) return true; - } - return false; - } - - constexpr index_size_type linearize(const value_type& idx) const GSL_NOEXCEPT - { - // TODO: Smarter impl. - // Check if past-the-end - index_size_type multiplier = 1; - index_size_type res = 0; - if (!less(idx, boundary_)) { - res = 1; - for (std::size_t i = rank; i-- > 0;) { - res += (idx[i] - 1) * multiplier; - multiplier *= boundary_[i]; - } - } - else - { - for (std::size_t i = rank; i-- > 0;) { - res += idx[i] * multiplier; - multiplier *= boundary_[i]; - } - } - return res; - } - - value_type boundary_; - std::remove_const_t curr_; -}; - -template -bounds_iterator operator+(typename bounds_iterator::difference_type n, - const bounds_iterator& rhs) GSL_NOEXCEPT -{ - return rhs + n; -} - -namespace details -{ - template - constexpr std::enable_if_t< - std::is_same::value, - typename Bounds::index_type> - make_stride(const Bounds& bnd) GSL_NOEXCEPT - { - return bnd.strides(); - } - - // Make a stride vector from bounds, assuming contiguous memory. - template - constexpr std::enable_if_t< - std::is_same::value, - typename Bounds::index_type> - make_stride(const Bounds& bnd) GSL_NOEXCEPT - { - auto extents = bnd.index_bounds(); - typename Bounds::size_type stride[Bounds::rank] = {}; - - stride[Bounds::rank - 1] = 1; - for (std::size_t i = 1; i < Bounds::rank; ++i) { - stride[Bounds::rank - i - 1] = stride[Bounds::rank - i] * extents[Bounds::rank - i]; - } - return {stride}; - } - - template - void verifyBoundsReshape(const BoundsSrc& src, const BoundsDest& dest) - { - static_assert(is_bounds::value && is_bounds::value, - "The src type and dest type must be bounds"); - static_assert(std::is_same::value, - "The source type must be a contiguous bounds"); - static_assert(BoundsDest::static_size == dynamic_range || - BoundsSrc::static_size == dynamic_range || - BoundsDest::static_size == BoundsSrc::static_size, - "The source bounds must have same size as dest bounds"); - Expects(src.size() == dest.size()); - } - -} // namespace details - -template -class contiguous_span_iterator; -template -class general_span_iterator; - -template -struct dim_t -{ - static const std::ptrdiff_t value = DimSize; -}; -template <> -struct dim_t -{ - static const std::ptrdiff_t value = dynamic_range; - const std::ptrdiff_t dvalue; - constexpr dim_t(std::ptrdiff_t size) GSL_NOEXCEPT : dvalue(size) {} -}; - -template = 0)>> -constexpr dim_t dim() GSL_NOEXCEPT -{ - return dim_t(); -} - -template > -constexpr dim_t dim(std::ptrdiff_t n) GSL_NOEXCEPT -{ - return dim_t<>(n); -} - -template -class multi_span; -template -class strided_span; - -namespace details -{ - template - struct SpanTypeTraits - { - using value_type = T; - using size_type = std::size_t; - }; - - template - struct SpanTypeTraits::type> - { - using value_type = typename Traits::span_traits::value_type; - using size_type = typename Traits::span_traits::size_type; - }; - - template - struct SpanArrayTraits - { - using type = multi_span; - using value_type = T; - using bounds_type = static_bounds; - using pointer = T*; - using reference = T&; - }; - template - struct SpanArrayTraits : SpanArrayTraits - { - }; - - template - BoundsType newBoundsHelperImpl(std::ptrdiff_t totalSize, std::true_type) // dynamic size - { - Expects(totalSize >= 0 && totalSize <= PTRDIFF_MAX); - return BoundsType{totalSize}; - } - template - BoundsType newBoundsHelperImpl(std::ptrdiff_t totalSize, std::false_type) // static size - { - Expects(BoundsType::static_size <= totalSize); - return {}; - } - template - BoundsType newBoundsHelper(std::ptrdiff_t totalSize) - { - static_assert(BoundsType::dynamic_rank <= 1, "dynamic rank must less or equal to 1"); - return newBoundsHelperImpl( - totalSize, std::integral_constant()); - } - - struct Sep - { - }; - - template - T static_as_multi_span_helper(Sep, Args... args) - { - return T{narrow_cast(args)...}; - } - template - std::enable_if_t< - !std::is_same>::value && !std::is_same::value, T> - static_as_multi_span_helper(Arg, Args... args) - { - return static_as_multi_span_helper(args...); - } - template - T static_as_multi_span_helper(dim_t val, Args... args) - { - return static_as_multi_span_helper(args..., val.dvalue); - } - - template - struct static_as_multi_span_static_bounds_helper - { - using type = static_bounds<(Dimensions::value)...>; - }; - - template - struct is_multi_span_oracle : std::false_type - { - }; - - template - struct is_multi_span_oracle> - : std::true_type - { - }; - - template - struct is_multi_span_oracle> : std::true_type - { - }; - - template - struct is_multi_span : is_multi_span_oracle> - { - }; -} - -template -class multi_span -{ - // TODO do we still need this? - template - friend class multi_span; - -public: - using bounds_type = static_bounds; - static const std::size_t Rank = bounds_type::rank; - using size_type = typename bounds_type::size_type; - using index_type = typename bounds_type::index_type; - using value_type = ValueType; - using const_value_type = std::add_const_t; - using pointer = std::add_pointer_t; - using reference = std::add_lvalue_reference_t; - using iterator = contiguous_span_iterator; - using const_span = multi_span; - using const_iterator = contiguous_span_iterator; - using reverse_iterator = std::reverse_iterator; - using const_reverse_iterator = std::reverse_iterator; - using sliced_type = - std::conditional_t>; - -private: - pointer data_; - bounds_type bounds_; - - friend iterator; - friend const_iterator; - -public: - // default constructor - same as constructing from nullptr_t - constexpr multi_span() GSL_NOEXCEPT : multi_span(nullptr, bounds_type{}) - { - static_assert(bounds_type::dynamic_rank != 0 || - (bounds_type::dynamic_rank == 0 && bounds_type::static_size == 0), - "Default construction of multi_span only possible " - "for dynamic or fixed, zero-length spans."); - } - - // construct from nullptr - get an empty multi_span - constexpr multi_span(std::nullptr_t) GSL_NOEXCEPT : multi_span(nullptr, bounds_type{}) - { - static_assert(bounds_type::dynamic_rank != 0 || - (bounds_type::dynamic_rank == 0 && bounds_type::static_size == 0), - "nullptr_t construction of multi_span only possible " - "for dynamic or fixed, zero-length spans."); - } - - // construct from nullptr with size of 0 (helps with template function calls) - template ::value>> - constexpr multi_span(std::nullptr_t, IntType size) GSL_NOEXCEPT - : multi_span(nullptr, bounds_type{}) - { - static_assert(bounds_type::dynamic_rank != 0 || - (bounds_type::dynamic_rank == 0 && bounds_type::static_size == 0), - "nullptr_t construction of multi_span only possible " - "for dynamic or fixed, zero-length spans."); - Expects(size == 0); - } - - // construct from a single element - constexpr multi_span(reference data) GSL_NOEXCEPT : multi_span(&data, bounds_type{1}) - { - static_assert(bounds_type::dynamic_rank > 0 || bounds_type::static_size == 0 || - bounds_type::static_size == 1, - "Construction from a single element only possible " - "for dynamic or fixed spans of length 0 or 1."); - } - - // prevent constructing from temporaries for single-elements - constexpr multi_span(value_type&&) = delete; - - // construct from pointer + length - constexpr multi_span(pointer ptr, size_type size) GSL_NOEXCEPT - : multi_span(ptr, bounds_type{size}) - { - } - - // construct from pointer + length - multidimensional - constexpr multi_span(pointer data, bounds_type bounds) GSL_NOEXCEPT : data_(data), - bounds_(std::move(bounds)) - { - Expects((bounds_.size() > 0 && data != nullptr) || bounds_.size() == 0); - } - - // construct from begin,end pointer pair - template ::value && - details::LessThan::value>> - constexpr multi_span(pointer begin, Ptr end) - : multi_span(begin, - details::newBoundsHelper(static_cast(end) - begin)) - { - Expects(begin != nullptr && end != nullptr && begin <= static_cast(end)); - } - - // construct from n-dimensions static array - template > - constexpr multi_span(T (&arr)[N]) - : multi_span(reinterpret_cast(arr), bounds_type{typename Helper::bounds_type{}}) - { - static_assert(std::is_convertible::value, - "Cannot convert from source type to target multi_span type."); - static_assert(std::is_convertible::value, - "Cannot construct a multi_span from an array with fewer elements."); - } - - // construct from n-dimensions dynamic array (e.g. new int[m][4]) - // (precedence will be lower than the 1-dimension pointer) - template > - constexpr multi_span(T* const& data, size_type size) - : multi_span(reinterpret_cast(data), typename Helper::bounds_type{size}) - { - static_assert(std::is_convertible::value, - "Cannot convert from source type to target multi_span type."); - } - - // construct from std::array - template - constexpr multi_span(std::array& arr) - : multi_span(arr.data(), bounds_type{static_bounds{}}) - { - static_assert( - std::is_convertible(*)[]>::value, - "Cannot convert from source type to target multi_span type."); - static_assert(std::is_convertible, bounds_type>::value, - "You cannot construct a multi_span from a std::array of smaller size."); - } - - // construct from const std::array - template - constexpr multi_span(const std::array& arr) - : multi_span(arr.data(), bounds_type{static_bounds{}}) - { - static_assert(std::is_convertible(*)[]>::value, - "Cannot convert from source type to target multi_span type."); - static_assert(std::is_convertible, bounds_type>::value, - "You cannot construct a multi_span from a std::array of smaller size."); - } - - // prevent constructing from temporary std::array - template - constexpr multi_span(std::array&& arr) = delete; - - // construct from containers - // future: could use contiguous_iterator_traits to identify only contiguous containers - // type-requirements: container must have .size(), operator[] which are value_type compatible - template ::value && - std::is_convertible::value && - std::is_same().size(), - *std::declval().data())>, - DataType>::value>> - constexpr multi_span(Cont& cont) - : multi_span(static_cast(cont.data()), - details::newBoundsHelper(narrow_cast(cont.size()))) - { - } - - // prevent constructing from temporary containers - template ::value && - std::is_convertible::value && - std::is_same().size(), - *std::declval().data())>, - DataType>::value>> - explicit constexpr multi_span(Cont&& cont) = delete; - - // construct from a convertible multi_span - template , - typename = std::enable_if_t::value && - std::is_convertible::value>> - constexpr multi_span(multi_span other) GSL_NOEXCEPT - : data_(other.data_), - bounds_(other.bounds_) - { - } - - // trivial copy and move - constexpr multi_span(const multi_span&) = default; - constexpr multi_span(multi_span&&) = default; - - // trivial assignment - constexpr multi_span& operator=(const multi_span&) = default; - constexpr multi_span& operator=(multi_span&&) = default; - - // first() - extract the first Count elements into a new multi_span - template - constexpr multi_span first() const GSL_NOEXCEPT - { - static_assert(Count >= 0, "Count must be >= 0."); - static_assert(bounds_type::static_size == dynamic_range || - Count <= bounds_type::static_size, - "Count is out of bounds."); - - Expects(bounds_type::static_size != dynamic_range || Count <= this->size()); - return {this->data(), Count}; - } - - // first() - extract the first count elements into a new multi_span - constexpr multi_span first(size_type count) const GSL_NOEXCEPT - { - Expects(count >= 0 && count <= this->size()); - return {this->data(), count}; - } - - // last() - extract the last Count elements into a new multi_span - template - constexpr multi_span last() const GSL_NOEXCEPT - { - static_assert(Count >= 0, "Count must be >= 0."); - static_assert(bounds_type::static_size == dynamic_range || - Count <= bounds_type::static_size, - "Count is out of bounds."); - - Expects(bounds_type::static_size != dynamic_range || Count <= this->size()); - return {this->data() + this->size() - Count, Count}; - } - - // last() - extract the last count elements into a new multi_span - constexpr multi_span last(size_type count) const GSL_NOEXCEPT - { - Expects(count >= 0 && count <= this->size()); - return {this->data() + this->size() - count, count}; - } - - // subspan() - create a subview of Count elements starting at Offset - template - constexpr multi_span subspan() const GSL_NOEXCEPT - { - static_assert(Count >= 0, "Count must be >= 0."); - static_assert(Offset >= 0, "Offset must be >= 0."); - static_assert(bounds_type::static_size == dynamic_range || - ((Offset <= bounds_type::static_size) && - Count <= bounds_type::static_size - Offset), - "You must describe a sub-range within bounds of the multi_span."); - - Expects(bounds_type::static_size != dynamic_range || - (Offset <= this->size() && Count <= this->size() - Offset)); - return {this->data() + Offset, Count}; - } - - // subspan() - create a subview of count elements starting at offset - // supplying dynamic_range for count will consume all available elements from offset - constexpr multi_span - subspan(size_type offset, size_type count = dynamic_range) const GSL_NOEXCEPT - { - Expects((offset >= 0 && offset <= this->size()) && - (count == dynamic_range || (count <= this->size() - offset))); - return {this->data() + offset, count == dynamic_range ? this->length() - offset : count}; - } - - // section - creates a non-contiguous, strided multi_span from a contiguous one - constexpr strided_span section(index_type origin, - index_type extents) const GSL_NOEXCEPT - { - size_type size = this->bounds().total_size() - this->bounds().linearize(origin); - return {&this->operator[](origin), size, - strided_bounds{extents, details::make_stride(bounds())}}; - } - - // length of the multi_span in elements - constexpr size_type size() const GSL_NOEXCEPT { return bounds_.size(); } - - // length of the multi_span in elements - constexpr size_type length() const GSL_NOEXCEPT { return this->size(); } - - // length of the multi_span in bytes - constexpr size_type size_bytes() const GSL_NOEXCEPT - { - return narrow_cast(sizeof(value_type)) * this->size(); - } - - // length of the multi_span in bytes - constexpr size_type length_bytes() const GSL_NOEXCEPT { return this->size_bytes(); } - - constexpr bool empty() const GSL_NOEXCEPT { return this->size() == 0; } - - static constexpr std::size_t rank() { return Rank; } - - template - constexpr size_type extent() const GSL_NOEXCEPT - { - static_assert(Dim < Rank, - "Dimension should be less than rank (dimension count starts from 0)."); - return bounds_.template extent(); - } - - template - constexpr size_type extent(IntType dim) const GSL_NOEXCEPT - { - return bounds_.extent(dim); - } - - constexpr bounds_type bounds() const GSL_NOEXCEPT { return bounds_; } - - constexpr pointer data() const GSL_NOEXCEPT { return data_; } - - template - constexpr reference operator()(FirstIndex idx) - { - return this->operator[](narrow_cast(idx)); - } - - template - constexpr reference operator()(FirstIndex firstIndex, OtherIndices... indices) - { - index_type idx = {narrow_cast(firstIndex), - narrow_cast(indices)...}; - return this->operator[](idx); - } - - constexpr reference operator[](const index_type& idx) const GSL_NOEXCEPT - { - return data_[bounds_.linearize(idx)]; - } - - template 1), typename Ret = std::enable_if_t> - constexpr Ret operator[](size_type idx) const GSL_NOEXCEPT - { - Expects(idx >= 0 && idx < bounds_.size()); // index is out of bounds of the array - const size_type ridx = idx * bounds_.stride(); - - // index is out of bounds of the underlying data - Expects(ridx < bounds_.total_size()); - return Ret{data_ + ridx, bounds_.slice()}; - } - - constexpr iterator begin() const GSL_NOEXCEPT { return iterator{this, true}; } - - constexpr iterator end() const GSL_NOEXCEPT { return iterator{this, false}; } - - constexpr const_iterator cbegin() const GSL_NOEXCEPT - { - return const_iterator{reinterpret_cast(this), true}; - } - - constexpr const_iterator cend() const GSL_NOEXCEPT - { - return const_iterator{reinterpret_cast(this), false}; - } - - constexpr reverse_iterator rbegin() const GSL_NOEXCEPT { return reverse_iterator{end()}; } - - constexpr reverse_iterator rend() const GSL_NOEXCEPT { return reverse_iterator{begin()}; } - - constexpr const_reverse_iterator crbegin() const GSL_NOEXCEPT - { - return const_reverse_iterator{cend()}; - } - - constexpr const_reverse_iterator crend() const GSL_NOEXCEPT - { - return const_reverse_iterator{cbegin()}; - } - - template , std::remove_cv_t>::value>> - constexpr bool - operator==(const multi_span& other) const GSL_NOEXCEPT - { - return bounds_.size() == other.bounds_.size() && - (data_ == other.data_ || std::equal(this->begin(), this->end(), other.begin())); - } - - template , std::remove_cv_t>::value>> - constexpr bool - operator!=(const multi_span& other) const GSL_NOEXCEPT - { - return !(*this == other); - } - - template , std::remove_cv_t>::value>> - constexpr bool - operator<(const multi_span& other) const GSL_NOEXCEPT - { - return std::lexicographical_compare(this->begin(), this->end(), other.begin(), other.end()); - } - - template , std::remove_cv_t>::value>> - constexpr bool - operator<=(const multi_span& other) const GSL_NOEXCEPT - { - return !(other < *this); - } - - template , std::remove_cv_t>::value>> - constexpr bool - operator>(const multi_span& other) const GSL_NOEXCEPT - { - return (other < *this); - } - - template , std::remove_cv_t>::value>> - constexpr bool - operator>=(const multi_span& other) const GSL_NOEXCEPT - { - return !(*this < other); - } -}; - -// -// Free functions for manipulating spans -// - -// reshape a multi_span into a different dimensionality -// DimCount and Enabled here are workarounds for a bug in MSVC 2015 -template 0), typename = std::enable_if_t> -constexpr auto as_multi_span(SpanType s, Dimensions2... dims) - -> multi_span -{ - static_assert(details::is_multi_span::value, - "Variadic as_multi_span() is for reshaping existing spans."); - using BoundsType = - typename multi_span::bounds_type; - auto tobounds = details::static_as_multi_span_helper(dims..., details::Sep{}); - details::verifyBoundsReshape(s.bounds(), tobounds); - return {s.data(), tobounds}; -} - -// convert a multi_span to a multi_span -template -multi_span as_bytes(multi_span s) GSL_NOEXCEPT -{ - static_assert(std::is_trivial>::value, - "The value_type of multi_span must be a trivial type."); - return {reinterpret_cast(s.data()), s.size_bytes()}; -} - -// convert a multi_span to a multi_span (a writeable byte multi_span) -// this is not currently a portable function that can be relied upon to work -// on all implementations. It should be considered an experimental extension -// to the standard GSL interface. -template -multi_span as_writeable_bytes(multi_span s) GSL_NOEXCEPT -{ - static_assert(std::is_trivial>::value, - "The value_type of multi_span must be a trivial type."); - return {reinterpret_cast(s.data()), s.size_bytes()}; -} - -// convert a multi_span to a multi_span -// this is not currently a portable function that can be relied upon to work -// on all implementations. It should be considered an experimental extension -// to the standard GSL interface. -template -constexpr auto -as_multi_span(multi_span s) GSL_NOEXCEPT -> multi_span< - const U, static_cast( - multi_span::bounds_type::static_size != dynamic_range - ? (static_cast( - multi_span::bounds_type::static_size) / - sizeof(U)) - : dynamic_range)> -{ - using ConstByteSpan = multi_span; - static_assert( - std::is_trivial>::value && - (ConstByteSpan::bounds_type::static_size == dynamic_range || - ConstByteSpan::bounds_type::static_size % narrow_cast(sizeof(U)) == 0), - "Target type must be a trivial type and its size must match the byte array size"); - - Expects((s.size_bytes() % narrow_cast(sizeof(U))) == 0 && - (s.size_bytes() / narrow_cast(sizeof(U))) < PTRDIFF_MAX); - return {reinterpret_cast(s.data()), - s.size_bytes() / narrow_cast(sizeof(U))}; -} - -// convert a multi_span to a multi_span -// this is not currently a portable function that can be relied upon to work -// on all implementations. It should be considered an experimental extension -// to the standard GSL interface. -template -constexpr auto as_multi_span(multi_span s) GSL_NOEXCEPT - -> multi_span( - multi_span::bounds_type::static_size != dynamic_range - ? static_cast( - multi_span::bounds_type::static_size) / - sizeof(U) - : dynamic_range)> -{ - using ByteSpan = multi_span; - static_assert( - std::is_trivial>::value && - (ByteSpan::bounds_type::static_size == dynamic_range || - ByteSpan::bounds_type::static_size % sizeof(U) == 0), - "Target type must be a trivial type and its size must match the byte array size"); - - Expects((s.size_bytes() % sizeof(U)) == 0); - return {reinterpret_cast(s.data()), - s.size_bytes() / narrow_cast(sizeof(U))}; -} - -template -constexpr auto as_multi_span(T* const& ptr, dim_t... args) - -> multi_span, Dimensions...> -{ - return {reinterpret_cast*>(ptr), - details::static_as_multi_span_helper>(args..., - details::Sep{})}; -} - -template -constexpr auto as_multi_span(T* arr, std::ptrdiff_t len) -> - typename details::SpanArrayTraits::type -{ - return {reinterpret_cast*>(arr), len}; -} - -template -constexpr auto as_multi_span(T (&arr)[N]) -> typename details::SpanArrayTraits::type -{ - return {arr}; -} - -template -constexpr multi_span as_multi_span(const std::array& arr) -{ - return {arr}; -} - -template -constexpr multi_span as_multi_span(const std::array&&) = delete; - -template -constexpr multi_span as_multi_span(std::array& arr) -{ - return {arr}; -} - -template -constexpr multi_span as_multi_span(T* begin, T* end) -{ - return {begin, end}; -} - -template -constexpr auto as_multi_span(Cont& arr) -> std::enable_if_t< - !details::is_multi_span>::value, - multi_span, dynamic_range>> -{ - Expects(arr.size() < PTRDIFF_MAX); - return {arr.data(), narrow_cast(arr.size())}; -} - -template -constexpr auto as_multi_span(Cont&& arr) -> std::enable_if_t< - !details::is_multi_span>::value, - multi_span, dynamic_range>> = delete; - -// from basic_string which doesn't have nonconst .data() member like other contiguous containers -template -constexpr auto as_multi_span(std::basic_string& str) - -> multi_span -{ - Expects(str.size() < PTRDIFF_MAX); - return {&str[0], narrow_cast(str.size())}; -} - -// strided_span is an extension that is not strictly part of the GSL at this time. -// It is kept here while the multidimensional interface is still being defined. -template -class strided_span -{ -public: - using bounds_type = strided_bounds; - using size_type = typename bounds_type::size_type; - using index_type = typename bounds_type::index_type; - using value_type = ValueType; - using const_value_type = std::add_const_t; - using pointer = std::add_pointer_t; - using reference = std::add_lvalue_reference_t; - using iterator = general_span_iterator; - using const_strided_span = strided_span; - using const_iterator = general_span_iterator; - using reverse_iterator = std::reverse_iterator; - using const_reverse_iterator = std::reverse_iterator; - using sliced_type = - std::conditional_t>; - -private: - pointer data_; - bounds_type bounds_; - - friend iterator; - friend const_iterator; - template - friend class strided_span; - -public: - // from raw data - constexpr strided_span(pointer ptr, size_type size, bounds_type bounds) - : data_(ptr), bounds_(std::move(bounds)) - { - Expects((bounds_.size() > 0 && ptr != nullptr) || bounds_.size() == 0); - // Bounds cross data boundaries - Expects(this->bounds().total_size() <= size); - (void) size; - } - - // from static array of size N - template - constexpr strided_span(value_type (&values)[N], bounds_type bounds) - : strided_span(values, N, std::move(bounds)) - { - } - - // from array view - template ::value, - typename = std::enable_if_t> - constexpr strided_span(multi_span av, bounds_type bounds) - : strided_span(av.data(), av.bounds().total_size(), std::move(bounds)) - { - } - - // convertible - template ::value>> - constexpr strided_span(const strided_span& other) - : data_(other.data_), bounds_(other.bounds_) - { - } - - // convert from bytes - template - constexpr strided_span< - typename std::enable_if::value, OtherValueType>::type, - Rank> - as_strided_span() const - { - static_assert((sizeof(OtherValueType) >= sizeof(value_type)) && - (sizeof(OtherValueType) % sizeof(value_type) == 0), - "OtherValueType should have a size to contain a multiple of ValueTypes"); - auto d = narrow_cast(sizeof(OtherValueType) / sizeof(value_type)); - - size_type size = this->bounds().total_size() / d; - return {const_cast(reinterpret_cast(this->data())), - size, - bounds_type{resize_extent(this->bounds().index_bounds(), d), - resize_stride(this->bounds().strides(), d)}}; - } - - constexpr strided_span section(index_type origin, index_type extents) const - { - size_type size = this->bounds().total_size() - this->bounds().linearize(origin); - return {&this->operator[](origin), size, - bounds_type{extents, details::make_stride(bounds())}}; - } - - constexpr reference operator[](const index_type& idx) const - { - return data_[bounds_.linearize(idx)]; - } - - template 1), typename Ret = std::enable_if_t> - constexpr Ret operator[](size_type idx) const - { - Expects(idx < bounds_.size()); // index is out of bounds of the array - const size_type ridx = idx * bounds_.stride(); - - // index is out of bounds of the underlying data - Expects(ridx < bounds_.total_size()); - return {data_ + ridx, bounds_.slice().total_size(), bounds_.slice()}; - } - - constexpr bounds_type bounds() const GSL_NOEXCEPT { return bounds_; } - - template - constexpr size_type extent() const GSL_NOEXCEPT - { - static_assert(Dim < Rank, - "dimension should be less than Rank (dimension count starts from 0)"); - return bounds_.template extent(); - } - - constexpr size_type size() const GSL_NOEXCEPT { return bounds_.size(); } - - constexpr pointer data() const GSL_NOEXCEPT { return data_; } - - constexpr explicit operator bool() const GSL_NOEXCEPT { return data_ != nullptr; } - - constexpr iterator begin() const { return iterator{this, true}; } - - constexpr iterator end() const { return iterator{this, false}; } - - constexpr const_iterator cbegin() const - { - return const_iterator{reinterpret_cast(this), true}; - } - - constexpr const_iterator cend() const - { - return const_iterator{reinterpret_cast(this), false}; - } - - constexpr reverse_iterator rbegin() const { return reverse_iterator{end()}; } - - constexpr reverse_iterator rend() const { return reverse_iterator{begin()}; } - - constexpr const_reverse_iterator crbegin() const { return const_reverse_iterator{cend()}; } - - constexpr const_reverse_iterator crend() const { return const_reverse_iterator{cbegin()}; } - - template , std::remove_cv_t>::value>> - constexpr bool - operator==(const strided_span& other) const GSL_NOEXCEPT - { - return bounds_.size() == other.bounds_.size() && - (data_ == other.data_ || std::equal(this->begin(), this->end(), other.begin())); - } - - template , std::remove_cv_t>::value>> - constexpr bool - operator!=(const strided_span& other) const GSL_NOEXCEPT - { - return !(*this == other); - } - - template , std::remove_cv_t>::value>> - constexpr bool - operator<(const strided_span& other) const GSL_NOEXCEPT - { - return std::lexicographical_compare(this->begin(), this->end(), other.begin(), other.end()); - } - - template , std::remove_cv_t>::value>> - constexpr bool - operator<=(const strided_span& other) const GSL_NOEXCEPT - { - return !(other < *this); - } - - template , std::remove_cv_t>::value>> - constexpr bool - operator>(const strided_span& other) const GSL_NOEXCEPT - { - return (other < *this); - } - - template , std::remove_cv_t>::value>> - constexpr bool - operator>=(const strided_span& other) const GSL_NOEXCEPT - { - return !(*this < other); - } - -private: - static index_type resize_extent(const index_type& extent, std::ptrdiff_t d) - { - // The last dimension of the array needs to contain a multiple of new type elements - Expects(extent[Rank - 1] >= d && (extent[Rank - 1] % d == 0)); - - index_type ret = extent; - ret[Rank - 1] /= d; - - return ret; - } - - template > - static index_type resize_stride(const index_type& strides, std::ptrdiff_t, void* = nullptr) - { - // Only strided arrays with regular strides can be resized - Expects(strides[Rank - 1] == 1); - - return strides; - } - - template 1), typename = std::enable_if_t> - static index_type resize_stride(const index_type& strides, std::ptrdiff_t d) - { - // Only strided arrays with regular strides can be resized - Expects(strides[Rank - 1] == 1); - // The strides must have contiguous chunks of - // memory that can contain a multiple of new type elements - Expects(strides[Rank - 2] >= d && (strides[Rank - 2] % d == 0)); - - for (std::size_t i = Rank - 1; i > 0; --i) { - // Only strided arrays with regular strides can be resized - Expects((strides[i - 1] >= strides[i]) && (strides[i - 1] % strides[i] == 0)); - } - - index_type ret = strides / d; - ret[Rank - 1] = 1; - - return ret; - } -}; - -template -class contiguous_span_iterator -{ -public: - using iterator_category = std::random_access_iterator_tag; - using value_type = typename Span::value_type; - using difference_type = std::ptrdiff_t; - using pointer = value_type*; - using reference = value_type&; - -private: - template - friend class multi_span; - - pointer data_; - const Span* m_validator; - void validateThis() const - { - // iterator is out of range of the array - Expects(data_ >= m_validator->data_ && data_ < m_validator->data_ + m_validator->size()); - } - contiguous_span_iterator(const Span* container, bool isbegin) - : data_(isbegin ? container->data_ : container->data_ + container->size()) - , m_validator(container) - { - } - -public: - reference operator*() const GSL_NOEXCEPT - { - validateThis(); - return *data_; - } - pointer operator->() const GSL_NOEXCEPT - { - validateThis(); - return data_; - } - contiguous_span_iterator& operator++() GSL_NOEXCEPT - { - ++data_; - return *this; - } - contiguous_span_iterator operator++(int) GSL_NOEXCEPT - { - auto ret = *this; - ++(*this); - return ret; - } - contiguous_span_iterator& operator--() GSL_NOEXCEPT - { - --data_; - return *this; - } - contiguous_span_iterator operator--(int) GSL_NOEXCEPT - { - auto ret = *this; - --(*this); - return ret; - } - contiguous_span_iterator operator+(difference_type n) const GSL_NOEXCEPT - { - contiguous_span_iterator ret{*this}; - return ret += n; - } - contiguous_span_iterator& operator+=(difference_type n) GSL_NOEXCEPT - { - data_ += n; - return *this; - } - contiguous_span_iterator operator-(difference_type n) const GSL_NOEXCEPT - { - contiguous_span_iterator ret{*this}; - return ret -= n; - } - contiguous_span_iterator& operator-=(difference_type n) GSL_NOEXCEPT { return *this += -n; } - difference_type operator-(const contiguous_span_iterator& rhs) const GSL_NOEXCEPT - { - Expects(m_validator == rhs.m_validator); - return data_ - rhs.data_; - } - reference operator[](difference_type n) const GSL_NOEXCEPT { return *(*this + n); } - bool operator==(const contiguous_span_iterator& rhs) const GSL_NOEXCEPT - { - Expects(m_validator == rhs.m_validator); - return data_ == rhs.data_; - } - bool operator!=(const contiguous_span_iterator& rhs) const GSL_NOEXCEPT - { - return !(*this == rhs); - } - bool operator<(const contiguous_span_iterator& rhs) const GSL_NOEXCEPT - { - Expects(m_validator == rhs.m_validator); - return data_ < rhs.data_; - } - bool operator<=(const contiguous_span_iterator& rhs) const GSL_NOEXCEPT - { - return !(rhs < *this); - } - bool operator>(const contiguous_span_iterator& rhs) const GSL_NOEXCEPT { return rhs < *this; } - bool operator>=(const contiguous_span_iterator& rhs) const GSL_NOEXCEPT - { - return !(rhs > *this); - } - void swap(contiguous_span_iterator& rhs) GSL_NOEXCEPT - { - std::swap(data_, rhs.data_); - std::swap(m_validator, rhs.m_validator); - } -}; - -template -contiguous_span_iterator operator+(typename contiguous_span_iterator::difference_type n, - const contiguous_span_iterator& rhs) GSL_NOEXCEPT -{ - return rhs + n; -} - -template -class general_span_iterator -{ -public: - using iterator_category = std::random_access_iterator_tag; - using value_type = typename Span::value_type; - using difference_type = std::ptrdiff_t; - using pointer = value_type*; - using reference = value_type&; - -private: - template - friend class strided_span; - - const Span* m_container; - typename Span::bounds_type::iterator m_itr; - general_span_iterator(const Span* container, bool isbegin) - : m_container(container) - , m_itr(isbegin ? m_container->bounds().begin() : m_container->bounds().end()) - { - } - -public: - reference operator*() GSL_NOEXCEPT { return (*m_container)[*m_itr]; } - pointer operator->() GSL_NOEXCEPT { return &(*m_container)[*m_itr]; } - general_span_iterator& operator++() GSL_NOEXCEPT - { - ++m_itr; - return *this; - } - general_span_iterator operator++(int) GSL_NOEXCEPT - { - auto ret = *this; - ++(*this); - return ret; - } - general_span_iterator& operator--() GSL_NOEXCEPT - { - --m_itr; - return *this; - } - general_span_iterator operator--(int) GSL_NOEXCEPT - { - auto ret = *this; - --(*this); - return ret; - } - general_span_iterator operator+(difference_type n) const GSL_NOEXCEPT - { - general_span_iterator ret{*this}; - return ret += n; - } - general_span_iterator& operator+=(difference_type n) GSL_NOEXCEPT - { - m_itr += n; - return *this; - } - general_span_iterator operator-(difference_type n) const GSL_NOEXCEPT - { - general_span_iterator ret{*this}; - return ret -= n; - } - general_span_iterator& operator-=(difference_type n) GSL_NOEXCEPT { return *this += -n; } - difference_type operator-(const general_span_iterator& rhs) const GSL_NOEXCEPT - { - Expects(m_container == rhs.m_container); - return m_itr - rhs.m_itr; - } - value_type operator[](difference_type n) const GSL_NOEXCEPT { return (*m_container)[m_itr[n]]; } - - bool operator==(const general_span_iterator& rhs) const GSL_NOEXCEPT - { - Expects(m_container == rhs.m_container); - return m_itr == rhs.m_itr; - } - bool operator!=(const general_span_iterator& rhs) const GSL_NOEXCEPT { return !(*this == rhs); } - bool operator<(const general_span_iterator& rhs) const GSL_NOEXCEPT - { - Expects(m_container == rhs.m_container); - return m_itr < rhs.m_itr; - } - bool operator<=(const general_span_iterator& rhs) const GSL_NOEXCEPT { return !(rhs < *this); } - bool operator>(const general_span_iterator& rhs) const GSL_NOEXCEPT { return rhs < *this; } - bool operator>=(const general_span_iterator& rhs) const GSL_NOEXCEPT { return !(rhs > *this); } - void swap(general_span_iterator& rhs) GSL_NOEXCEPT - { - std::swap(m_itr, rhs.m_itr); - std::swap(m_container, rhs.m_container); - } -}; - -template -general_span_iterator operator+(typename general_span_iterator::difference_type n, - const general_span_iterator& rhs) GSL_NOEXCEPT -{ - return rhs + n; -} - -} // namespace gsl - -#undef GSL_NOEXCEPT - -#ifdef _MSC_VER -#if _MSC_VER < 1910 - -#undef constexpr -#pragma pop_macro("constexpr") -#endif // _MSC_VER < 1910 - -#pragma warning(pop) - -#endif // _MSC_VER - -#if __GNUC__ > 6 -#pragma GCC diagnostic pop -#endif // __GNUC__ > 6 - -#endif // GSL_MULTI_SPAN_H diff --git a/src/structures/lsmtree/segments/gsl/pointers b/src/structures/lsmtree/segments/gsl/pointers deleted file mode 100644 index 517b8f2..0000000 --- a/src/structures/lsmtree/segments/gsl/pointers +++ /dev/null @@ -1,193 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -// Copyright (c) 2015 Microsoft Corporation. All rights reserved. -// -// This code is licensed under the MIT License (MIT). -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef GSL_POINTERS_H -#define GSL_POINTERS_H - -#include "gsl_assert" // for Ensures, Expects - -#include // for forward -#include // for ptrdiff_t, nullptr_t, ostream, size_t -#include // for shared_ptr, unique_ptr -#include // for hash -#include // for enable_if_t, is_convertible, is_assignable - -#if defined(_MSC_VER) && _MSC_VER < 1910 -#pragma push_macro("constexpr") -#define constexpr /*constexpr*/ - -#endif // defined(_MSC_VER) && _MSC_VER < 1910 - -namespace gsl -{ - -// -// GSL.owner: ownership pointers -// -using std::unique_ptr; -using std::shared_ptr; - -// -// owner -// -// owner is designed as a bridge for code that must deal directly with owning pointers for some reason -// -// T must be a pointer type -// - disallow construction from any type other than pointer type -// -template ::value>> -using owner = T; - -// -// not_null -// -// Restricts a pointer or smart pointer to only hold non-null values. -// -// Has zero size overhead over T. -// -// If T is a pointer (i.e. T == U*) then -// - allow construction from U* -// - disallow construction from nullptr_t -// - disallow default construction -// - ensure construction from null U* fails -// - allow implicit conversion to U* -// -template -class not_null -{ -public: - static_assert(std::is_assignable::value, "T cannot be assigned nullptr."); - - template ::value>> - constexpr explicit not_null(U&& u) : ptr_(std::forward(u)) - { - Expects(ptr_ != nullptr); - } - - template ::value>> - constexpr explicit not_null(T u) : ptr_(u) - { - Expects(ptr_ != nullptr); - } - - template ::value>> - constexpr not_null(const not_null& other) : not_null(other.get()) - { - } - - not_null(not_null&& other) = default; - not_null(const not_null& other) = default; - not_null& operator=(const not_null& other) = default; - - constexpr T get() const - { - Ensures(ptr_ != nullptr); - return ptr_; - } - - constexpr operator T() const { return get(); } - constexpr T operator->() const { return get(); } - constexpr decltype(auto) operator*() const { return *get(); } - - // prevents compilation when someone attempts to assign a null pointer constant - not_null(std::nullptr_t) = delete; - not_null& operator=(std::nullptr_t) = delete; - - // unwanted operators...pointers only point to single objects! - not_null& operator++() = delete; - not_null& operator--() = delete; - not_null operator++(int) = delete; - not_null operator--(int) = delete; - not_null& operator+=(std::ptrdiff_t) = delete; - not_null& operator-=(std::ptrdiff_t) = delete; - void operator[](std::ptrdiff_t) const = delete; - -private: - T ptr_; -}; - -template -std::ostream& operator<<(std::ostream& os, const not_null& val) -{ - os << val.get(); - return os; -} - -template -auto operator==(const not_null& lhs, const not_null& rhs) -> decltype(lhs.get() == rhs.get()) -{ - return lhs.get() == rhs.get(); -} - -template -auto operator!=(const not_null& lhs, const not_null& rhs) -> decltype(lhs.get() != rhs.get()) -{ - return lhs.get() != rhs.get(); -} - -template -auto operator<(const not_null& lhs, const not_null& rhs) -> decltype(lhs.get() < rhs.get()) -{ - return lhs.get() < rhs.get(); -} - -template -auto operator<=(const not_null& lhs, const not_null& rhs) -> decltype(lhs.get() <= rhs.get()) -{ - return lhs.get() <= rhs.get(); -} - -template -auto operator>(const not_null& lhs, const not_null& rhs) -> decltype(lhs.get() > rhs.get()) -{ - return lhs.get() > rhs.get(); -} - -template -auto operator>=(const not_null& lhs, const not_null& rhs) -> decltype(lhs.get() >= rhs.get()) -{ - return lhs.get() >= rhs.get(); -} - -// more unwanted operators -template -std::ptrdiff_t operator-(const not_null&, const not_null&) = delete; -template -not_null operator-(const not_null&, std::ptrdiff_t) = delete; -template -not_null operator+(const not_null&, std::ptrdiff_t) = delete; -template -not_null operator+(std::ptrdiff_t, const not_null&) = delete; - -} // namespace gsl - -namespace std -{ -template -struct hash> -{ - std::size_t operator()(const gsl::not_null& value) const { return hash{}(value); } -}; - -} // namespace std - -#if defined(_MSC_VER) && _MSC_VER < 1910 -#undef constexpr -#pragma pop_macro("constexpr") - -#endif // defined(_MSC_VER) && _MSC_VER < 1910 - -#endif // GSL_POINTERS_H diff --git a/src/structures/lsmtree/segments/gsl/span b/src/structures/lsmtree/segments/gsl/span deleted file mode 100644 index 7e3ac68..0000000 --- a/src/structures/lsmtree/segments/gsl/span +++ /dev/null @@ -1,766 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -// Copyright (c) 2015 Microsoft Corporation. All rights reserved. -// -// This code is licensed under the MIT License (MIT). -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef GSL_SPAN_H -#define GSL_SPAN_H - -#include "gsl_assert" // for Expects -#include "gsl_byte" // for byte -#include "gsl_util" // for narrow_cast, narrow - -#include // for lexicographical_compare -#include // for array -#include // for ptrdiff_t, size_t, nullptr_t -#include // for reverse_iterator, distance, random_access_... -#include -#include -#include // for enable_if_t, declval, is_convertible, inte... -#include - -#ifdef _MSC_VER -#pragma warning(push) - -// turn off some warnings that are noisy about our Expects statements -#pragma warning(disable : 4127) // conditional expression is constant -#pragma warning(disable : 4702) // unreachable code - -// blanket turn off warnings from CppCoreCheck for now -// so people aren't annoyed by them when running the tool. -// more targeted suppressions will be added in a future update to the GSL -#pragma warning(disable : 26481 26482 26483 26485 26490 26491 26492 26493 26495) - -#if _MSC_VER < 1910 -#pragma push_macro("constexpr") -#define constexpr /*constexpr*/ -#define GSL_USE_STATIC_CONSTEXPR_WORKAROUND - -#endif // _MSC_VER < 1910 -#else // _MSC_VER - -// See if we have enough C++17 power to use a static constexpr data member -// without needing an out-of-line definition -#if !(defined(__cplusplus) && (__cplusplus >= 201703L)) -#define GSL_USE_STATIC_CONSTEXPR_WORKAROUND -#endif // !(defined(__cplusplus) && (__cplusplus >= 201703L)) - -#endif // _MSC_VER - -// GCC 7 does not like the signed unsigned missmatch (size_t ptrdiff_t) -// While there is a conversion from signed to unsigned, it happens at -// compiletime, so the compiler wouldn't have to warn indiscriminently, but -// could check if the source value actually doesn't fit into the target type -// and only warn in those cases. -#if __GNUC__ > 6 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wsign-conversion" -#endif - -namespace gsl -{ - -// [views.constants], constants -constexpr const std::ptrdiff_t dynamic_extent = -1; - -template -class span; - -// implementation details -namespace details -{ - template - struct is_span_oracle : std::false_type - { - }; - - template - struct is_span_oracle> : std::true_type - { - }; - - template - struct is_span : public is_span_oracle> - { - }; - - template - struct is_std_array_oracle : std::false_type - { - }; - - template - struct is_std_array_oracle> : std::true_type - { - }; - - template - struct is_std_array : public is_std_array_oracle> - { - }; - - template - struct is_allowed_extent_conversion - : public std::integral_constant - { - }; - - template - struct is_allowed_element_type_conversion - : public std::integral_constant::value> - { - }; - - template - class span_iterator - { - using element_type_ = typename Span::element_type; - - public: - -#ifdef _MSC_VER - // Tell Microsoft standard library that span_iterators are checked. - using _Unchecked_type = typename Span::pointer; -#endif - - using iterator_category = std::random_access_iterator_tag; - using value_type = std::remove_cv_t; - using difference_type = typename Span::index_type; - - using reference = std::conditional_t&; - using pointer = std::add_pointer_t; - - span_iterator() = default; - - constexpr span_iterator(const Span* span, typename Span::index_type idx) noexcept - : span_(span), index_(idx) - {} - - friend span_iterator; - template* = nullptr> - constexpr span_iterator(const span_iterator& other) noexcept - : span_iterator(other.span_, other.index_) - { - } - - constexpr reference operator*() const - { - Expects(index_ != span_->size()); - return *(span_->data() + index_); - } - - constexpr pointer operator->() const - { - Expects(index_ != span_->size()); - return span_->data() + index_; - } - - constexpr span_iterator& operator++() - { - Expects(0 <= index_ && index_ != span_->size()); - ++index_; - return *this; - } - - constexpr span_iterator operator++(int) - { - auto ret = *this; - ++(*this); - return ret; - } - - constexpr span_iterator& operator--() - { - Expects(index_ != 0 && index_ <= span_->size()); - --index_; - return *this; - } - - constexpr span_iterator operator--(int) - { - auto ret = *this; - --(*this); - return ret; - } - - constexpr span_iterator operator+(difference_type n) const - { - auto ret = *this; - return ret += n; - } - - friend constexpr span_iterator operator+(difference_type n, span_iterator const& rhs) - { - return rhs + n; - } - - constexpr span_iterator& operator+=(difference_type n) - { - Expects((index_ + n) >= 0 && (index_ + n) <= span_->size()); - index_ += n; - return *this; - } - - constexpr span_iterator operator-(difference_type n) const - { - auto ret = *this; - return ret -= n; - } - - constexpr span_iterator& operator-=(difference_type n) { return *this += -n; } - - constexpr difference_type operator-(span_iterator rhs) const - { - Expects(span_ == rhs.span_); - return index_ - rhs.index_; - } - - constexpr reference operator[](difference_type n) const - { - return *(*this + n); - } - - constexpr friend bool operator==(span_iterator lhs, - span_iterator rhs) noexcept - { - return lhs.span_ == rhs.span_ && lhs.index_ == rhs.index_; - } - - constexpr friend bool operator!=(span_iterator lhs, - span_iterator rhs) noexcept - { - return !(lhs == rhs); - } - - constexpr friend bool operator<(span_iterator lhs, - span_iterator rhs) noexcept - { - return lhs.index_ < rhs.index_; - } - - constexpr friend bool operator<=(span_iterator lhs, - span_iterator rhs) noexcept - { - return !(rhs < lhs); - } - - constexpr friend bool operator>(span_iterator lhs, - span_iterator rhs) noexcept - { - return rhs < lhs; - } - - constexpr friend bool operator>=(span_iterator lhs, - span_iterator rhs) noexcept - { - return !(rhs > lhs); - } - -#ifdef _MSC_VER - // MSVC++ iterator debugging support; allows STL algorithms in 15.8+ - // to unwrap span_iterator to a pointer type after a range check in STL - // algorithm calls - friend constexpr void _Verify_range(span_iterator lhs, - span_iterator rhs) noexcept - { // test that [lhs, rhs) forms a valid range inside an STL algorithm - Expects(lhs.span_ == rhs.span_ // range spans have to match - && lhs.index_ <= rhs.index_); // range must not be transposed - } - - constexpr void _Verify_offset(const difference_type n) const noexcept - { // test that the iterator *this + n is a valid range in an STL - // algorithm call - Expects((index_ + n) >= 0 && (index_ + n) <= span_->size()); - } - - constexpr pointer _Unwrapped() const noexcept - { // after seeking *this to a high water mark, or using one of the - // _Verify_xxx functions above, unwrap this span_iterator to a raw - // pointer - return span_->data() + index_; - } - - // Tell the STL that span_iterator should not be unwrapped if it can't - // validate in advance, even in release / optimized builds: -#if defined(GSL_USE_STATIC_CONSTEXPR_WORKAROUND) - static constexpr const bool _Unwrap_when_unverified = false; -#else - static constexpr bool _Unwrap_when_unverified = false; -#endif - constexpr void _Seek_to(const pointer p) noexcept - { // adjust the position of *this to previously verified location p - // after _Unwrapped - index_ = p - span_->data(); - } -#endif - - protected: - const Span* span_ = nullptr; - std::ptrdiff_t index_ = 0; - }; - - template - class extent_type - { - public: - using index_type = std::ptrdiff_t; - - static_assert(Ext >= 0, "A fixed-size span must be >= 0 in size."); - - constexpr extent_type() noexcept {} - - template - constexpr extent_type(extent_type ext) - { - static_assert(Other == Ext || Other == dynamic_extent, - "Mismatch between fixed-size extent and size of initializing data."); - Expects(ext.size() == Ext); - } - - constexpr extent_type(index_type size) { Expects(size == Ext); } - - constexpr index_type size() const noexcept { return Ext; } - }; - - template <> - class extent_type - { - public: - using index_type = std::ptrdiff_t; - - template - explicit constexpr extent_type(extent_type ext) : size_(ext.size()) - { - } - - explicit constexpr extent_type(index_type size) : size_(size) { Expects(size >= 0); } - - constexpr index_type size() const noexcept { return size_; } - - private: - index_type size_; - }; - - template - struct calculate_subspan_type - { - using type = span; - }; -} // namespace details - -// [span], class template span -template -class span -{ -public: - // constants and types - using element_type = ElementType; - using value_type = std::remove_cv_t; - using index_type = std::ptrdiff_t; - using pointer = element_type*; - using reference = element_type&; - - using iterator = details::span_iterator, false>; - using const_iterator = details::span_iterator, true>; - using reverse_iterator = std::reverse_iterator; - using const_reverse_iterator = std::reverse_iterator; - - using size_type = index_type; - -#if defined(GSL_USE_STATIC_CONSTEXPR_WORKAROUND) - static constexpr const index_type extent { Extent }; -#else - static constexpr index_type extent { Extent }; -#endif - - // [span.cons], span constructors, copy, assignment, and destructor - template " SFINAE, - // since "std::enable_if_t" is ill-formed when Extent is greater than 0. - class = std::enable_if_t<(Dependent || Extent <= 0)>> - constexpr span() noexcept : storage_(nullptr, details::extent_type<0>()) - { - } - - constexpr span(pointer ptr, index_type count) : storage_(ptr, count) {} - - constexpr span(pointer firstElem, pointer lastElem) - : storage_(firstElem, std::distance(firstElem, lastElem)) - { - } - - template - constexpr span(element_type (&arr)[N]) noexcept - : storage_(KnownNotNull{&arr[0]}, details::extent_type()) - { - } - - template > - constexpr span(std::array& arr) noexcept - : storage_(&arr[0], details::extent_type()) - { - } - - template - constexpr span(const std::array, N>& arr) noexcept - : storage_(&arr[0], details::extent_type()) - { - } - - // NB: the SFINAE here uses .data() as a incomplete/imperfect proxy for the requirement - // on Container to be a contiguous sequence container. - template ::value && !details::is_std_array::value && - std::is_convertible::value && - std::is_convertible().data())>::value>> - constexpr span(Container& cont) : span(cont.data(), narrow(cont.size())) - { - } - - template ::value && !details::is_span::value && - std::is_convertible::value && - std::is_convertible().data())>::value>> - constexpr span(const Container& cont) : span(cont.data(), narrow(cont.size())) - { - } - - constexpr span(const span& other) noexcept = default; - - template < - class OtherElementType, std::ptrdiff_t OtherExtent, - class = std::enable_if_t< - details::is_allowed_extent_conversion::value && - details::is_allowed_element_type_conversion::value>> - constexpr span(const span& other) - : storage_(other.data(), details::extent_type(other.size())) - { - } - - ~span() noexcept = default; - constexpr span& operator=(const span& other) noexcept = default; - - // [span.sub], span subviews - template - constexpr span first() const - { - Expects(Count >= 0 && Count <= size()); - return {data(), Count}; - } - - template - constexpr span last() const - { - Expects(Count >= 0 && size() - Count >= 0); - return {data() + (size() - Count), Count}; - } - - template - constexpr auto subspan() const -> typename details::calculate_subspan_type::type - { - Expects((Offset >= 0 && size() - Offset >= 0) && - (Count == dynamic_extent || (Count >= 0 && Offset + Count <= size()))); - - return {data() + Offset, Count == dynamic_extent ? size() - Offset : Count}; - } - - constexpr span first(index_type count) const - { - Expects(count >= 0 && count <= size()); - return {data(), count}; - } - - constexpr span last(index_type count) const - { - return make_subspan(size() - count, dynamic_extent, subspan_selector{}); - } - - constexpr span subspan(index_type offset, - index_type count = dynamic_extent) const - { - return make_subspan(offset, count, subspan_selector{}); - } - - - // [span.obs], span observers - constexpr index_type size() const noexcept { return storage_.size(); } - constexpr index_type size_bytes() const noexcept - { - return size() * narrow_cast(sizeof(element_type)); - } - constexpr bool empty() const noexcept { return size() == 0; } - - // [span.elem], span element access - constexpr reference operator[](index_type idx) const - { - Expects(idx >= 0 && idx < storage_.size()); - return data()[idx]; - } - - constexpr reference at(index_type idx) const { return this->operator[](idx); } - constexpr reference operator()(index_type idx) const { return this->operator[](idx); } - constexpr pointer data() const noexcept { return storage_.data(); } - - // [span.iter], span iterator support - constexpr iterator begin() const noexcept { return {this, 0}; } - constexpr iterator end() const noexcept { return {this, size()}; } - - constexpr const_iterator cbegin() const noexcept { return {this, 0}; } - constexpr const_iterator cend() const noexcept { return {this, size()}; } - - constexpr reverse_iterator rbegin() const noexcept { return reverse_iterator{end()}; } - constexpr reverse_iterator rend() const noexcept { return reverse_iterator{begin()}; } - - constexpr const_reverse_iterator crbegin() const noexcept { return const_reverse_iterator{cend()}; } - constexpr const_reverse_iterator crend() const noexcept { return const_reverse_iterator{cbegin()}; } - -#ifdef _MSC_VER - // Tell MSVC how to unwrap spans in range-based-for - constexpr pointer _Unchecked_begin() const noexcept { return data(); } - constexpr pointer _Unchecked_end() const noexcept { return data() + size(); } -#endif // _MSC_VER - -private: - - // Needed to remove unnecessary null check in subspans - struct KnownNotNull - { - pointer p; - }; - - // this implementation detail class lets us take advantage of the - // empty base class optimization to pay for only storage of a single - // pointer in the case of fixed-size spans - template - class storage_type : public ExtentType - { - public: - // KnownNotNull parameter is needed to remove unnecessary null check - // in subspans and constructors from arrays - template - constexpr storage_type(KnownNotNull data, OtherExtentType ext) : ExtentType(ext), data_(data.p) - { - Expects(ExtentType::size() >= 0); - } - - - template - constexpr storage_type(pointer data, OtherExtentType ext) : ExtentType(ext), data_(data) - { - Expects(ExtentType::size() >= 0); - Expects(data || ExtentType::size() == 0); - } - - constexpr pointer data() const noexcept { return data_; } - - private: - pointer data_; - }; - - storage_type> storage_; - - // The rest is needed to remove unnecessary null check - // in subspans and constructors from arrays - constexpr span(KnownNotNull ptr, index_type count) : storage_(ptr, count) {} - - template - class subspan_selector {}; - - template - span make_subspan(index_type offset, - index_type count, - subspan_selector) const - { - span tmp(*this); - return tmp.subspan(offset, count); - } - - span make_subspan(index_type offset, - index_type count, - subspan_selector) const - { - Expects(offset >= 0 && size() - offset >= 0); - if (count == dynamic_extent) - { - return { KnownNotNull{ data() + offset }, size() - offset }; - } - - Expects(count >= 0 && size() - offset >= count); - return { KnownNotNull{ data() + offset }, count }; - } -}; - -#if defined(GSL_USE_STATIC_CONSTEXPR_WORKAROUND) -template -constexpr const typename span::index_type span::extent; -#endif - - -// [span.comparison], span comparison operators -template -constexpr bool operator==(span l, - span r) -{ - return std::equal(l.begin(), l.end(), r.begin(), r.end()); -} - -template -constexpr bool operator!=(span l, - span r) -{ - return !(l == r); -} - -template -constexpr bool operator<(span l, - span r) -{ - return std::lexicographical_compare(l.begin(), l.end(), r.begin(), r.end()); -} - -template -constexpr bool operator<=(span l, - span r) -{ - return !(l > r); -} - -template -constexpr bool operator>(span l, - span r) -{ - return r < l; -} - -template -constexpr bool operator>=(span l, - span r) -{ - return !(l < r); -} - -namespace details -{ - // if we only supported compilers with good constexpr support then - // this pair of classes could collapse down to a constexpr function - - // we should use a narrow_cast<> to go to std::size_t, but older compilers may not see it as - // constexpr - // and so will fail compilation of the template - template - struct calculate_byte_size - : std::integral_constant(sizeof(ElementType) * - static_cast(Extent))> - { - }; - - template - struct calculate_byte_size - : std::integral_constant - { - }; -} - -// [span.objectrep], views of object representation -template -span::value> -as_bytes(span s) noexcept -{ - return {reinterpret_cast(s.data()), s.size_bytes()}; -} - -template ::value>> -span::value> -as_writeable_bytes(span s) noexcept -{ - return {reinterpret_cast(s.data()), s.size_bytes()}; -} - -// -// make_span() - Utility functions for creating spans -// -template -constexpr span make_span(ElementType* ptr, typename span::index_type count) -{ - return span(ptr, count); -} - -template -constexpr span make_span(ElementType* firstElem, ElementType* lastElem) -{ - return span(firstElem, lastElem); -} - -template -constexpr span make_span(ElementType (&arr)[N]) noexcept -{ - return span(arr); -} - -template -constexpr span make_span(Container& cont) -{ - return span(cont); -} - -template -constexpr span make_span(const Container& cont) -{ - return span(cont); -} - -template -constexpr span make_span(Ptr& cont, std::ptrdiff_t count) -{ - return span(cont, count); -} - -template -constexpr span make_span(Ptr& cont) -{ - return span(cont); -} - -// Specialization of gsl::at for span -template -constexpr ElementType& at(span s, index i) -{ - // No bounds checking here because it is done in span::operator[] called below - return s[i]; -} - -} // namespace gsl - -#ifdef _MSC_VER -#if _MSC_VER < 1910 -#undef constexpr -#pragma pop_macro("constexpr") - -#endif // _MSC_VER < 1910 - -#pragma warning(pop) -#endif // _MSC_VER - -#if __GNUC__ > 6 -#pragma GCC diagnostic pop -#endif // __GNUC__ > 6 - -#endif // GSL_SPAN_H diff --git a/src/structures/lsmtree/segments/gsl/string_span b/src/structures/lsmtree/segments/gsl/string_span deleted file mode 100644 index 8ab0619..0000000 --- a/src/structures/lsmtree/segments/gsl/string_span +++ /dev/null @@ -1,730 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -// Copyright (c) 2015 Microsoft Corporation. All rights reserved. -// -// This code is licensed under the MIT License (MIT). -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef GSL_STRING_SPAN_H -#define GSL_STRING_SPAN_H - -#include "gsl_assert" // for Ensures, Expects -#include "gsl_util" // for narrow_cast -#include "span" // for operator!=, operator==, dynamic_extent - -#include // for equal, lexicographical_compare -#include // for array -#include // for ptrdiff_t, size_t, nullptr_t -#include // for PTRDIFF_MAX -#include -#include // for basic_string, allocator, char_traits -#include // for declval, is_convertible, enable_if_t, add_... - -#ifdef _MSC_VER -#pragma warning(push) - -// blanket turn off warnings from CppCoreCheck for now -// so people aren't annoyed by them when running the tool. -// more targeted suppressions will be added in a future update to the GSL -#pragma warning(disable : 26481 26482 26483 26485 26490 26491 26492 26493 26495) - -#if _MSC_VER < 1910 -#pragma push_macro("constexpr") -#define constexpr /*constexpr*/ - -#endif // _MSC_VER < 1910 -#endif // _MSC_VER - -// In order to test the library, we need it to throw exceptions that we can catch -#ifdef GSL_THROW_ON_CONTRACT_VIOLATION -#define GSL_NOEXCEPT /*noexcept*/ -#else -#define GSL_NOEXCEPT noexcept -#endif // GSL_THROW_ON_CONTRACT_VIOLATION - -namespace gsl -{ -// -// czstring and wzstring -// -// These are "tag" typedefs for C-style strings (i.e. null-terminated character arrays) -// that allow static analysis to help find bugs. -// -// There are no additional features/semantics that we can find a way to add inside the -// type system for these types that will not either incur significant runtime costs or -// (sometimes needlessly) break existing programs when introduced. -// - -template -using basic_zstring = CharT*; - -template -using czstring = basic_zstring; - -template -using cwzstring = basic_zstring; - -template -using cu16zstring = basic_zstring; - -template -using cu32zstring = basic_zstring; - -template -using zstring = basic_zstring; - -template -using wzstring = basic_zstring; - -template -using u16zstring = basic_zstring; - -template -using u32zstring = basic_zstring; - -namespace details -{ - template - std::ptrdiff_t string_length(const CharT* str, std::ptrdiff_t n) - { - if (str == nullptr || n <= 0) return 0; - - const span str_span{str, n}; - - std::ptrdiff_t len = 0; - while (len < n && str_span[len]) len++; - - return len; - } -} - -// -// ensure_sentinel() -// -// Provides a way to obtain an span from a contiguous sequence -// that ends with a (non-inclusive) sentinel value. -// -// Will fail-fast if sentinel cannot be found before max elements are examined. -// -template -span ensure_sentinel(T* seq, std::ptrdiff_t max = PTRDIFF_MAX) -{ - auto cur = seq; - while ((cur - seq) < max && *cur != Sentinel) ++cur; - Ensures(*cur == Sentinel); - return {seq, cur - seq}; -} - -// -// ensure_z - creates a span for a zero terminated strings. -// Will fail fast if a null-terminator cannot be found before -// the limit of size_type. -// -template -span ensure_z(CharT* const& sz, std::ptrdiff_t max = PTRDIFF_MAX) -{ - return ensure_sentinel(sz, max); -} - -template -span ensure_z(CharT (&sz)[N]) -{ - return ensure_z(&sz[0], static_cast(N)); -} - -template -span::type, dynamic_extent> -ensure_z(Cont& cont) -{ - return ensure_z(cont.data(), static_cast(cont.size())); -} - -template -class basic_string_span; - -namespace details -{ - template - struct is_basic_string_span_oracle : std::false_type - { - }; - - template - struct is_basic_string_span_oracle> : std::true_type - { - }; - - template - struct is_basic_string_span : is_basic_string_span_oracle> - { - }; -} - -// -// string_span and relatives -// -template -class basic_string_span -{ -public: - using element_type = CharT; - using pointer = std::add_pointer_t; - using reference = std::add_lvalue_reference_t; - using const_reference = std::add_lvalue_reference_t>; - using impl_type = span; - - using index_type = typename impl_type::index_type; - using iterator = typename impl_type::iterator; - using const_iterator = typename impl_type::const_iterator; - using reverse_iterator = typename impl_type::reverse_iterator; - using const_reverse_iterator = typename impl_type::const_reverse_iterator; - - // default (empty) - constexpr basic_string_span() GSL_NOEXCEPT = default; - - // copy - constexpr basic_string_span(const basic_string_span& other) GSL_NOEXCEPT = default; - - // assign - constexpr basic_string_span& operator=(const basic_string_span& other) GSL_NOEXCEPT = default; - - constexpr basic_string_span(pointer ptr, index_type length) : span_(ptr, length) {} - constexpr basic_string_span(pointer firstElem, pointer lastElem) : span_(firstElem, lastElem) {} - - // From static arrays - if 0-terminated, remove 0 from the view - // All other containers allow 0s within the length, so we do not remove them - template - constexpr basic_string_span(element_type (&arr)[N]) : span_(remove_z(arr)) - { - } - - template > - constexpr basic_string_span(std::array& arr) GSL_NOEXCEPT : span_(arr) - { - } - - template > - constexpr basic_string_span(const std::array& arr) GSL_NOEXCEPT - : span_(arr) - { - } - - // Container signature should work for basic_string after C++17 version exists - template - constexpr basic_string_span(std::basic_string& str) - : span_(&str[0], narrow_cast(str.length())) - { - } - - template - constexpr basic_string_span(const std::basic_string& str) - : span_(&str[0], str.length()) - { - } - - // from containers. Containers must have a pointer type and data() function signatures - template ::value && - std::is_convertible::value && - std::is_convertible().data())>::value>> - constexpr basic_string_span(Container& cont) : span_(cont) - { - } - - template ::value && - std::is_convertible::value && - std::is_convertible().data())>::value>> - constexpr basic_string_span(const Container& cont) : span_(cont) - { - } - - // from string_span - template < - class OtherValueType, std::ptrdiff_t OtherExtent, - class = std::enable_if_t::impl_type, impl_type>::value>> - constexpr basic_string_span(basic_string_span other) - : span_(other.data(), other.length()) - { - } - - template - constexpr basic_string_span first() const - { - return {span_.template first()}; - } - - constexpr basic_string_span first(index_type count) const - { - return {span_.first(count)}; - } - - template - constexpr basic_string_span last() const - { - return {span_.template last()}; - } - - constexpr basic_string_span last(index_type count) const - { - return {span_.last(count)}; - } - - template - constexpr basic_string_span subspan() const - { - return {span_.template subspan()}; - } - - constexpr basic_string_span - subspan(index_type offset, index_type count = dynamic_extent) const - { - return {span_.subspan(offset, count)}; - } - - constexpr reference operator[](index_type idx) const { return span_[idx]; } - constexpr reference operator()(index_type idx) const { return span_[idx]; } - - constexpr pointer data() const { return span_.data(); } - - constexpr index_type length() const GSL_NOEXCEPT { return span_.size(); } - constexpr index_type size() const GSL_NOEXCEPT { return span_.size(); } - constexpr index_type size_bytes() const GSL_NOEXCEPT { return span_.size_bytes(); } - constexpr index_type length_bytes() const GSL_NOEXCEPT { return span_.length_bytes(); } - constexpr bool empty() const GSL_NOEXCEPT { return size() == 0; } - - constexpr iterator begin() const GSL_NOEXCEPT { return span_.begin(); } - constexpr iterator end() const GSL_NOEXCEPT { return span_.end(); } - - constexpr const_iterator cbegin() const GSL_NOEXCEPT { return span_.cbegin(); } - constexpr const_iterator cend() const GSL_NOEXCEPT { return span_.cend(); } - - constexpr reverse_iterator rbegin() const GSL_NOEXCEPT { return span_.rbegin(); } - constexpr reverse_iterator rend() const GSL_NOEXCEPT { return span_.rend(); } - - constexpr const_reverse_iterator crbegin() const GSL_NOEXCEPT { return span_.crbegin(); } - constexpr const_reverse_iterator crend() const GSL_NOEXCEPT { return span_.crend(); } - -private: - static impl_type remove_z(pointer const& sz, std::ptrdiff_t max) - { - return {sz, details::string_length(sz, max)}; - } - - template - static impl_type remove_z(element_type (&sz)[N]) - { - return remove_z(&sz[0], narrow_cast(N)); - } - - impl_type span_; -}; - -template -using string_span = basic_string_span; - -template -using cstring_span = basic_string_span; - -template -using wstring_span = basic_string_span; - -template -using cwstring_span = basic_string_span; - -template -using u16string_span = basic_string_span; - -template -using cu16string_span = basic_string_span; - -template -using u32string_span = basic_string_span; - -template -using cu32string_span = basic_string_span; - -// -// to_string() allow (explicit) conversions from string_span to string -// - -template -std::basic_string::type> -to_string(basic_string_span view) -{ - return {view.data(), static_cast(view.length())}; -} - -template , - typename Allocator = std::allocator, typename gCharT, std::ptrdiff_t Extent> -std::basic_string to_basic_string(basic_string_span view) -{ - return {view.data(), static_cast(view.length())}; -} - -template -basic_string_span::value> -as_bytes(basic_string_span s) noexcept -{ - return { reinterpret_cast(s.data()), s.size_bytes() }; -} - -template ::value>> -basic_string_span::value> -as_writeable_bytes(basic_string_span s) noexcept -{ - return {reinterpret_cast(s.data()), s.size_bytes()}; -} - -// zero-terminated string span, used to convert -// zero-terminated spans to legacy strings -template -class basic_zstring_span -{ -public: - using value_type = CharT; - using const_value_type = std::add_const_t; - - using pointer = std::add_pointer_t; - using const_pointer = std::add_pointer_t; - - using zstring_type = basic_zstring; - using const_zstring_type = basic_zstring; - - using impl_type = span; - using string_span_type = basic_string_span; - - constexpr basic_zstring_span(impl_type s) GSL_NOEXCEPT : span_(s) - { - // expects a zero-terminated span - Expects(s[s.size() - 1] == '\0'); - } - - // copy - constexpr basic_zstring_span(const basic_zstring_span& other) = default; - - // move - constexpr basic_zstring_span(basic_zstring_span&& other) = default; - - // assign - constexpr basic_zstring_span& operator=(const basic_zstring_span& other) = default; - - // move assign - constexpr basic_zstring_span& operator=(basic_zstring_span&& other) = default; - - constexpr bool empty() const GSL_NOEXCEPT { return span_.size() == 0; } - - constexpr string_span_type as_string_span() const GSL_NOEXCEPT - { - auto sz = span_.size(); - return { span_.data(), sz > 1 ? sz - 1 : 0 }; - } - constexpr string_span_type ensure_z() const GSL_NOEXCEPT { return gsl::ensure_z(span_); } - - constexpr const_zstring_type assume_z() const GSL_NOEXCEPT { return span_.data(); } - -private: - impl_type span_; -}; - -template -using zstring_span = basic_zstring_span; - -template -using wzstring_span = basic_zstring_span; - -template -using u16zstring_span = basic_zstring_span; - -template -using u32zstring_span = basic_zstring_span; - -template -using czstring_span = basic_zstring_span; - -template -using cwzstring_span = basic_zstring_span; - -template -using cu16zstring_span = basic_zstring_span; - -template -using cu32zstring_span = basic_zstring_span; - -// operator == -template ::value || - std::is_convertible>>::value>> -bool operator==(const gsl::basic_string_span& one, const T& other) GSL_NOEXCEPT -{ - const gsl::basic_string_span> tmp(other); - return std::equal(one.begin(), one.end(), tmp.begin(), tmp.end()); -} - -template ::value && - std::is_convertible>>::value>> -bool operator==(const T& one, const gsl::basic_string_span& other) GSL_NOEXCEPT -{ - gsl::basic_string_span> tmp(one); - return std::equal(tmp.begin(), tmp.end(), other.begin(), other.end()); -} - -// operator != -template , Extent>>::value>> -bool operator!=(gsl::basic_string_span one, const T& other) GSL_NOEXCEPT -{ - return !(one == other); -} - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename = std::enable_if_t< - std::is_convertible, Extent>>::value && - !gsl::details::is_basic_string_span::value>> -bool operator!=(const T& one, gsl::basic_string_span other) GSL_NOEXCEPT -{ - return !(one == other); -} - -// operator< -template , Extent>>::value>> -bool operator<(gsl::basic_string_span one, const T& other) GSL_NOEXCEPT -{ - const gsl::basic_string_span, Extent> tmp(other); - return std::lexicographical_compare(one.begin(), one.end(), tmp.begin(), tmp.end()); -} - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename = std::enable_if_t< - std::is_convertible, Extent>>::value && - !gsl::details::is_basic_string_span::value>> -bool operator<(const T& one, gsl::basic_string_span other) GSL_NOEXCEPT -{ - gsl::basic_string_span, Extent> tmp(one); - return std::lexicographical_compare(tmp.begin(), tmp.end(), other.begin(), other.end()); -} - -#ifndef _MSC_VER - -// VS treats temp and const containers as convertible to basic_string_span, -// so the cases below are already covered by the previous operators - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename DataType = typename T::value_type, - typename = std::enable_if_t< - !gsl::details::is_span::value && !gsl::details::is_basic_string_span::value && - std::is_convertible::value && - std::is_same().size(), *std::declval().data())>, - DataType>::value>> -bool operator<(gsl::basic_string_span one, const T& other) GSL_NOEXCEPT -{ - gsl::basic_string_span, Extent> tmp(other); - return std::lexicographical_compare(one.begin(), one.end(), tmp.begin(), tmp.end()); -} - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename DataType = typename T::value_type, - typename = std::enable_if_t< - !gsl::details::is_span::value && !gsl::details::is_basic_string_span::value && - std::is_convertible::value && - std::is_same().size(), *std::declval().data())>, - DataType>::value>> -bool operator<(const T& one, gsl::basic_string_span other) GSL_NOEXCEPT -{ - gsl::basic_string_span, Extent> tmp(one); - return std::lexicographical_compare(tmp.begin(), tmp.end(), other.begin(), other.end()); -} -#endif - -// operator <= -template , Extent>>::value>> -bool operator<=(gsl::basic_string_span one, const T& other) GSL_NOEXCEPT -{ - return !(other < one); -} - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename = std::enable_if_t< - std::is_convertible, Extent>>::value && - !gsl::details::is_basic_string_span::value>> -bool operator<=(const T& one, gsl::basic_string_span other) GSL_NOEXCEPT -{ - return !(other < one); -} - -#ifndef _MSC_VER - -// VS treats temp and const containers as convertible to basic_string_span, -// so the cases below are already covered by the previous operators - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename DataType = typename T::value_type, - typename = std::enable_if_t< - !gsl::details::is_span::value && !gsl::details::is_basic_string_span::value && - std::is_convertible::value && - std::is_same().size(), *std::declval().data())>, - DataType>::value>> -bool operator<=(gsl::basic_string_span one, const T& other) GSL_NOEXCEPT -{ - return !(other < one); -} - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename DataType = typename T::value_type, - typename = std::enable_if_t< - !gsl::details::is_span::value && !gsl::details::is_basic_string_span::value && - std::is_convertible::value && - std::is_same().size(), *std::declval().data())>, - DataType>::value>> -bool operator<=(const T& one, gsl::basic_string_span other) GSL_NOEXCEPT -{ - return !(other < one); -} -#endif - -// operator> -template , Extent>>::value>> -bool operator>(gsl::basic_string_span one, const T& other) GSL_NOEXCEPT -{ - return other < one; -} - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename = std::enable_if_t< - std::is_convertible, Extent>>::value && - !gsl::details::is_basic_string_span::value>> -bool operator>(const T& one, gsl::basic_string_span other) GSL_NOEXCEPT -{ - return other < one; -} - -#ifndef _MSC_VER - -// VS treats temp and const containers as convertible to basic_string_span, -// so the cases below are already covered by the previous operators - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename DataType = typename T::value_type, - typename = std::enable_if_t< - !gsl::details::is_span::value && !gsl::details::is_basic_string_span::value && - std::is_convertible::value && - std::is_same().size(), *std::declval().data())>, - DataType>::value>> -bool operator>(gsl::basic_string_span one, const T& other) GSL_NOEXCEPT -{ - return other < one; -} - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename DataType = typename T::value_type, - typename = std::enable_if_t< - !gsl::details::is_span::value && !gsl::details::is_basic_string_span::value && - std::is_convertible::value && - std::is_same().size(), *std::declval().data())>, - DataType>::value>> -bool operator>(const T& one, gsl::basic_string_span other) GSL_NOEXCEPT -{ - return other < one; -} -#endif - -// operator >= -template , Extent>>::value>> -bool operator>=(gsl::basic_string_span one, const T& other) GSL_NOEXCEPT -{ - return !(one < other); -} - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename = std::enable_if_t< - std::is_convertible, Extent>>::value && - !gsl::details::is_basic_string_span::value>> -bool operator>=(const T& one, gsl::basic_string_span other) GSL_NOEXCEPT -{ - return !(one < other); -} - -#ifndef _MSC_VER - -// VS treats temp and const containers as convertible to basic_string_span, -// so the cases below are already covered by the previous operators - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename DataType = typename T::value_type, - typename = std::enable_if_t< - !gsl::details::is_span::value && !gsl::details::is_basic_string_span::value && - std::is_convertible::value && - std::is_same().size(), *std::declval().data())>, - DataType>::value>> -bool operator>=(gsl::basic_string_span one, const T& other) GSL_NOEXCEPT -{ - return !(one < other); -} - -template < - typename CharT, std::ptrdiff_t Extent = gsl::dynamic_extent, typename T, - typename DataType = typename T::value_type, - typename = std::enable_if_t< - !gsl::details::is_span::value && !gsl::details::is_basic_string_span::value && - std::is_convertible::value && - std::is_same().size(), *std::declval().data())>, - DataType>::value>> -bool operator>=(const T& one, gsl::basic_string_span other) GSL_NOEXCEPT -{ - return !(one < other); -} -#endif -} // namespace gsl - -#undef GSL_NOEXCEPT - -#ifdef _MSC_VER -#pragma warning(pop) - -#if _MSC_VER < 1910 -#undef constexpr -#pragma pop_macro("constexpr") - -#endif // _MSC_VER < 1910 -#endif // _MSC_VER - -#endif // GSL_STRING_SPAN_H diff --git a/src/structures/lsmtree/segments/helpers.cpp b/src/structures/lsmtree/segments/helpers.cpp index 6f0a03c..28fa44f 100644 --- a/src/structures/lsmtree/segments/helpers.cpp +++ b/src/structures/lsmtree/segments/helpers.cpp @@ -6,7 +6,7 @@ #include #include "structures/lsmtree/segments/helpers.h" -#include "structures/lsmtree/segments/uuid.h" +#include "common/uuid.h" namespace structures::lsmtree::segments::helpers { diff --git a/src/structures/memtable/memtable.cpp b/src/structures/memtable/memtable.cpp index 2df5f7c..03c9cf9 100644 --- a/src/structures/memtable/memtable.cpp +++ b/src/structures/memtable/memtable.cpp @@ -1,6 +1,7 @@ #include "structures/memtable/memtable.h" #include +#include #include #include @@ -95,6 +96,7 @@ auto memtable_t::record_t::operator>(const memtable_t::record_t &record) const - auto memtable_t::record_t::operator==(const record_t &record) const -> bool { + spdlog::info("memtable::==: this.{} record.{}", m_key.m_key, record.m_key.m_key); return m_key == record.m_key; } diff --git a/src/structures/skiplist/CMakeLists.txt b/src/structures/skiplist/CMakeLists.txt index b6e3870..dfaa4ef 100644 --- a/src/structures/skiplist/CMakeLists.txt +++ b/src/structures/skiplist/CMakeLists.txt @@ -13,7 +13,7 @@ add_executable(SkipListTest skiplist_test.cpp) set_target_properties(SkipListTest PROPERTIES CXX_STANDARD 23) target_link_libraries( SkipListTest - Catch2::Catch2WithMain + gtest::gtest spdlog::spdlog fmt::fmt LSMTree @@ -22,6 +22,5 @@ target_link_libraries( Config ) -# Register Catch2 tests with CTest -include(Catch) -catch_discover_tests(SkipListTest) +# include(GoogleTest) +# gtest_discover_tests(SkipListTest) diff --git a/src/structures/skiplist/skiplist_test.cpp b/src/structures/skiplist/skiplist_test.cpp index b78d46f..a3b8b59 100644 --- a/src/structures/skiplist/skiplist_test.cpp +++ b/src/structures/skiplist/skiplist_test.cpp @@ -1,13 +1,8 @@ -// -// Created by nikon on 1/22/22. -// +#include #include -#include -#include "skiplist.h" - -#include +#include "structures/lsmtree/lsmtree.h" using namespace structures; @@ -22,7 +17,7 @@ struct test_record_t using test_skiplist_t = skiplist::skiplist_t>; -TEST_CASE("Emplace and Find", "[SkipList]") +TEST(SkipListTest, EmplaceAndFind) { test_skiplist_t sl; @@ -32,22 +27,22 @@ TEST_CASE("Emplace and Find", "[SkipList]") // Get what we put auto rec = sl.find("rec2"); - REQUIRE(rec.has_value()); - REQUIRE(rec->m_key == "rec2"); - REQUIRE(rec->m_value == "val2"); + EXPECT_TRUE(rec.has_value()); + EXPECT_EQ(rec->m_key, "rec2"); + EXPECT_EQ(rec->m_value, "val2"); // Return nullopt on non-existing keys rec = sl.find("nonexst"); - REQUIRE_FALSE(rec.has_value()); + EXPECT_FALSE(rec.has_value()); sl.emplace({"rec3", "val4"}); rec = sl.find("rec3"); - REQUIRE(rec.has_value()); - REQUIRE(rec->m_key == "rec3"); - REQUIRE(rec->m_value == "val4"); + EXPECT_TRUE(rec.has_value()); + EXPECT_EQ(rec->m_key, "rec3"); + EXPECT_EQ(rec->m_value, "val4"); } -TEST_CASE("Calculate size using iterators", "[SkipList]") +TEST(SkipListTest, CalcSizeWithIters) { test_skiplist_t sl; @@ -62,10 +57,10 @@ TEST_CASE("Calculate size using iterators", "[SkipList]") size++; } - REQUIRE(size == sl.size()); + EXPECT_EQ(size, sl.size()); } -TEST_CASE("Check records pointed by iterators", "[SkipList]") +TEST(SkipListTest, IterValid) { test_skiplist_t sl; @@ -74,34 +69,33 @@ TEST_CASE("Check records pointed by iterators", "[SkipList]") sl.emplace({"rec2", "val2"}); auto begin{sl.begin()}; - REQUIRE(begin->m_key == "rec1"); + EXPECT_EQ(begin->m_key, "rec1"); + begin++; - REQUIRE(begin->m_key == "rec2"); - // begin++; - // REQUIRE(begin->m_key == "rec3"); - // begin++; - // REQUIRE(begin == sl.end()); + EXPECT_EQ(begin->m_key, "rec2"); } -TEST_CASE("std::find_if", "[SkipList]") +TEST(SkipListTest, StdFindIf) { - test_skiplist_t sl; + test_skiplist_t skiplist; - sl.emplace({"rec1", "val1"}); - sl.emplace({"rec3", "val3"}); - sl.emplace({"rec2", "val2"}); + skiplist.emplace({.m_key = "rec1", .m_value = "val1"}); + skiplist.emplace({.m_key = "rec3", .m_value = "val3"}); + skiplist.emplace({.m_key = "rec2", .m_value = "val2"}); - auto it = std::find_if(std::begin(sl), - std::end(sl), - [](const test_record_t &record) { return record.m_key == "rec1"; }); - STATIC_CHECK(std::is_same_v); - REQUIRE(it != sl.end()); - REQUIRE(it->m_key == "rec1"); - REQUIRE(it->m_value == "val1"); - - it = std::find_if(std::begin(sl), - std::end(sl), - [](const test_record_t &record) { return record.m_key == "rec4"; }); - STATIC_CHECK(std::is_same_v); - REQUIRE(it == sl.end()); + { + auto iter = std::ranges::find_if( + skiplist, [](const test_record_t &record) -> bool { return record.m_key == "rec1"; } + ); + EXPECT_NE(iter, skiplist.end()); + EXPECT_EQ(iter->m_key, "rec1"); + EXPECT_EQ(iter->m_value, "val1"); + } + + { + auto it = std::ranges::find_if( + skiplist, [](const test_record_t &record) -> bool { return record.m_key == "rec4"; } + ); + EXPECT_EQ(it, skiplist.end()); + } } diff --git a/src/wal/wal_test.cpp b/src/wal/wal_test.cpp index b7b5c1f..39f6930 100644 --- a/src/wal/wal_test.cpp +++ b/src/wal/wal_test.cpp @@ -63,7 +63,7 @@ class WALTest : public testing::TestWithParam std::filesystem::remove(GetTemporaryFilepath()); } - static auto GetWAL() -> std::optional> + static auto GetWAL() { return wal::wal_builder_t{} .set_file_path(GetTemporaryFilepath()) @@ -84,13 +84,12 @@ std::vector WALTest::Recs = { TEST_P(WALTest, SuccessfullyCreatesEmptyWAL) { auto wal = WALTest::GetWAL(); - EXPECT_TRUE(wal.has_value()); - EXPECT_EQ(wal->size(), 0); - EXPECT_TRUE(wal->empty()); + EXPECT_EQ(wal.value()->size(), 0); + EXPECT_TRUE(wal.value()->empty()); - auto res{wal->read(0)}; + auto res{wal.value()->read(0)}; EXPECT_FALSE(res.has_value()); } @@ -99,12 +98,12 @@ TEST_P(WALTest, AppendAndCheckSize) auto wal = WALTest::GetWAL(); EXPECT_TRUE(wal.has_value()); - EXPECT_TRUE(wal->add(Recs[0])); + EXPECT_TRUE(wal.value()->add(Recs[0])); - EXPECT_EQ(wal->size(), 1); - EXPECT_FALSE(wal->empty()); + EXPECT_EQ(wal.value()->size(), 1); + EXPECT_FALSE(wal.value()->empty()); - auto res{wal->read(0)}; + auto res{wal.value()->read(0)}; EXPECT_TRUE(res.has_value()); EXPECT_EQ(res->key, Recs[0].key); EXPECT_EQ(res->value, Recs[0].value); @@ -120,16 +119,16 @@ TEST_P(WALTest, AppendSameEntryAndCheckSize) for (const auto &rec : recs) { - EXPECT_TRUE(wal->add(rec)); + EXPECT_TRUE(wal.value()->add(rec)); } - const auto walSize = wal->size(); + const auto walSize = wal.value()->size(); EXPECT_EQ(walSize, recs.size()); - EXPECT_FALSE(wal->empty()); + EXPECT_FALSE(wal.value()->empty()); for (const auto &[idx, rec] : std::views::enumerate(recs)) { - auto res{wal->read(idx)}; + auto res{wal.value()->read(idx)}; EXPECT_TRUE(res.has_value()); EXPECT_EQ(res->key, rec.key); EXPECT_EQ(res->value, rec.value); @@ -146,14 +145,14 @@ TEST_P(WALTest, AppendAndRestAndCheckSize) for (const auto &rec : recs) { - EXPECT_TRUE(wal->add(rec)); + EXPECT_TRUE(wal.value()->add(rec)); } - EXPECT_EQ(wal->size(), recs.size()); - EXPECT_FALSE(wal->empty()); - EXPECT_TRUE(wal->reset()); - EXPECT_EQ(wal->size(), 0); - EXPECT_TRUE(wal->empty()); + EXPECT_EQ(wal.value()->size(), recs.size()); + EXPECT_FALSE(wal.value()->empty()); + EXPECT_TRUE(wal.value()->reset()); + EXPECT_EQ(wal.value()->size(), 0); + EXPECT_TRUE(wal.value()->empty()); } TEST_P(WALTest, AppendAndResetLastNAndCheckSize) @@ -167,23 +166,23 @@ TEST_P(WALTest, AppendAndResetLastNAndCheckSize) for (const auto &rec : recs) { - EXPECT_TRUE(wal->add(rec)); + EXPECT_TRUE(wal.value()->add(rec)); } - const auto walSizeBeforeResetLastN = wal->size(); + const auto walSizeBeforeResetLastN = wal.value()->size(); EXPECT_EQ(walSizeBeforeResetLastN, recs.size()); - EXPECT_FALSE(wal->empty()); + EXPECT_FALSE(wal.value()->empty()); - EXPECT_TRUE(wal->reset_last_n(lastN)); + EXPECT_TRUE(wal.value()->reset_last_n(lastN)); - const auto walSizeAfteresetLastN = wal->size(); + const auto walSizeAfteresetLastN = wal.value()->size(); EXPECT_EQ(walSizeAfteresetLastN, walSizeBeforeResetLastN - lastN); - EXPECT_FALSE(wal->empty()); + EXPECT_FALSE(wal.value()->empty()); for (const auto &[idx, rec] : recs | std::views::enumerate | std::views::take(recs.size() - lastN)) { - auto res{wal->read(idx)}; + auto res{wal.value()->read(idx)}; EXPECT_TRUE(res.has_value()); EXPECT_EQ(res->key, rec.key); EXPECT_EQ(res->value, rec.value); @@ -194,9 +193,9 @@ TEST_P(WALTest, ResetLastNOnEmptyWAL) { auto wal = WALTest::GetWAL(); - EXPECT_FALSE(wal->reset_last_n(std::numeric_limits::min())); - EXPECT_FALSE(wal->reset_last_n(std::numeric_limits::max() / 2)); - EXPECT_FALSE(wal->reset_last_n(std::numeric_limits::max())); + EXPECT_FALSE(wal.value()->reset_last_n(std::numeric_limits::min())); + EXPECT_FALSE(wal.value()->reset_last_n(std::numeric_limits::max() / 2)); + EXPECT_FALSE(wal.value()->reset_last_n(std::numeric_limits::max())); } TEST_P(WALTest, ResetAndAdd) @@ -209,18 +208,18 @@ TEST_P(WALTest, ResetAndAdd) for (const auto &rec : recs) { - EXPECT_TRUE(wal->add(rec)); + EXPECT_TRUE(wal.value()->add(rec)); } - EXPECT_EQ(wal->size(), recs.size()); + EXPECT_EQ(wal.value()->size(), recs.size()); - EXPECT_TRUE(wal->reset()); - EXPECT_EQ(wal->size(), 0); + EXPECT_TRUE(wal.value()->reset()); + EXPECT_EQ(wal.value()->size(), 0); for (const auto &rec : recs) { - EXPECT_TRUE(wal->add(rec)); + EXPECT_TRUE(wal.value()->add(rec)); } - EXPECT_EQ(wal->size(), recs.size()); + EXPECT_EQ(wal.value()->size(), recs.size()); } TEST_P(WALTest, ResetLasnNAndAdd) @@ -234,26 +233,26 @@ TEST_P(WALTest, ResetLasnNAndAdd) for (const auto &rec : recs) { - EXPECT_TRUE(wal->add(rec)); + EXPECT_TRUE(wal.value()->add(rec)); } - const auto walSizeBeforeResetLastN = wal->size(); + const auto walSizeBeforeResetLastN = wal.value()->size(); EXPECT_EQ(walSizeBeforeResetLastN, recs.size()); - EXPECT_FALSE(wal->empty()); + EXPECT_FALSE(wal.value()->empty()); - EXPECT_TRUE(wal->reset_last_n(lastN)); + EXPECT_TRUE(wal.value()->reset_last_n(lastN)); - const auto walSizeAfteresetLastN = wal->size(); + const auto walSizeAfteresetLastN = wal.value()->size(); EXPECT_EQ(walSizeAfteresetLastN, walSizeBeforeResetLastN - lastN); - EXPECT_FALSE(wal->empty()); + EXPECT_FALSE(wal.value()->empty()); - EXPECT_TRUE(wal->add(recs[4])); - EXPECT_EQ(wal->size(), walSizeBeforeResetLastN - lastN + 1); + EXPECT_TRUE(wal.value()->add(recs[4])); + EXPECT_EQ(wal.value()->size(), walSizeBeforeResetLastN - lastN + 1); for (const auto &[idx, rec] : recs | std::views::enumerate | std::views::take(recs.size() - lastN + 1)) { - auto res{wal->read(idx)}; + auto res{wal.value()->read(idx)}; EXPECT_TRUE(res.has_value()); EXPECT_EQ(res->key, rec.key); EXPECT_EQ(res->value, rec.value); @@ -265,7 +264,7 @@ TEST_P(WALTest, GetRecordsOnEmptyWAL) auto wal = WALTest::GetWAL(); EXPECT_TRUE(wal.has_value()); - EXPECT_EQ(wal->records().size(), 0); + EXPECT_EQ(wal.value()->records().size(), 0); } TEST_P(WALTest, AppendAndGetRecords) @@ -278,13 +277,13 @@ TEST_P(WALTest, AppendAndGetRecords) for (const auto &rec : recs) { - EXPECT_TRUE(wal->add(rec)); + EXPECT_TRUE(wal.value()->add(rec)); } - EXPECT_EQ(wal->size(), recs.size()); - EXPECT_FALSE(wal->empty()); + EXPECT_EQ(wal.value()->size(), recs.size()); + EXPECT_FALSE(wal.value()->empty()); - std::vector walRecs = wal->records(); + std::vector walRecs = wal.value()->records(); EXPECT_EQ(recs.size(), walRecs.size()); for (const auto &[idx, rec] : std::views::enumerate(recs)) @@ -299,8 +298,8 @@ TEST_P(WALTest, ReadOutOfBounds) auto wal = WALTest::GetWAL(); ASSERT_TRUE(wal.has_value()); - EXPECT_FALSE(wal->read(-1).has_value()); - EXPECT_FALSE(wal->read(1000).has_value()); + EXPECT_FALSE(wal.value()->read(-1).has_value()); + EXPECT_FALSE(wal.value()->read(1000).has_value()); } // ---- @@ -322,9 +321,9 @@ TEST_P(WALTest, CreateAndRecoverInMemoryWAL) // Fill it will mock data for (const auto &rec : WALTest::Recs) { - EXPECT_TRUE(initialWal->add(rec)); + EXPECT_TRUE(initialWal.value()->add(rec)); } - EXPECT_EQ(initialWal->size(), WALTest::Recs.size()); + EXPECT_EQ(initialWal.value()->size(), WALTest::Recs.size()); // Recover the WAL auto recoveredWal = wal::wal_builder_t{} @@ -332,9 +331,9 @@ TEST_P(WALTest, CreateAndRecoverInMemoryWAL) .build(wal::log_storage_type_k::in_memory_k); // Main assertions - EXPECT_NE(initialWal->size(), recoveredWal->size()); - EXPECT_EQ(recoveredWal->size(), 0); - EXPECT_TRUE(recoveredWal->empty()); + EXPECT_NE(initialWal.value()->size(), recoveredWal.value()->size()); + EXPECT_EQ(recoveredWal.value()->size(), 0); + EXPECT_TRUE(recoveredWal.value()->empty()); } TEST_P(WALTest, CreateAndRecoverFileBasedWAL) @@ -352,24 +351,24 @@ TEST_P(WALTest, CreateAndRecoverFileBasedWAL) // Fill it will mock data for (const auto &rec : WALTest::Recs) { - EXPECT_TRUE(initialWal->add(rec)); + EXPECT_TRUE(initialWal.value()->add(rec)); } - EXPECT_EQ(initialWal->size(), WALTest::Recs.size()); - auto initialRecords = initialWal->records(); + EXPECT_EQ(initialWal.value()->size(), WALTest::Recs.size()); + auto initialRecords = initialWal.value()->records(); // Recover the WAL auto recoveredWal = wal::wal_builder_t{} .set_file_path(GetTemporaryFilepath()) .build(wal::log_storage_type_k::file_based_persistent_k); - auto recoveredRecords = initialWal->records(); + auto recoveredRecords = initialWal.value()->records(); // Main assertions - EXPECT_EQ(initialWal->size(), recoveredWal->size()); + EXPECT_EQ(initialWal.value()->size(), recoveredWal.value()->size()); - for (std::size_t idx{0}; idx < initialWal->size(); idx++) + for (std::size_t idx{0}; idx < initialWal.value()->size(); idx++) { - EXPECT_EQ(initialWal->read(idx)->key, recoveredWal->read(idx)->key); - EXPECT_EQ(initialWal->read(idx)->value, recoveredWal->read(idx)->value); + EXPECT_EQ(initialWal.value()->read(idx)->key, recoveredWal.value()->read(idx)->key); + EXPECT_EQ(initialWal.value()->read(idx)->value, recoveredWal.value()->read(idx)->value); EXPECT_EQ(initialRecords[idx].key, recoveredRecords[idx].key); EXPECT_EQ(initialRecords[idx].value, recoveredRecords[idx].value); From b31c4247fc8f1e453b41dda6ccfc44bb3c4e1729 Mon Sep 17 00:00:00 2001 From: lnikon Date: Fri, 21 Nov 2025 12:28:40 +0400 Subject: [PATCH 16/17] Testing: - Implement binary buffer reader/writer - Implement tests for serialization layer --- .vscode/launch.json | 51 ++- src/CMakeLists.txt | 2 + src/common/CMakeLists.txt | 21 ++ src/common/helpers.cpp | 69 ++++ src/include/common/helpers.cpp | 28 -- src/include/common/helpers.h | 35 ++ src/include/serialization/buffer_reader.h | 151 ++++++++ src/include/serialization/buffer_writer.h | 123 +++++++ src/include/serialization/common.h | 44 +++ src/include/serialization/concepts.h | 25 ++ src/include/serialization/crc32.h | 64 ++++ src/include/serialization/endian_integer.h | 71 ++++ src/serialization/CMakeLists.txt | 71 ++++ src/serialization/buffer_reader.cpp | 1 + .../buffer_reader_writer_test.cpp | 205 +++++++++++ src/serialization/buffer_writer.cpp | 1 + src/serialization/common.cpp | 21 ++ src/serialization/crc32.cpp | 31 ++ src/serialization/crc32_test.cpp | 61 +++ src/serialization/endian_integer_test.cpp | 57 +++ src/serialization/varint_test.cpp | 124 +++++++ src/structures/lsmtree/CMakeLists.txt | 3 +- .../compaction/level_zero_compactation.cpp | 0 src/structures/lsmtree/levels/level.cpp | 7 +- src/structures/lsmtree/levels/levels.cpp | 2 +- src/structures/lsmtree/lsmtree.cpp | 52 ++- src/structures/lsmtree/lsmtree_test.cpp | 143 -------- src/structures/lsmtree/lsmtree_test_basic.cpp | 346 ++++++++++++++++++ .../segments/lsmtree_regular_segment.cpp | 12 +- 29 files changed, 1607 insertions(+), 214 deletions(-) create mode 100644 src/common/CMakeLists.txt create mode 100644 src/common/helpers.cpp delete mode 100644 src/include/common/helpers.cpp create mode 100644 src/include/serialization/buffer_reader.h create mode 100644 src/include/serialization/buffer_writer.h create mode 100644 src/include/serialization/common.h create mode 100644 src/include/serialization/concepts.h create mode 100644 src/include/serialization/crc32.h create mode 100644 src/include/serialization/endian_integer.h create mode 100644 src/serialization/CMakeLists.txt create mode 100644 src/serialization/buffer_reader.cpp create mode 100644 src/serialization/buffer_reader_writer_test.cpp create mode 100644 src/serialization/buffer_writer.cpp create mode 100644 src/serialization/common.cpp create mode 100644 src/serialization/crc32.cpp create mode 100644 src/serialization/crc32_test.cpp create mode 100644 src/serialization/endian_integer_test.cpp create mode 100644 src/serialization/varint_test.cpp delete mode 100644 src/structures/lsmtree/compaction/level_zero_compactation.cpp delete mode 100644 src/structures/lsmtree/lsmtree_test.cpp create mode 100644 src/structures/lsmtree/lsmtree_test_basic.cpp diff --git a/.vscode/launch.json b/.vscode/launch.json index e95e180..660a478 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -162,13 +162,61 @@ "debuggerPath": "/usr/bin/gdb" }, }, + { + "name": "Debug - BufferReaderWriter", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/build/BufferReaderWriterTest", + // Path to the compiled executable + "args": [ + "--gtest_filter=BufferReaderWriterTest.ReadYourWrites" + ], + // Arguments to pass to the program + "stopAtEntry": false, + // Set to true to stop at the program's entry point + "cwd": "${workspaceFolder}", + // Current working directory + "environment": [], + "externalConsole": false, + // Set to true if you want to use an external terminal + "MIMode": "gdb", + // Use "lldb" if you're using clang on macOS + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ], + // Ensure your program is built before launching + "miDebuggerPath": "/usr/bin/gdb", + // Path to the gdb or lldb debugger + "logging": { + "trace": true, + // Enable trace for debugging the launch.json config + "traceResponse": true, + "engineLogging": false + }, + "launchCompleteCommand": "exec-run", + "targetArchitecture": "x86_64", + "pipeTransport": { + "pipeCwd": "", + "pipeProgram": "/bin/bash", + "pipeArgs": [ + "-c" + ], + "debuggerPath": "/usr/bin/gdb" + }, + }, { "name": "Debug - LSMTreeTest", "type": "cppdbg", "request": "launch", "program": "${workspaceFolder}/build/LSMTreeTest", // Path to the compiled executable - "args": [], + "args": [ + "--gtest_filter=LSMTreeTestBasicCRUD.MultipleLargeKeyValuePairsFlush" + ], // Arguments to pass to the program "stopAtEntry": false, // Set to true to stop at the program's entry point @@ -186,7 +234,6 @@ "ignoreFailures": true } ], - "preLaunchTask": "build", // Ensure your program is built before launching "miDebuggerPath": "/usr/bin/gdb", // Path to the gdb or lldb debugger diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fb20add..b0adba5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,6 +10,8 @@ add_subdirectory(server) add_subdirectory(raft) add_subdirectory(wal) add_subdirectory(posix_wrapper) +add_subdirectory(common) +add_subdirectory(serialization) add_executable(Main main.cpp) set_target_properties(Main PROPERTIES CXX_STANDARD 23) diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt new file mode 100644 index 0000000..7bce91c --- /dev/null +++ b/src/common/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.25) +project(frankie) + +add_library(Common helpers.cpp) + +set_target_properties(Common PROPERTIES CXX_STANDARD 23) +target_include_directories(Common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include) +target_link_libraries( + Common + PUBLIC + libassert::assert + fmt::fmt + nlohmann_json::nlohmann_json + nlohmann_json_schema_validator + cxxopts::cxxopts + magic_enum::magic_enum +) + +# add_executable(ConfigTest "db_test.cpp") set_target_properties(ConfigTest +# PROPERTIES CXX_STANDARD 23) target_link_libraries( ConfigTest boost::boost +# Catch2::Catch2 spdlog::spdlog fmt::fmt LSMTree MemTable Config) diff --git a/src/common/helpers.cpp b/src/common/helpers.cpp new file mode 100644 index 0000000..4e2136e --- /dev/null +++ b/src/common/helpers.cpp @@ -0,0 +1,69 @@ +#include "common/helpers.h" +#include + +namespace common +{ + +auto uuid() -> std::string +{ + std::random_device rnd; + auto seed_data = std::array{}; + std::ranges::generate(seed_data, std::ref(rnd)); + + std::seed_seq seq(std::begin(seed_data), std::end(seed_data)); + std::mt19937 generator(seq); + uuids::uuid_random_generator gen{generator}; + + return to_string(uuids::basic_uuid_random_generator{gen}()); +} + +auto segment_name() -> std::string +{ + return fmt::format("segment_{}", uuid()); +} + +auto segment_path(const std::filesystem::path &datadir, const std::string &name) + -> std::filesystem::path +{ + return datadir / name; +} + +auto generateRandomString(std::size_t length) noexcept -> std::string +{ + std::mt19937_64 gen(std::random_device{}()); + std::uniform_int_distribution dis(1, 255); + + std::string result; + result.reserve(length); + + size_t remaining = length; + while (remaining >= sizeof(uint64_t)) + { + uint64_t chunk = dis(gen); + result.append(reinterpret_cast(&chunk), sizeof(uint64_t)); + remaining -= sizeof(uint64_t); + } + + std::uniform_int_distribution dis8; + while ((remaining--) != 0U) + { + result += static_cast(dis8(gen)); + } + return result; +} + +auto generateRandomStringPairVector(const std::size_t length) noexcept + -> std::vector> +{ + std::vector> result; + result.reserve(length); + for (std::string::size_type size = 0; size < length; size++) + { + result.emplace_back( + generateRandomString(generateRandomNumber(64, 64)), + generateRandomString(generateRandomNumber(64, 64)) + ); + } + return result; +} +} // namespace common diff --git a/src/include/common/helpers.cpp b/src/include/common/helpers.cpp deleted file mode 100644 index 2db2db3..0000000 --- a/src/include/common/helpers.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "common/helpers.h" - -namespace common -{ - -auto uuid() -> std::string -{ - std::random_device rnd; - auto seed_data = std::array{}; - std::ranges::generate(seed_data, std::ref(rnd)); - - std::seed_seq seq(std::begin(seed_data), std::end(seed_data)); - std::mt19937 generator(seq); - uuids::uuid_random_generator gen{generator}; - - return to_string(uuids::basic_uuid_random_generator{gen}()); -} - -auto segment_name() -> std::string -{ - return fmt::format("segment_{}", uuid()); -} - -auto segment_path(const std::filesystem::path &datadir, const std::string &name) - -> std::filesystem::path -{ - return datadir / name; -} diff --git a/src/include/common/helpers.h b/src/include/common/helpers.h index 557ce74..38bcff4 100644 --- a/src/include/common/helpers.h +++ b/src/include/common/helpers.h @@ -11,6 +11,36 @@ namespace common { +template +auto generateRandomNumber( + const TNumber min = std::numeric_limits::min(), + const TNumber max = std::numeric_limits::max() +) noexcept -> TNumber +{ + std::mt19937 rng{std::random_device{}()}; + if constexpr (std::is_same_v) + { + return std::uniform_int_distribution(min, max)(rng); + } + else if (std::is_same_v) + { + return std::uniform_int_distribution(min, max)(rng); + } + else if (std::is_same_v) + { + return std::uniform_real_distribution(min, max)(rng); + } + else if (std::is_same_v) + { + return std::uniform_real_distribution(min, max)(rng); + } + else + { + // TODO(vahag): better handle this case + return 0; + } +} + auto uuid() -> std::string; auto segment_name() -> std::string; @@ -18,4 +48,9 @@ auto segment_name() -> std::string; auto segment_path(const std::filesystem::path &datadir, const std::string &name) -> std::filesystem::path; +auto generateRandomString(std::size_t length) noexcept -> std::string; + +auto generateRandomStringPairVector(const std::size_t length) noexcept + -> std::vector>; + } // namespace common diff --git a/src/include/serialization/buffer_reader.h b/src/include/serialization/buffer_reader.h new file mode 100644 index 0000000..4809e35 --- /dev/null +++ b/src/include/serialization/buffer_reader.h @@ -0,0 +1,151 @@ +#pragma once + +#include +#include +#include +#include + +#include "serialization/common.h" +#include "serialization/concepts.h" + +namespace serialization +{ + +class buffer_reader_t +{ + public: + buffer_reader_t(std::span buffer) + : m_buffer{buffer} + { + } + + template + [[nodiscard]] auto read_endian_integer(T &out) noexcept -> buffer_reader_t & + { + if (m_error) + { + return *this; + } + + auto bytes{read_raw_bytes(T::SIZE)}; + if (m_error) + { + return *this; + } + out = T::from_bytes(bytes); + + return *this; + } + + [[nodiscard]] auto read_varint(std::uint64_t &out) noexcept -> buffer_reader_t & + { + if (m_error) + { + return *this; + } + + std::uint64_t result{0}; + std::size_t shift{0}; + std::size_t byteCount{0}; + + while (byteCount < MAX_VARINT_BYTES) + { + const auto bytes{read_raw_bytes(1)}; + if (has_error()) + { + return *this; + } + + result |= (std::to_integer((bytes[0] & std::byte{0x7F})) << shift); + + if ((bytes[0] & std::byte{0x80}) == std::byte{0}) + { + out = result; + return *this; + } + + shift += 7; + byteCount++; + } + + m_error = serialization_error_k::invalid_variant_k; + return *this; + } + + [[nodiscard]] auto read_string(std::string_view &out) noexcept -> buffer_reader_t & + { + if (m_error) + { + return *this; + } + + std::uint64_t count{0}; + (void)read_varint(count); + if (m_error) + { + return *this; + } + + auto span{read_raw_bytes(count)}; + if (span.empty() || has_error()) + { + return *this; + } + out = to_string_view(span); + + return *this; + } + + [[nodiscard]] auto read_bytes(const std::size_t count, std::span &out) noexcept + -> buffer_reader_t & + { + if (has_error()) + { + return *this; + } + + out = read_raw_bytes(count); + + return *this; + } + + [[nodiscard]] auto bytes_read() const noexcept -> std::size_t + { + return m_position; + } + + [[nodiscard]] auto remaining() const noexcept -> std::size_t + { + return m_buffer.size() - m_position; + } + + [[nodiscard]] auto error() const noexcept -> std::optional + { + return m_error; + } + + [[nodiscard]] auto has_error() const noexcept -> bool + { + return m_error.has_value(); + } + + private: + auto read_raw_bytes(std::size_t count) noexcept -> std::span + { + if (count + m_position > m_buffer.size()) + { + m_error = serialization_error_k::truncated_file_k; + return {}; + } + + auto result{m_buffer.subspan(m_position, count)}; + m_position += count; + return result; + } + + std::span m_buffer; + std::size_t m_position{0}; + std::optional m_error{std::nullopt}; +}; + +} // namespace serialization diff --git a/src/include/serialization/buffer_writer.h b/src/include/serialization/buffer_writer.h new file mode 100644 index 0000000..95e76a7 --- /dev/null +++ b/src/include/serialization/buffer_writer.h @@ -0,0 +1,123 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "serialization/common.h" +#include "serialization/concepts.h" + +namespace serialization +{ + +class buffer_writer_t final +{ + public: + buffer_writer_t(std::span buffer) + : m_buffer{buffer} + { + } + + template + [[nodiscard]] auto write_endian_integer(const T value) noexcept -> buffer_writer_t & + { + if (m_error) + { + return *this; + } + + write_raw_bytes(value.bytes()); + + return *this; + } + + [[nodiscard]] auto write_varint(std::uint64_t value) noexcept -> buffer_writer_t & + { + if (m_error) + { + return *this; + } + + std::array scratch{}; + std::size_t count{0}; + + while (value > 127) + { + scratch[count++] = + std::byte{static_cast((value & 0x7F))} | std::byte{0x80}; + value >>= 7; + } + scratch[count++] = std::byte{static_cast((value & 0x7F))}; + + write_raw_bytes(std::span(scratch.data(), count)); + + return *this; + } + + [[nodiscard]] auto write_bytes(std::span buffer) noexcept -> buffer_writer_t & + { + if (m_error) + { + return *this; + } + + write_raw_bytes(buffer); + + return *this; + } + + [[nodiscard]] auto write_string(const std::string_view str) noexcept -> buffer_writer_t & + { + if (m_error) + { + return *this; + } + + (void)write_varint(str.size()); + if (m_error) + { + return *this; + } + + write_raw_bytes(to_span(str)); + + return *this; + } + + [[nodiscard]] auto bytes_written() const noexcept -> std::size_t + { + return m_position; + } + + [[nodiscard]] auto error() const noexcept -> std::optional + { + return m_error; + } + + [[nodiscard]] auto has_error() const noexcept -> bool + { + return m_error.has_value(); + } + + private: + void write_raw_bytes(std::span bytes) noexcept + { + if (m_position + bytes.size() > m_buffer.size()) + { + m_error = serialization_error_k::buffer_overflow_k; + return; + } + + std::memcpy(&m_buffer[m_position], bytes.data(), bytes.size()); + + m_position += bytes.size(); + } + + std::span m_buffer; + std::size_t m_position{0}; + std::optional m_error{std::nullopt}; +}; + +} // namespace serialization diff --git a/src/include/serialization/common.h b/src/include/serialization/common.h new file mode 100644 index 0000000..c300f0d --- /dev/null +++ b/src/include/serialization/common.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +namespace serialization +{ + +// serialization_error_k represents common serialization errors +enum class serialization_error_k : std::int8_t +{ + undefined_k = -1, + + // Write errors + buffer_overflow_k, + + // Read errors + truncated_file_k, + + // Format errors + invalid_magic_k, + version_mismatch_k, + checksum_mismatch_k, + corrupted_data_k, + + // Encoding errors + invalid_variant_k, + invalid_offset_k +}; + +// MAX_VARINT_BYTES represents maximum number of bytes required to encode 64 bit integer with varint +constexpr const std::size_t MAX_VARINT_BYTES{10}; + +// byte_t represents a single byte +using byte_t = std::byte; + +// to_span converts std::string_view into std::byte array +[[nodiscard]] auto to_span(std::string_view view) noexcept -> std::span; + +// to_string_view converts std::byte span into a std::string_view +[[nodiscard]] auto to_string_view(std::span span) noexcept -> std::string_view; + +} // namespace serialization diff --git a/src/include/serialization/concepts.h b/src/include/serialization/concepts.h new file mode 100644 index 0000000..859161e --- /dev/null +++ b/src/include/serialization/concepts.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +namespace serialization +{ + +// Forward declaration +template class endian_integer; + +// Nothing is an endian_integer +template struct is_endian_integer : std::false_type +{ +}; + +// Except the endian_integer itself +template struct is_endian_integer> : std::true_type +{ +}; + +template +concept EndianInteger = is_endian_integer::value; + +} // namespace serialization diff --git a/src/include/serialization/crc32.h b/src/include/serialization/crc32.h new file mode 100644 index 0000000..faf4bd6 --- /dev/null +++ b/src/include/serialization/crc32.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include + +namespace serialization +{ + +namespace detail +{ + +static constexpr const std::uint32_t CRC32_DEFAULT_VALUE{0xFFFFFFFF}; +static constexpr const std::int32_t CRC32_BITS{8}; +static constexpr const std::size_t CRC32_TABLE_SIZE{256}; +static constexpr const std::uint32_t CRC32_POLYNOMIAL = 0xEDB88320; + +using crc32_table_t = std::array; + +constexpr auto generate_crc32_table() noexcept -> crc32_table_t +{ + crc32_table_t table{}; + for (std::uint32_t i{0}; i < table.size(); i++) + { + std::uint32_t crc = i; + for (std::uint8_t bit{0}; bit < CRC32_BITS; bit++) + { + if ((crc & 1) != 0) + { + crc = (crc >> 1) ^ CRC32_POLYNOMIAL; + } + else + { + crc = crc >> 1; + } + } + table[i] = crc; + } + return table; +} + +} // namespace detail + +class crc32_t final +{ + public: + constexpr crc32_t() = default; + + // update updates the stored crc with new data, without XORing with 0xFFFFFFFF + void update(std::span data) noexcept; + + // finalize returns the stored crc XORed with 0xFFFFFFFF + [[nodiscard]] auto finalize() const noexcept -> std::uint32_t; + + // reset sets crc to 0xFFFFFFFF + void reset() noexcept; + + private: + static constexpr const auto TABLE{detail::generate_crc32_table()}; + + std::uint32_t m_crc{detail::CRC32_DEFAULT_VALUE}; +}; + +} // namespace serialization diff --git a/src/include/serialization/endian_integer.h b/src/include/serialization/endian_integer.h new file mode 100644 index 0000000..58c728f --- /dev/null +++ b/src/include/serialization/endian_integer.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "serialization/concepts.h" + +namespace serialization +{ + +template class endian_integer +{ + public: + static constexpr size_t SIZE = sizeof(T); + + endian_integer(T value) + { + T converted = value; + + if constexpr (std::endian::native != Target) + { + std::cout << "converted" << '\n'; + converted = std::byteswap(value); + } + + std::memcpy(m_bytes.data(), &converted, sizeof(T)); + } + + [[nodiscard]] auto bytes() const noexcept -> std::span + { + return m_bytes; + } + + [[nodiscard]] auto get() const noexcept -> T + { + T value; + std::memcpy(&value, m_bytes.data(), sizeof(T)); + + if constexpr (std::endian::native != Target) + { + return std::byteswap(value); + } + + return value; + } + + static auto from_bytes(std::span bytes) -> endian_integer + { + endian_integer result{0}; + std::memcpy(result.m_bytes.data(), bytes.data(), bytes.size()); + return result; + } + + private: + std::array m_bytes; +}; + +using le_int8_t = endian_integer; +using le_int16_t = endian_integer; +using le_int32_t = endian_integer; +using le_int64_t = endian_integer; +using le_uint8_t = endian_integer; +using le_uint16_t = endian_integer; +using le_uint32_t = endian_integer; +using le_uint64_t = endian_integer; + +} // namespace serialization diff --git a/src/serialization/CMakeLists.txt b/src/serialization/CMakeLists.txt new file mode 100644 index 0000000..d6a8926 --- /dev/null +++ b/src/serialization/CMakeLists.txt @@ -0,0 +1,71 @@ +cmake_minimum_required(VERSION 3.25) +project(frankie) + +add_library( + Serialization + buffer_writer.cpp + buffer_reader.cpp + crc32.cpp + common.cpp +) +set_target_properties(Serialization PROPERTIES CXX_STANDARD 23) +target_include_directories( + Serialization + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include +) + +# BufferReaderWriterTest +add_executable(BufferReaderWriterTest buffer_reader_writer_test.cpp) +set_target_properties(BufferReaderWriterTest PROPERTIES CXX_STANDARD 23) +target_link_libraries( + BufferReaderWriterTest + gtest::gtest + spdlog::spdlog + fmt::fmt + Serialization +) + +include(GoogleTest) +gtest_discover_tests(BufferReaderWriterTest) + +# EndianIntegerTest +add_executable(EndianIntegerTest endian_integer_test.cpp) +set_target_properties(EndianIntegerTest PROPERTIES CXX_STANDARD 23) +target_link_libraries( + EndianIntegerTest + gtest::gtest + spdlog::spdlog + fmt::fmt + Serialization +) + +include(GoogleTest) +gtest_discover_tests(EndianIntegerTest) + +# VarintTest +add_executable(VarintTest varint_test.cpp) +set_target_properties(VarintTest PROPERTIES CXX_STANDARD 23) +target_link_libraries( + VarintTest + gtest::gtest + spdlog::spdlog + fmt::fmt + Serialization +) + +include(GoogleTest) +gtest_discover_tests(VarintTest) + +# CRC32 +add_executable(CRC32Test crc32_test.cpp) +set_target_properties(VarintTest PROPERTIES CXX_STANDARD 23) +target_link_libraries( + CRC32Test + gtest::gtest + spdlog::spdlog + fmt::fmt + Serialization +) + +include(GoogleTest) +gtest_discover_tests(CRC32Test) diff --git a/src/serialization/buffer_reader.cpp b/src/serialization/buffer_reader.cpp new file mode 100644 index 0000000..32fd9f3 --- /dev/null +++ b/src/serialization/buffer_reader.cpp @@ -0,0 +1 @@ +#include "serialization/buffer_reader.h" diff --git a/src/serialization/buffer_reader_writer_test.cpp b/src/serialization/buffer_reader_writer_test.cpp new file mode 100644 index 0000000..a9e2215 --- /dev/null +++ b/src/serialization/buffer_reader_writer_test.cpp @@ -0,0 +1,205 @@ +#include + +#include +#include +#include + +#include "serialization/buffer_reader.h" +#include "serialization/buffer_writer.h" +#include "serialization/endian_integer.h" + +using namespace serialization; + +class MyFixture : public testing::Test +{ + protected: + void SetUp() override + { + } + + void TearDown() override + { + } +}; + +TEST(EndianInteger, Get) +{ + const std::int8_t podInt8{12}; + const le_int8_t expectedInt8{podInt8}; + + EXPECT_EQ(expectedInt8.get(), podInt8); +} + +TEST(EndianInteger, FromBytes) +{ + const std::int8_t podInt8{12}; + const le_int8_t expectedInt8{podInt8}; + const std::span bytes{}; + + EXPECT_EQ(expectedInt8.get(), podInt8); +} + +TEST(BufferReaderWriterTest, ReadYourWrites) +{ + // Raw data + static constexpr const std::uint64_t bytesCount{4}; + static constexpr const std::array rawBytes{ + std::byte{0x40}, std::byte{0xFC}, std::byte{0x04}, std::byte{0xAF} + }; + + // Expected values + const le_int8_t expectedInt8{12}; + const le_int8_t expectedInt8neg{-4}; + static constexpr const std::string expectedString{"t!s$ a str "}; + static constexpr const std::span expectedBytes{rawBytes}; + + // Buffer to serialize into and deserialize from + const std::size_t bufferSize{ + sizeof(expectedInt8) + sizeof(expectedInt8neg) + 1 + expectedString.size() + + sizeof(bytesCount) + expectedBytes.size() + }; + std::array buffer{}; + + // Serilize into buffer + buffer_writer_t writer(buffer); + auto status{writer.write_endian_integer(expectedInt8) + .write_endian_integer(expectedInt8neg) + .write_string(expectedString) + .write_endian_integer(le_uint64_t{bytesCount}) + .write_bytes(expectedBytes) + .has_error()}; + EXPECT_FALSE(status); + EXPECT_EQ(writer.bytes_written(), bufferSize); + + // Derserialize from buffer + buffer_reader_t reader(buffer); + le_int8_t int8{0}; + le_int8_t int8neg{0}; + std::string_view actualString; + le_uint64_t actualBytesCount{0}; + std::span actualBytes; + status = reader.read_endian_integer(int8) + .read_endian_integer(int8neg) + .read_string(actualString) + .read_endian_integer(actualBytesCount) + .read_bytes(bytesCount, actualBytes) + .has_error(); + EXPECT_FALSE(status); + EXPECT_EQ(reader.bytes_read(), writer.bytes_written()); + + // Check that we read what we wrote + EXPECT_EQ(int8.get(), expectedInt8.get()); + EXPECT_EQ(int8neg.get(), expectedInt8neg.get()); + EXPECT_EQ(expectedString, actualString); + EXPECT_EQ(bytesCount, actualBytesCount.get()); + EXPECT_TRUE(std::ranges::equal(expectedBytes, actualBytes)); +} + +TEST(BufferReaderWriterTest, VarintSmallValues) +{ + // Values 0-127 should encode as 1 byte + std::array buffer{}; + buffer_writer_t writer(buffer); + + writer.write_varint(0); // Should write: [0x00] + writer.write_varint(1); // Should write: [0x01] + writer.write_varint(127); // Should write: [0x7F] + + EXPECT_FALSE(writer.has_error()); + EXPECT_EQ(writer.bytes_written(), 3); // 1 + 1 + 1 = 3 bytes + + // Read back + buffer_reader_t reader(buffer); + uint64_t v0, v1, v127; + reader.read_varint(v0).read_varint(v1).read_varint(v127); + + EXPECT_FALSE(reader.has_error()); + EXPECT_EQ(v0, 0); + EXPECT_EQ(v1, 1); + EXPECT_EQ(v127, 127); +} + +TEST(BufferReaderWriterTest, VarintMediumValues) +{ + // Value 300 from the example (should be 2 bytes: [0xAC, 0x02]) + std::array buffer{}; + buffer_writer_t writer(buffer); + + writer.write_varint(300); + + EXPECT_FALSE(writer.has_error()); + EXPECT_EQ(writer.bytes_written(), 2); + + // Verify exact encoding + EXPECT_EQ(buffer[0], std::byte{0xAC}); + EXPECT_EQ(buffer[1], std::byte{0x02}); + + // Read back + buffer_reader_t reader(buffer); + uint64_t value; + reader.read_varint(value); + + EXPECT_FALSE(reader.has_error()); + EXPECT_EQ(value, 300); +} + +TEST(BufferReaderWriterTest, VarintLargeValues) +{ + // Test larger values + std::array buffer{}; + buffer_writer_t writer(buffer); + + writer.write_varint(16384); // 3 bytes + writer.write_varint(1u << 20); // 4 bytes + writer.write_varint(1ull << 32); // 5 bytes + + EXPECT_FALSE(writer.has_error()); + + buffer_reader_t reader(buffer); + uint64_t v1, v2, v3; + reader.read_varint(v1).read_varint(v2).read_varint(v3); + + EXPECT_FALSE(reader.has_error()); + EXPECT_EQ(v1, 16384); + EXPECT_EQ(v2, 1u << 20); + EXPECT_EQ(v3, 1ull << 32); +} + +TEST(BufferReaderWriterTest, EmptyString) +{ + // Empty strings should work + std::array buffer{}; + buffer_writer_t writer(buffer); + + writer.write_string(""); + + EXPECT_FALSE(writer.has_error()); + EXPECT_EQ(writer.bytes_written(), 1); // Just varint(0) + + buffer_reader_t reader(buffer); + std::string_view str; + reader.read_string(str); + + EXPECT_FALSE(reader.has_error()); + EXPECT_TRUE(str.empty()); +} + +TEST(BufferReaderWriterTest, VarintOverflow) +{ + // Test error handling: reading a malformed varint + std::array buffer{}; + + // Fill with 11 bytes all having continuation bit set + for (int i = 0; i < 11; i++) + { + buffer[i] = std::byte{0xFF}; + } + + buffer_reader_t reader(buffer); + uint64_t value; + reader.read_varint(value); + + // Should error after 10 bytes + EXPECT_TRUE(reader.has_error()); + EXPECT_EQ(*reader.error(), serialization_error_k::invalid_variant_k); +} diff --git a/src/serialization/buffer_writer.cpp b/src/serialization/buffer_writer.cpp new file mode 100644 index 0000000..51ba845 --- /dev/null +++ b/src/serialization/buffer_writer.cpp @@ -0,0 +1 @@ +#include "serialization/buffer_writer.h" diff --git a/src/serialization/common.cpp b/src/serialization/common.cpp new file mode 100644 index 0000000..1ff6681 --- /dev/null +++ b/src/serialization/common.cpp @@ -0,0 +1,21 @@ +#include "serialization/common.h" + +namespace serialization +{ + +[[nodiscard]] auto to_span(std::string_view view) noexcept -> std::span +{ + return std::as_bytes(std::span(view.data(), view.size())); +} + +[[nodiscard]] auto to_string_view(std::span span) noexcept -> std::string_view +{ + if (span.empty()) + { + return {}; + } + + return {reinterpret_cast(span.data()), span.size()}; +} + +} // namespace serialization diff --git a/src/serialization/crc32.cpp b/src/serialization/crc32.cpp new file mode 100644 index 0000000..8d07bef --- /dev/null +++ b/src/serialization/crc32.cpp @@ -0,0 +1,31 @@ +#include "serialization/crc32.h" +#include +#include + +namespace serialization +{ + +// update updates the stored crc with new data, without XORing with 0xFFFFFFFF +void crc32_t::update(std::span data) noexcept +{ + for (const std::byte byte : data) + { + const std::uint32_t index{(std::to_integer(byte) ^ m_crc) & 0xFF}; + + m_crc = (m_crc >> detail::CRC32_BITS) ^ TABLE[index]; + } +} + +// finalize returns the stored crc XORed with 0xFFFFFFFF +[[nodiscard]] auto crc32_t::finalize() const noexcept -> std::uint32_t +{ + return m_crc ^ detail::CRC32_DEFAULT_VALUE; +} + +// reset sets crc to 0xFFFFFFFF +void crc32_t::reset() noexcept +{ + m_crc = detail::CRC32_DEFAULT_VALUE; +} + +} // namespace serialization diff --git a/src/serialization/crc32_test.cpp b/src/serialization/crc32_test.cpp new file mode 100644 index 0000000..e6d9a44 --- /dev/null +++ b/src/serialization/crc32_test.cpp @@ -0,0 +1,61 @@ +#include "gtest/gtest.h" + +#include "serialization/crc32.h" + +using namespace serialization; + +TEST(CRC32Test, TableGeneration) +{ + // First few known values from standard CRC32 table + // These are from IEEE 802.3 / ZIP / PNG standards + + static constexpr const auto TABLE = detail::generate_crc32_table(); + EXPECT_EQ(TABLE[0], 0x00000000); + EXPECT_EQ(TABLE[1], 0x77073096); + EXPECT_EQ(TABLE[2], 0xEE0E612C); + EXPECT_EQ(TABLE[255], 0x2D02EF8D); +} + +TEST(CRC32Test, KnownValues) +{ + // Test against known CRC32 values + + // CRC32("") = 0x00000000 + crc32_t crc1; + EXPECT_EQ(crc1.finalize(), 0x00000000); + + // CRC32("123456789") = 0xCBF43926 + crc32_t crc2; + std::string test = "123456789"; + crc2.update(std::as_bytes(std::span(test))); + EXPECT_EQ(crc2.finalize(), 0xCBF43926); + + // CRC32("The quick brown fox jumps over the lazy dog") = 0x414FA339 + crc32_t crc3; + std::string test2 = "The quick brown fox jumps over the lazy dog"; + crc3.update(std::as_bytes(std::span(test2))); + EXPECT_EQ(crc3.finalize(), 0x414FA339); +} + +TEST(CRC32Test, IncrementalUpdate) +{ + // Verify incremental updates work + std::string data = "Hello, World!"; + auto substr1 = data.substr(0, 5); + auto substr2 = data.substr(5, 2); + auto substr3 = data.substr(7); + + // Single update + crc32_t crc1; + crc1.update(std::as_bytes(std::span(data))); + uint32_t result1 = crc1.finalize(); + + // Split into chunks + crc32_t crc2; + crc2.update(std::as_bytes(std::span(substr1))); // "Hello" + crc2.update(std::as_bytes(std::span(substr2))); // ", " + crc2.update(std::as_bytes(std::span(substr3))); // "World!" + uint32_t result2 = crc2.finalize(); + + EXPECT_EQ(result1, result2); +} diff --git a/src/serialization/endian_integer_test.cpp b/src/serialization/endian_integer_test.cpp new file mode 100644 index 0000000..0364e27 --- /dev/null +++ b/src/serialization/endian_integer_test.cpp @@ -0,0 +1,57 @@ +#include + +#include "gtest/gtest.h" + +#include "serialization/endian_integer.h" + +using namespace serialization; + +template + requires std::is_integral_v +auto to_bytes(T value) noexcept -> std::array +{ + std::array bytes; + std::memcpy(bytes.data(), &value, sizeof(T)); + return bytes; +} + +template struct TypeValuePair +{ + using Type = T; + static constexpr const T Value = V; +}; + +using TestTypes = ::testing::Types< + TypeValuePair, + TypeValuePair, + TypeValuePair, + TypeValuePair, + TypeValuePair, + TypeValuePair, + TypeValuePair, + TypeValuePair>; + +template class EndianIntegerFixture : public testing::Test +{ + protected: + using IntType = T::Type; + static constexpr IntType Value = T::Value; +}; + +TYPED_TEST_SUITE(EndianIntegerFixture, TestTypes); + +TYPED_TEST(EndianIntegerFixture, Get) +{ + using IntType = typename TestFixture::IntType; + constexpr IntType podInt = TestFixture::Value; + + EXPECT_EQ(endian_integer{podInt}.get(), podInt); +} + +TYPED_TEST(EndianIntegerFixture, FromBytes) +{ + using IntType = typename TestFixture::IntType; + constexpr IntType podInt = TestFixture::Value; + + EXPECT_TRUE(std::ranges::equal(endian_integer{podInt}.bytes(), to_bytes(podInt))); +} diff --git a/src/serialization/varint_test.cpp b/src/serialization/varint_test.cpp new file mode 100644 index 0000000..7898be9 --- /dev/null +++ b/src/serialization/varint_test.cpp @@ -0,0 +1,124 @@ +#include + +#include +#include +#include + +#include "serialization/buffer_reader.h" +#include "serialization/buffer_writer.h" +#include "serialization/common.h" + +using namespace serialization; + +TEST(BufferReaderWriterTest, VarintSmallValues) +{ + // Values 0-127 should encode as 1 byte + std::array buffer{}; + buffer_writer_t writer(buffer); + + (void)writer.write_varint(0); // Should write: [0x00] + (void)writer.write_varint(1); // Should write: [0x01] + (void)writer.write_varint(127); // Should write: [0x7F] + + EXPECT_FALSE(writer.has_error()); + EXPECT_EQ(writer.bytes_written(), 3); // 1 + 1 + 1 = 3 bytes + + // Read back + buffer_reader_t reader(buffer); + uint64_t v0 = 0; + uint64_t v1 = 0; + uint64_t v127 = 0; + (void)reader.read_varint(v0).read_varint(v1).read_varint(v127); + + EXPECT_FALSE(reader.has_error()); + EXPECT_EQ(v0, 0); + EXPECT_EQ(v1, 1); + EXPECT_EQ(v127, 127); +} + +TEST(BufferReaderWriterTest, VarintMediumValues) +{ + // Value 300 from the example (should be 2 bytes: [0xAC, 0x02]) + std::array buffer{}; + buffer_writer_t writer(buffer); + + (void)writer.write_varint(300); + + EXPECT_FALSE(writer.has_error()); + EXPECT_EQ(writer.bytes_written(), 2); + + // Verify exact encoding + EXPECT_EQ(buffer[0], std::byte{0xAC}); + EXPECT_EQ(buffer[1], std::byte{0x02}); + + // Read back + buffer_reader_t reader(buffer); + uint64_t value = 0; + (void)reader.read_varint(value); + + EXPECT_FALSE(reader.has_error()); + EXPECT_EQ(value, 300); +} + +TEST(BufferReaderWriterTest, VarintLargeValues) +{ + // Test larger values + std::array buffer{}; + buffer_writer_t writer(buffer); + + (void)writer.write_varint(16384); // 3 bytes + (void)writer.write_varint(1u << 20); // 4 bytes + (void)writer.write_varint(1ull << 32); // 5 bytes + + EXPECT_FALSE(writer.has_error()); + + buffer_reader_t reader(buffer); + uint64_t v1{0}; + uint64_t v2{0}; + uint64_t v3{0}; + (void)reader.read_varint(v1).read_varint(v2).read_varint(v3); + + EXPECT_FALSE(reader.has_error()); + EXPECT_EQ(v1, 16384); + EXPECT_EQ(v2, 1u << 20); + EXPECT_EQ(v3, 1ull << 32); +} + +TEST(BufferReaderWriterTest, EmptyString) +{ + // Empty strings should work + std::array buffer{}; + buffer_writer_t writer(buffer); + + (void)writer.write_string(""); + + EXPECT_FALSE(writer.has_error()); + EXPECT_EQ(writer.bytes_written(), 1); // Just varint(0) + + buffer_reader_t reader(buffer); + std::string_view str; + (void)reader.read_string(str); + + EXPECT_FALSE(reader.has_error()); + EXPECT_TRUE(str.empty()); +} + +TEST(BufferReaderWriterTest, VarintOverflow) +{ + // Test error handling: reading a malformed varint + std::array buffer{}; + + // Fill with 11 bytes all having continuation bit set + for (int i = 0; i < 11; i++) + { + buffer[i] = std::byte{0xFF}; + } + + buffer_reader_t reader(buffer); + uint64_t value; + (void)reader.read_varint(value); + + // Should error after 10 bytes + EXPECT_TRUE(reader.has_error()); + EXPECT_EQ(*reader.error(), serialization_error_k::invalid_variant_k); +} diff --git a/src/structures/lsmtree/CMakeLists.txt b/src/structures/lsmtree/CMakeLists.txt index 772853f..28e3d44 100644 --- a/src/structures/lsmtree/CMakeLists.txt +++ b/src/structures/lsmtree/CMakeLists.txt @@ -31,7 +31,7 @@ target_link_libraries( libuuid::libuuid ) -add_executable(LSMTreeTest lsmtree_test.cpp) +add_executable(LSMTreeTest lsmtree_test_basic.cpp) set_target_properties(LSMTreeTest PROPERTIES CXX_STANDARD 23) target_link_libraries( LSMTreeTest @@ -48,6 +48,7 @@ target_link_libraries( FS DB Concurrency + Common ) include(GoogleTest) diff --git a/src/structures/lsmtree/compaction/level_zero_compactation.cpp b/src/structures/lsmtree/compaction/level_zero_compactation.cpp deleted file mode 100644 index e69de29..0000000 diff --git a/src/structures/lsmtree/levels/level.cpp b/src/structures/lsmtree/levels/level.cpp index c83dc67..18d77b2 100644 --- a/src/structures/lsmtree/levels/level.cpp +++ b/src/structures/lsmtree/levels/level.cpp @@ -102,12 +102,7 @@ auto level_t::record(const key_t &key) const noexcept { if (const auto &result = pSegment->record(key); !result.empty()) { - spdlog::debug( - "Found record {} at level {} in segment {}", - key.m_key, - index(), - pSegment->get_name() - ); + spdlog::debug("Found record at level {} in segment {}", index(), pSegment->get_name()); return result[0]; } } diff --git a/src/structures/lsmtree/levels/levels.cpp b/src/structures/lsmtree/levels/levels.cpp index 8e2b3b4..42fb1cf 100644 --- a/src/structures/lsmtree/levels/levels.cpp +++ b/src/structures/lsmtree/levels/levels.cpp @@ -163,7 +163,7 @@ auto levels_t::record(const key_t &key) const noexcept -> std::optionalrecord(key); if (result) { - spdlog::debug("Found key {} at level {}", key.m_key, currentLevel->index()); + spdlog::debug("Found key at level {}", currentLevel->index()); break; } } diff --git a/src/structures/lsmtree/lsmtree.cpp b/src/structures/lsmtree/lsmtree.cpp index 81b0582..85d8369 100644 --- a/src/structures/lsmtree/lsmtree.cpp +++ b/src/structures/lsmtree/lsmtree.cpp @@ -93,34 +93,33 @@ auto lsmtree_t::get(const key_t &key) noexcept -> std::optional if (!result.has_value()) { spdlog::info( - "cant find record {} in memtable... searching in flush queues size={}", - key.m_key, + "cant find record in memtable... searching in flush queues size={}", m_flushing_queue.size() ); result = m_flushing_queue.find(key); if (result.has_value()) { - spdlog::info("found record {} in flushing queues", key.m_key); + spdlog::info("found record in flushing queues"); } } else { - spdlog::info("found record {} in memtable", key.m_key); + spdlog::info("found record in memtable"); } // If key isn't in in-memory table, then it probably was flushed. // Lookup for the key in on-disk segments if (!result.has_value()) { - spdlog::info("cant find record {} in flush queues... searching in levels", key.m_key); + spdlog::info("cant find record in flush queues... searching in levels"); result = m_pLevels->record(key); if (!result.has_value()) { - spdlog::info("cant find record {} in levels :(((", key.m_key); + spdlog::info("cant find record in levels :((("); } else { - spdlog::info("found record {} in levels", key.m_key); + spdlog::info("found record in levels"); } } @@ -164,26 +163,25 @@ void lsmtree_t::memtable_flush_task(std::stop_token stoken) noexcept return; } - // auto memtables = m_flushing_queue.reserve(); - // if (!memtables.has_value()) - // { - // absl::SleepFor(absl::Milliseconds(50)); - // continue; - // } - // - // auto idx{1}; - // for (auto &memtable : memtables.value()) - // { - // spdlog::debug("LSMTree: Flushing {}/{} memtable into disk", idx++, - // memtables->size()); - // - // // TODO: Assert will crash the program, maybe we should - // // return an error code? - // absl::WriterMutexLock lock{&m_mutex}; - // bool ok{m_pLevels->flush_to_level0(std::move(memtable))}; - // assert(ok); - // } - // m_flushing_queue.consume(); + auto memtables = m_flushing_queue.reserve(); + if (!memtables.has_value()) + { + absl::SleepFor(absl::Milliseconds(50)); + continue; + } + + auto idx{1}; + for (auto &memtable : memtables.value()) + { + spdlog::debug("LSMTree: Flushing {}/{} memtable into disk", idx++, memtables->size()); + + // TODO: Assert will crash the program, maybe we should + // return an error code? + absl::WriterMutexLock lock{&m_mutex}; + bool ok{m_pLevels->flush_to_level0(std::move(memtable))}; + // assert(ok); + } + m_flushing_queue.consume(); } } diff --git a/src/structures/lsmtree/lsmtree_test.cpp b/src/structures/lsmtree/lsmtree_test.cpp deleted file mode 100644 index b893a09..0000000 --- a/src/structures/lsmtree/lsmtree_test.cpp +++ /dev/null @@ -1,143 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -#include "structures/lsmtree/lsmtree.h" -#include "config/config.h" - -namespace -{ -template -auto generateRandomNumber( - const TNumber min = std::numeric_limits::min(), - const TNumber max = std::numeric_limits::max() -) noexcept -> TNumber -{ - std::mt19937 rng{std::random_device{}()}; - if constexpr (std::is_same_v) - { - return std::uniform_int_distribution(min, max)(rng); - } - else if (std::is_same_v) - { - return std::uniform_int_distribution(min, max)(rng); - } - else if (std::is_same_v) - { - return std::uniform_real_distribution(min, max)(rng); - } - else if (std::is_same_v) - { - return std::uniform_real_distribution(min, max)(rng); - } - else - { - // TODO(vahag): better handle this case - return 0; - } -} - -auto generateRandomString(std::size_t length) noexcept -> std::string -{ - static const auto &alphabet = "0123456789" - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - - std::string result; - result.reserve(length); - while ((length--) != 0U) - { - result += alphabet[generateRandomNumber(0, sizeof(alphabet) - 2)]; - } - - return result; -} - -auto generateRandomStringPairVector(const std::size_t length) noexcept - -> std::vector> -{ - std::vector> result; - result.reserve(length); - for (std::string::size_type size = 0; size < length; size++) - { - result.emplace_back( - generateRandomString(generateRandomNumber(64, 64)), - generateRandomString(generateRandomNumber(64, 64)) - ); - } - return result; -} - -auto generateConfig( - std::uint64_t memtableThreshold, - std::uint64_t levelZeroThreshold, - std::int64_t levelNonZeroThreshold -) -{ - auto pResult{config::make_shared()}; - pResult->LSMTreeConfig.DiskFlushThresholdSize = memtableThreshold; - pResult->LSMTreeConfig.LevelZeroCompactionThreshold = levelZeroThreshold; - pResult->LSMTreeConfig.LevelNonZeroCompactionThreshold = levelNonZeroThreshold; - return pResult; -} - -using record_t = structures::memtable::memtable_t::record_t; - -} // namespace - -namespace raft::v1 -{ -template auto operator>>(TStream &stream, LogEntry &record) -> TStream & -{ - record.ParseFromString(stream.str()); - return stream; -} -} // namespace raft::v1 - -TEST(LSMTreeTest, FlushRegularSegment) -{ - using namespace structures; - - spdlog::set_level(spdlog::level::debug); - - const auto &randomKeys = generateRandomStringPairVector(16); - - auto pConfig{generateConfig(16, 64, 128)}; - - auto pManifest{db::manifest::make_shared( - pConfig->manifest_path(), - wal::wal_builder_t{} - .build(wal::log_storage_type_k::in_memory_k) - .value() - )}; - auto pWAL{ - wal::wal_builder_t{}.build(wal::log_storage_type_k::in_memory_k).value() - }; - auto pLsm{structures::lsmtree::lsmtree_builder_t{}.build(pConfig, pManifest, pWAL)}; - - for (const auto &kv : randomKeys) - { - auto rec{record_t{}}; - rec.m_key.m_key = kv.first; - rec.m_value.m_value = kv.second; - auto status{pLsm->put(rec)}; - EXPECT_TRUE(status.has_value()); - } - - for (const auto &kv : randomKeys) - { - auto rec{record_t{}}; - rec.m_key.m_key = kv.first; - rec.m_value.m_value = kv.second; - - auto expected = pLsm->get(rec.m_key); - - EXPECT_TRUE(expected.has_value()); - EXPECT_EQ(expected.value().m_key, rec.m_key); - EXPECT_EQ(expected.value().m_value, rec.m_value); - } -} diff --git a/src/structures/lsmtree/lsmtree_test_basic.cpp b/src/structures/lsmtree/lsmtree_test_basic.cpp new file mode 100644 index 0000000..ebedb9d --- /dev/null +++ b/src/structures/lsmtree/lsmtree_test_basic.cpp @@ -0,0 +1,346 @@ +#include +#include +#include +#include +#include +#include + +#include "structures/lsmtree/lsmtree.h" +#include "config/config.h" +#include "common/helpers.h" + +namespace +{ + +constexpr auto operator""_B(unsigned long long int x) -> std::size_t +{ + return x; +} + +constexpr auto operator""_KiB(unsigned long long int x) -> std::size_t +{ + return 1024ULL * x; +} + +constexpr auto operator""_MiB(unsigned long long int x) -> std::size_t +{ + return 1024_KiB * x; +} + +constexpr auto operator""_GiB(unsigned long long int x) -> std::size_t +{ + return 1024_MiB * x; +} + +auto generateConfig( + std::uint64_t memtableThreshold, + std::uint64_t levelZeroThreshold, + std::int64_t levelNonZeroThreshold +) +{ + auto pResult{config::make_shared()}; + pResult->LSMTreeConfig.DiskFlushThresholdSize = memtableThreshold; + pResult->LSMTreeConfig.LevelZeroCompactionThreshold = levelZeroThreshold; + pResult->LSMTreeConfig.LevelNonZeroCompactionThreshold = levelNonZeroThreshold; + return pResult; +} + +auto generateLSMTree(config::shared_ptr_t pConfig) +{ + auto pManifest{db::manifest::make_shared( + pConfig->manifest_path(), + wal::wal_builder_t{} + .build(wal::log_storage_type_k::in_memory_k) + .value() + )}; + auto pWAL{ + wal::wal_builder_t{}.build(wal::log_storage_type_k::in_memory_k).value() + }; + + return structures::lsmtree::lsmtree_builder_t{}.build(pConfig, pManifest, pWAL); +} + +using record_t = structures::memtable::memtable_t::record_t; + +} // namespace + +namespace raft::v1 +{ +template auto operator>>(TStream &stream, LogEntry &record) -> TStream & +{ + record.ParseFromString(stream.str()); + return stream; +} +} // namespace raft::v1 + +TEST(LSMTreeTestBasicCRUD, PutAndGet) +{ + auto pConfig{generateConfig(16_B, 64_B, 128_B)}; + auto pLsm{generateLSMTree(pConfig)}; + + record_t rec; + rec.m_key.m_key = "Alice"; + rec.m_value.m_value = "123"; + + auto status{pLsm->put(rec)}; + EXPECT_TRUE(status.has_value()); + + auto expected{pLsm->get(rec.m_key)}; + EXPECT_TRUE(expected.has_value()); + EXPECT_EQ(expected->m_key, rec.m_key); + EXPECT_EQ(expected->m_value, rec.m_value); +} + +TEST(LSMTreeTestBasicCRUD, PutMultipleRecords) +{ + const auto &randomKeys = common::generateRandomStringPairVector(16_B); + auto pConfig{generateConfig(16_B, 64_B, 128_B)}; + auto pLsm{generateLSMTree(pConfig)}; + + for (const auto &kv : randomKeys) + { + auto rec{record_t{}}; + rec.m_key.m_key = kv.first; + rec.m_value.m_value = kv.second; + auto status{pLsm->put(rec)}; + EXPECT_TRUE(status.has_value()); + } + + for (const auto &kv : randomKeys) + { + auto rec{record_t{}}; + rec.m_key.m_key = kv.first; + rec.m_value.m_value = kv.second; + + auto expected = pLsm->get(rec.m_key); + + EXPECT_TRUE(expected.has_value()); + EXPECT_EQ(expected.value().m_key, rec.m_key); + EXPECT_EQ(expected.value().m_value, rec.m_value); + } +} + +TEST(LSMTreeTestBasicCRUD, GetNonExistentKey) +{ + const auto &randomKeys = common::generateRandomStringPairVector(16_B); + auto pConfig{generateConfig(16_B, 64_B, 128_B)}; + auto pLsm{generateLSMTree(pConfig)}; + + record_t rec; + rec.m_key.m_key = "Alice"; + rec.m_value.m_value = "123"; + + auto expected{pLsm->get(rec.m_key)}; + EXPECT_FALSE(expected.has_value()); +} + +TEST(LSMTreeTestBasicCRUD, UpdateExistingKey) +{ + const auto &randomKeys = common::generateRandomStringPairVector(16_B); + auto pConfig{generateConfig(16_B, 64_B, 128_B)}; + auto pLsm{generateLSMTree(pConfig)}; + + record_t rec; + rec.m_key.m_key = "Alice"; + rec.m_value.m_value = "123"; + + auto status{pLsm->put(rec)}; + EXPECT_TRUE(status.has_value()); + + auto expected{pLsm->get(rec.m_key)}; + EXPECT_TRUE(expected.has_value()); + EXPECT_EQ(expected->m_key, rec.m_key); + EXPECT_EQ(expected->m_value, rec.m_value); + + rec.m_value.m_value = "456"; + status = pLsm->put(rec); + EXPECT_TRUE(status.has_value()); + + expected = pLsm->get(rec.m_key); + EXPECT_TRUE(expected.has_value()); + EXPECT_EQ(expected->m_key, rec.m_key); + EXPECT_EQ(expected->m_value, rec.m_value); +} + +TEST(LSMTreeTestBasicCRUD, EmptyTreeGet) +{ + const auto &randomKeys = common::generateRandomStringPairVector(16_B); + auto pConfig{generateConfig(16_B, 64_B, 128_B)}; + auto pLsm{generateLSMTree(pConfig)}; + + record_t rec; + rec.m_key.m_key = "Alice"; + rec.m_value.m_value = "123"; + + auto expected{pLsm->get(rec.m_key)}; + EXPECT_FALSE(expected.has_value()); +} + +TEST(LSMTreeTestBasicCRUD, LargeKeyValuePair) +{ + auto pConfig{generateConfig(16_MiB, 64_MiB, 128_MiB)}; + auto pLsm{generateLSMTree(pConfig)}; + + record_t rec; + rec.m_key.m_key = common::generateRandomString(4_MiB); + rec.m_value.m_value = common::generateRandomString(8_MiB); + + auto status{pLsm->put(rec)}; + EXPECT_TRUE(status.has_value()); + + auto expected{pLsm->get(rec.m_key)}; + EXPECT_TRUE(expected.has_value()); + EXPECT_EQ(expected->m_key, rec.m_key); + EXPECT_EQ(expected->m_value, rec.m_value); +} + +TEST(LSMTreeTestBasicCRUD, LargeKeyValuePairFlush) +{ + auto pConfig{generateConfig(16_MiB, 64_MiB, 128_MiB)}; + auto pLsm{generateLSMTree(pConfig)}; + + record_t rec; + rec.m_key.m_key = common::generateRandomString(14_MiB); + rec.m_value.m_value = common::generateRandomString(18_MiB); + + auto status{pLsm->put(rec)}; + EXPECT_TRUE(status.has_value()); + + auto expected{pLsm->get(rec.m_key)}; + EXPECT_TRUE(expected.has_value()); + EXPECT_EQ(expected->m_key, rec.m_key); + EXPECT_EQ(expected->m_value, rec.m_value); +} + +TEST(LSMTreeTestBasicCRUD, MultipleLargeKeyValuePairsFlush) +{ + auto pConfig{generateConfig(2_MiB, 4_MiB, 8_MiB)}; + auto pLsm{generateLSMTree(pConfig)}; + + const auto recordCount{16}; + std::vector records; + records.reserve(recordCount); + + auto count{recordCount}; + while ((count--) != 0ULL) + { + record_t rec; + rec.m_key.m_key = common::generateRandomString(64_KiB); + rec.m_value.m_value = common::generateRandomString(64_KiB); + records.emplace_back(std::move(rec)); + } + + for (const auto &rec : records) + { + auto status{pLsm->put(rec)}; + EXPECT_TRUE(status.has_value()); + } + + std::this_thread::sleep_for(std::chrono::seconds(1)); + + std::size_t idx{0}; + for (const auto &rec : records) + { + spdlog::info("Verifying record {}/{}", ++idx, recordCount); + auto expected{pLsm->get(rec.m_key)}; + EXPECT_TRUE(expected.has_value()); + spdlog::info( + "expected->m_key.m_key.size()={}, rec.m_key.m_key={}", + expected->m_key.m_key.size(), + rec.m_key.m_key.size() + ); + EXPECT_EQ(expected->m_key.m_key, rec.m_key.m_key); + // EXPECT_EQ(expected->m_value.m_value, rec.m_value.m_value); + ++idx; + } +} + +TEST(LSMTreeTestBasicCRUD, MultipleSmallKeyValuePairsFlush) +{ + auto pConfig{generateConfig(16_KiB, 64_KiB, 128_KiB)}; + auto pLsm{generateLSMTree(pConfig)}; + + const auto recordCount{16}; + std::vector records; + records.reserve(recordCount); + + auto count{recordCount}; + while ((count--) != 0ULL) + { + record_t rec; + rec.m_key.m_key = common::generateRandomString(14_KiB); + rec.m_value.m_value = common::generateRandomString(18_KiB); + records.emplace_back(std::move(rec)); + } + + for (const auto &rec : records) + { + auto status{pLsm->put(rec)}; + EXPECT_TRUE(status.has_value()); + } + + for (const auto &rec : records) + { + auto expected{pLsm->get(rec.m_key)}; + EXPECT_TRUE(expected.has_value()); + EXPECT_EQ(expected->m_key, rec.m_key); + EXPECT_EQ(expected->m_value, rec.m_value); + } +} + +TEST(LSMTreeTestBasicCRUD, EmptyKeyValue) +{ + auto pConfig{generateConfig(16_B, 64_B, 128_B)}; + auto pLsm{generateLSMTree(pConfig)}; + + // Empty key and value + record_t rec; + rec.m_key.m_key = ""; + rec.m_value.m_value = ""; + + auto status{pLsm->put(rec)}; + EXPECT_TRUE(status.has_value()); + + auto expected{pLsm->get(rec.m_key)}; + EXPECT_TRUE(expected.has_value()); + EXPECT_EQ(expected->m_key, rec.m_key); + EXPECT_EQ(expected->m_value, rec.m_value); + + // Empty value + rec.m_key.m_key = "alice"; + rec.m_value.m_value = ""; + + status = pLsm->put(rec); + EXPECT_TRUE(status.has_value()); + + expected = pLsm->get(rec.m_key); + EXPECT_TRUE(expected.has_value()); + EXPECT_EQ(expected->m_key, rec.m_key); + EXPECT_EQ(expected->m_value, rec.m_value); + + // Empty key + rec.m_key.m_key = ""; + rec.m_value.m_value = "123"; + + status = pLsm->put(rec); + EXPECT_TRUE(status.has_value()); + + expected = pLsm->get(rec.m_key); + EXPECT_TRUE(expected.has_value()); + EXPECT_EQ(expected->m_key, rec.m_key); + EXPECT_EQ(expected->m_value, rec.m_value); +} + +int main(int argc, char **argv) +{ + spdlog::set_level(spdlog::level::debug); + + // std::filesystem::create_directory("./segments"); + + testing::InitGoogleTest(&argc, argv); + auto res{RUN_ALL_TESTS()}; + + // std::filesystem::remove_all("./segments"); + + return res; +} diff --git a/src/structures/lsmtree/segments/lsmtree_regular_segment.cpp b/src/structures/lsmtree/segments/lsmtree_regular_segment.cpp index 4cb28d5..0af9ddd 100644 --- a/src/structures/lsmtree/segments/lsmtree_regular_segment.cpp +++ b/src/structures/lsmtree/segments/lsmtree_regular_segment.cpp @@ -55,7 +55,7 @@ regular_segment_t::regular_segment_t( auto regular_segment_t::record(const hashindex::hashindex_t::offset_t &offset) -> std::optional { - std::fstream ss{get_path(), std::ios::in}; + std::fstream ss{get_path(), std::ios::in | std::fstream::binary}; ss.seekg(offset); memtable_t::record_t record; @@ -101,7 +101,7 @@ void regular_segment_t::flush() const auto hashIndexBlockOffset{stringStream.tellp()}; for (const auto &[key, offset] : m_hashIndex) { - stringStream << key.m_key << ' ' << offset << '\n'; + stringStream << key.m_key << ' ' << offset; // << '\n'; } // Calculate size of the index block @@ -111,14 +111,14 @@ void regular_segment_t::flush() const auto footerBlockOffset{stringStream.tellp()}; // Serialize footer - stringStream << hashIndexBlockOffset << ' ' << hashIndexBlockSize << std::endl; + stringStream << hashIndexBlockOffset << ' ' << hashIndexBlockSize; // << '\n'; // Add padding to the footer end const auto footerPaddingSize{footerSize - (stringStream.tellp() - footerBlockOffset)}; - stringStream << std::string(footerPaddingSize, ' ') << std::endl; + stringStream << std::string(footerPaddingSize, ' '); // << '\n'; // Flush the segment into the disk - std::fstream stream(get_path(), std::fstream::trunc | std::fstream::out); + std::fstream stream{get_path(), std::fstream::binary | std::fstream::trunc | std::fstream::out}; if (!stream.is_open()) { throw std::runtime_error("unable to flush segment for path " + m_path.string()); @@ -210,7 +210,7 @@ void regular_segment_t::restore() // TODO(lnikon): Add validations on file size. Need 'RandomAccessFile'. void regular_segment_t::restore_index() { - std::fstream sst(m_path); + std::fstream sst(m_path, std::fstream::binary); if (!sst.is_open()) { // TODO(lnikon): Better way to handle this case. Without exceptions. From ec323b9258b718d76158e8861cc28814d41755e2 Mon Sep 17 00:00:00 2001 From: lnikon Date: Mon, 1 Dec 2025 00:48:44 +0400 Subject: [PATCH 17/17] Testing: - Introduce sequence numbers for record ordering - Rewrite segment flush logic with binary buffer writer --- docs/sstable_binary_format_spec_v1.md | 126 +++++++++ src/db/db.cpp | 20 +- src/include/serialization/buffer_writer.h | 2 +- src/include/serialization/common.h | 3 + .../structures/lsmtree/lsmtree_types.h | 2 + .../structures/lsmtree/segment_serializer.h | 22 ++ src/include/structures/memtable/memtable.h | 34 +-- src/include/structures/skiplist/skiplist.h | 22 +- src/serialization/common.cpp | 11 + src/structures/lsmtree/CMakeLists.txt | 1 + src/structures/lsmtree/lsmtree.cpp | 26 +- .../segments/lsmtree_regular_segment.cpp | 266 ++++++++++++++---- .../lsmtree/segments/segment_serializer.cpp | 0 src/structures/memtable/CMakeLists.txt | 1 - src/structures/memtable/memtable.cpp | 36 +-- src/structures/memtable/memtable_test.cpp | 29 +- 16 files changed, 459 insertions(+), 142 deletions(-) create mode 100644 docs/sstable_binary_format_spec_v1.md create mode 100644 src/include/structures/lsmtree/segment_serializer.h create mode 100644 src/structures/lsmtree/segments/segment_serializer.cpp diff --git a/docs/sstable_binary_format_spec_v1.md b/docs/sstable_binary_format_spec_v1.md new file mode 100644 index 0000000..292a59f --- /dev/null +++ b/docs/sstable_binary_format_spec_v1.md @@ -0,0 +1,126 @@ +# SortedStringTable Binary Format Specification v1 + +## Overview +This document describes the tape format of the on-disk segments. + +## Layout + +``` +HEADER (24 bytes): ++--------+--------+--------+--------+--------+--------+ +| magic | ver | body | index | size | crc32 | +| (u32) | (u32) | offset | offset | (u32) | (u32) | +| bytes | bytes | (u32) | (u32) | bytes | bytes | +| 0-3 | 4-7 | 8-11 | 12-15 | 16-19 | 20-23 | ++--------+--------+--------+--------+--------+--------+ + +BODY (variable): ++----------+-----+----------+-----+-----------+ +| key_len | key | val_len | val | timestamp | +| (varint) | var | (varint) | var | (u32) | ++----------+-----+----------+-----+-----------+ +Repeated for each entry + +INDEX (variable): ++----------+-----+----------+ +| key_len | key | body_off | +| (varint) | var | (u32) | ++----------+-----+----------+ +Every entry (dense) or every k-th entry (sparse) +``` + +## Sections + +### Header + +``` +Header { + u32 magic = 0xDEADBEEF; + u32 version = 1; + u32 body_offset; // where body starts + u32 index_offset; // where index starts + u32 payload_size; // total size (body + index) + u32 checksum; // CRC32 of entire file except this field +}; +``` + +### Body + +``` +Body { + for each entry in table: + varint key_len + u8[key_len] key + varint value_len + u8[value_len] value + u32 timestamp +}; +``` + +### Index + +``` +Index { + for each entry in table: // dense initially + varint key_len + u8[key_len] key + u32 offset_in_body +}; +``` + +## CRC32 Computation +CRC32 calculated based on the whole file excluding the checksum field itself. + +## Edge Cases + +1. **Truncated File**: If file ends before reaching `index_offset`, + deserialization fails with error "File truncated". + +2. **Checksum Mismatch**: If computed CRC32 ≠ stored checksum, + throw error "Data corruption detected". File must not be read. + +3. **Bad Magic Number**: If magic != 0xDEADBEEF, throw error + "Invalid SST file format". + +4. **Empty Segment**: 0 entries allowed; `body_offset == index_offset`. + Valid serialized state with header only. + +5. **Invalid Offsets**: If `body_offset < 24` or `index_offset < body_offset`, + file is malformed, throw error "Invalid file structure". + +6. **Payload Size Mismatch**: If `payload_size != (file_size - 24)`, + file may be truncated or corrupted, throw error "Payload size mismatch". + +## Example + + **Input Table:** + - Entry 1: key="foo", value="bar", timestamp=100 + - Entry 2: key="x", value="yz", timestamp=200 + + **Binary Output (hex dump):** + +``` + HEADER (bytes 0-23): + EF BE AD DE 01 00 00 00 18 00 00 00 2E 00 00 00 24 00 00 00 08 E2 98 1F + ^magic ^version ^body_off ^index_off ^payload_sz ^checksum + + BODY (bytes 24-45, 22 bytes total): + 03 66 6F 6F 03 62 61 72 64 00 00 00 01 78 02 79 7A C8 00 00 00 + ^key_len(3) ^"foo" ^val_len(3) ^"bar" ^ts(100) ^key_len(1) ^"x" ^val_len(2) ^"yz" ^ts(200) + + INDEX (bytes 46-59, 14 bytes total): + 03 66 6F 6F 00 00 00 00 01 78 0D 00 00 00 + ^key_len(3) ^"foo" ^offset(0) ^key_len(1) ^"x" ^offset(13) + + **Offset Summary:** + | Section | Offset | Size | End | + |---------|--------|------|------| + | Header | 0 | 24 | 23 | + | Body | 24 | 22 | 45 | + | Index | 46 | 14 | 59 | + | **Total** | **0** | **60** | **59** | + + **CRC32 Calculation:** + Computed over bytes 0-19 (header without checksum) + 24-59 (body + index) = 0x1F98E208 +``` + diff --git a/src/db/db.cpp b/src/db/db.cpp index 2b24bd7..e4d0d27 100644 --- a/src/db/db.cpp +++ b/src/db/db.cpp @@ -72,11 +72,13 @@ db_t::db_t( assert(m_pLSMtree); // Setup Raft callbacks - m_pConsensusModule->setOnCommitCallback([this](const raft::v1::LogEntry &entry) - { return onRaftCommit(entry); }); + m_pConsensusModule->setOnCommitCallback( + [this](const raft::v1::LogEntry &entry) -> bool { return onRaftCommit(entry); } + ); - m_pConsensusModule->setOnLeaderChangeCallback([this](bool isLeader) - { onLeaderChange(isLeader); }); + m_pConsensusModule->setOnLeaderChangeCallback( + [this](bool isLeader) -> void { onLeaderChange(isLeader); } + ); } auto db_t::start() -> bool @@ -296,7 +298,7 @@ void db_t::handleClientRequest(client_request_t request) // Handle Raft result asynchronously m_requestPool.enqueue( - [this, requestId, deadline, raftFuture = std::move(raftFuture)] mutable + [this, requestId, deadline, raftFuture = std::move(raftFuture)] mutable -> void { // Check if already timed out auto timeout = deadline - std::chrono::steady_clock::now(); @@ -367,9 +369,11 @@ auto db_t::onRaftCommit(const raft::v1::LogEntry &entry) -> bool tinykvpp::v1::DatabaseOperation op; op.ParseFromString(entry.payload()); - structures::memtable::memtable_t::record_t record; - record.m_key.m_key = op.key(); - record.m_value.m_value = op.value(); + structures::memtable::memtable_t::record_t record{ + structures::memtable::memtable_t::record_t::key_t{op.key()}, + structures::memtable::memtable_t::record_t::value_t{op.value()}, + structures::memtable::memtable_t::record_t::sequence_number_t{entry.index()} + }; switch (op.type()) { diff --git a/src/include/serialization/buffer_writer.h b/src/include/serialization/buffer_writer.h index 95e76a7..aa41faf 100644 --- a/src/include/serialization/buffer_writer.h +++ b/src/include/serialization/buffer_writer.h @@ -43,7 +43,7 @@ class buffer_writer_t final std::array scratch{}; std::size_t count{0}; - while (value > 127) + while (value >= 128) { scratch[count++] = std::byte{static_cast((value & 0x7F))} | std::byte{0x80}; diff --git a/src/include/serialization/common.h b/src/include/serialization/common.h index c300f0d..b3cd896 100644 --- a/src/include/serialization/common.h +++ b/src/include/serialization/common.h @@ -41,4 +41,7 @@ using byte_t = std::byte; // to_string_view converts std::byte span into a std::string_view [[nodiscard]] auto to_string_view(std::span span) noexcept -> std::string_view; +// varint_size returns minimum number of bytes required to represent the value +[[nodiscard]] auto varint_size(std::size_t value) noexcept -> std::size_t; + } // namespace serialization diff --git a/src/include/structures/lsmtree/lsmtree_types.h b/src/include/structures/lsmtree/lsmtree_types.h index 212fcea..3981e77 100644 --- a/src/include/structures/lsmtree/lsmtree_types.h +++ b/src/include/structures/lsmtree/lsmtree_types.h @@ -11,5 +11,7 @@ using memtable_t = memtable::memtable_t; using record_t = memtable::memtable_t::record_t; using key_t = memtable_t::record_t::key_t; using value_t = memtable_t::record_t::value_t; +using sequence_number_t = memtable_t::record_t::sequence_number_t; +using timestamp_t = memtable_t::record_t::timestamp_t; } // namespace structures::lsmtree diff --git a/src/include/structures/lsmtree/segment_serializer.h b/src/include/structures/lsmtree/segment_serializer.h new file mode 100644 index 0000000..34de9e7 --- /dev/null +++ b/src/include/structures/lsmtree/segment_serializer.h @@ -0,0 +1,22 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "serialization/buffer_writer.h" +#include "serialization/common.h" +#include "serialization/endian_integer.h" +#include "structures/lsmtree/lsmtree_types.h" +#include "structures/lsmtree/segments/lsmtree_regular_segment.h" +#include "structures/memtable/memtable.h" + +namespace structures::lsmtree::segments::regular_segment +{ + +} // namespace structures::lsmtree::segments::regular_segment diff --git a/src/include/structures/memtable/memtable.h b/src/include/structures/memtable/memtable.h index df79bf0..2e9112f 100644 --- a/src/include/structures/memtable/memtable.h +++ b/src/include/structures/memtable/memtable.h @@ -18,6 +18,8 @@ class memtable_t public: struct record_t { + using sequence_number_t = std::uint64_t; + struct key_t { using storage_type_t = std::string; @@ -57,23 +59,19 @@ class memtable_t struct timestamp_t { // Used to differentiate between keys with the same timestamp - using clock_t = std::chrono::high_resolution_clock; - using underlying_value_type_t = std::chrono::time_point; - - timestamp_t(); - - auto operator<(const timestamp_t &other) const -> bool; - - static void swap(timestamp_t &lhs, timestamp_t &rhs); + using clock_t = std::chrono::system_clock; + using precision_t = std::chrono::nanoseconds; + using time_point_t = std::chrono::time_point; template void write(TSTream &outStream) const; template void read(TSTream &outStream); - underlying_value_type_t m_value; + time_point_t m_value{clock_t::now()}; }; - record_t(key_t key, value_t value); + // TODO(lnikon): Get rid of default ctor record_t() = default; + record_t(key_t key, value_t value, sequence_number_t sequenceNumber); auto operator<(const record_t &record) const -> bool; auto operator>(const record_t &record) const -> bool; @@ -84,9 +82,10 @@ class memtable_t template void write(TSTream &outStream) const; template void read(TSTream &outStream); - key_t m_key; - value_t m_value; - timestamp_t m_timestamp; + key_t m_key; + value_t m_value; + sequence_number_t m_sequenceNumber{0}; + timestamp_t m_timestamp; }; struct record_comparator_by_key_t @@ -106,13 +105,6 @@ class memtable_t using const_iterator = typename storage_t::const_iterator; using value_type = typename storage_t::value_type; - memtable_t() = default; - memtable_t(const memtable_t &) = default; - auto operator=(const memtable_t &) -> memtable_t & = default; - memtable_t(memtable_t &&) = default; - auto operator=(memtable_t &&) -> memtable_t & = default; - ~memtable_t() = default; - /** * @brief * @@ -227,7 +219,7 @@ template void memtable_t::record_t::value_t::read(TSTream &ou // ------------------------------------------------ template void memtable_t::record_t::timestamp_t::write(TSTream &outStream) const { - outStream << m_value.time_since_epoch().count(); + outStream << static_cast(m_value.time_since_epoch().count()); } template void memtable_t::record_t::timestamp_t::read(TSTream &outStream) diff --git a/src/include/structures/skiplist/skiplist.h b/src/include/structures/skiplist/skiplist.h index 4a52588..e0aa88c 100644 --- a/src/include/structures/skiplist/skiplist.h +++ b/src/include/structures/skiplist/skiplist.h @@ -13,7 +13,7 @@ namespace structures::skiplist const std::int64_t max_height = 12; -template class skiplist_t +template class skiplist_t { public: template struct iterator_base; @@ -23,19 +23,19 @@ template class skiplist_t struct node_t { - node_t(record_gt rec, std::int64_t level) + node_t(TRecord rec, std::int64_t level) : record(std::move(rec)), forward(level + 1, nullptr) { } - record_gt record; + TRecord record; std::vector forward; }; - using value_type = record_gt; - using reference = record_gt &; - using const_reference = const record_gt &; + using value_type = TRecord; + using reference = TRecord &; + using const_reference = const TRecord &; using difference_type = std::ptrdiff_t; using size_type = std::size_t; using index_type = std::size_t; @@ -134,7 +134,7 @@ template class skiplist_t return const_iterator(nullptr); } - auto find(const typename record_gt::key_t &key) const noexcept -> std::optional + auto find(const typename TRecord::key_t &key) const noexcept -> std::optional { node_shared_ptr_t current{m_head}; for (std::int64_t i = m_level; i >= 0; i--) @@ -154,7 +154,7 @@ template class skiplist_t return std::nullopt; } - void emplace(record_gt &&record) + void emplace(TRecord &&record) { const auto newLevel{random_level()}; if (newLevel > m_level) @@ -180,7 +180,7 @@ template class skiplist_t { // Insert new node node_shared_ptr_t newNode = - std::make_shared(std::forward(record), m_level); + std::make_shared(std::forward(record), m_level); for (std::int64_t i{0}; i <= newLevel; i++) { newNode->forward[i] = to_be_updated[i]->forward[i]; @@ -192,7 +192,7 @@ template class skiplist_t else if (current->record.m_key == record.m_key) { // Update value of the existing node - current->record = std::forward(record); + current->record = std::forward(record); } } @@ -216,7 +216,7 @@ template class skiplist_t return level; } - node_shared_ptr_t m_head = std::make_shared(record_gt{}, max_height); + node_shared_ptr_t m_head = std::make_shared(TRecord{}, max_height); std::int64_t m_level{0}; std::size_t m_size{0}; }; diff --git a/src/serialization/common.cpp b/src/serialization/common.cpp index 1ff6681..c2ac801 100644 --- a/src/serialization/common.cpp +++ b/src/serialization/common.cpp @@ -18,4 +18,15 @@ namespace serialization return {reinterpret_cast(span.data()), span.size()}; } +[[nodiscard]] auto varint_size(std::size_t value) noexcept -> std::size_t +{ + std::size_t count{1}; + while (value >= 128) + { + value >>= 7; + count++; + } + return count; +} + } // namespace serialization diff --git a/src/structures/lsmtree/CMakeLists.txt b/src/structures/lsmtree/CMakeLists.txt index 28e3d44..308e262 100644 --- a/src/structures/lsmtree/CMakeLists.txt +++ b/src/structures/lsmtree/CMakeLists.txt @@ -49,6 +49,7 @@ target_link_libraries( DB Concurrency Common + Serialization ) include(GoogleTest) diff --git a/src/structures/lsmtree/lsmtree.cpp b/src/structures/lsmtree/lsmtree.cpp index 85d8369..78bee07 100644 --- a/src/structures/lsmtree/lsmtree.cpp +++ b/src/structures/lsmtree/lsmtree.cpp @@ -215,23 +215,25 @@ auto lsmtree_builder_t::build_memtable_from_wal(wal_t pWal) noexcept -> std::optional { memtable::memtable_t table; - for (const auto &records{pWal->records()}; const auto &record : records) + + // walEntry contains Raft related fields (term + index) and a generic payload, + // which in this case encapsulates DatabaseOperation + for (const auto &walEntries{pWal->records()}; const raft::v1::LogEntry &walEntry : walEntries) { - tinykvpp::v1::DatabaseOperation op; - op.ParseFromString(record.payload()); + tinykvpp::v1::DatabaseOperation dbOperation; + dbOperation.ParseFromString(walEntry.payload()); - switch (op.type()) + switch (dbOperation.type()) { case tinykvpp::v1::DatabaseOperation::TYPE_PUT: { - structures::memtable::memtable_t::record_t item; - item.m_key.m_key = op.key(); - item.m_value.m_value = op.value(); - - table.emplace(std::move(item)); - - spdlog::debug("Recovered record {} from WAL", record.payload()); + table.emplace( + {structures::memtable::memtable_t::record_t::key_t{dbOperation.key()}, + structures::memtable::memtable_t::record_t::value_t{dbOperation.value()}, + structures::memtable::memtable_t::record_t::sequence_number_t{walEntry.index()}} + ); + spdlog::debug("Recovered record {} from WAL", walEntry.payload()); break; } case tinykvpp::v1::DatabaseOperation::TYPE_DELETE: @@ -241,7 +243,7 @@ auto lsmtree_builder_t::build_memtable_from_wal(wal_t pWal) noexcept } default: { - spdlog::error("Unknown WAL operation {}", std::to_underlying(op.type())); + spdlog::error("Unknown WAL operation {}", std::to_underlying(dbOperation.type())); break; } }; diff --git a/src/structures/lsmtree/segments/lsmtree_regular_segment.cpp b/src/structures/lsmtree/segments/lsmtree_regular_segment.cpp index 0af9ddd..f4a74ca 100644 --- a/src/structures/lsmtree/segments/lsmtree_regular_segment.cpp +++ b/src/structures/lsmtree/segments/lsmtree_regular_segment.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -7,10 +8,204 @@ #include #include +#include +#include +#include "serialization/buffer_writer.h" +#include "serialization/common.h" +#include "serialization/endian_integer.h" #include "structures/lsmtree/lsmtree_types.h" +#include "structures/memtable/memtable.h" #include "structures/lsmtree/segments/lsmtree_regular_segment.h" +namespace +{ + +constexpr const std::uint32_t SEGMENT_HEADER_MAGIC_NUMBER{0xDEADBEEF}; +constexpr const std::uint32_t SEGMENT_VERSION_NUMBER{1}; + +[[nodiscard]] auto calculate_header_size_bytes() noexcept -> std::uint32_t +{ + std::uint32_t headerSize{0}; + headerSize += 4; // magic + headerSize += 4; // version + headerSize += 4; // body_offset + headerSize += 4; // index_offset + headerSize += 4; // size + headerSize += 4; // crc32 + return headerSize; +} + +[[nodiscard]] auto +calculate_body_size_bytes(const structures::memtable::memtable_t &memtable) noexcept + -> std::uint32_t +{ + std::uint32_t memtableSize{0}; + for (const auto &record : memtable) + { + memtableSize += serialization::varint_size(record.m_key.m_key.length()); + memtableSize += record.m_key.m_key.length(); + memtableSize += serialization::varint_size(record.m_value.m_value.length()); + memtableSize += record.m_value.m_value.length(); + memtableSize += sizeof(std::uint64_t); // u64 timestamp + } + return memtableSize; +} + +[[nodiscard]] auto +calculate_index_size_bytes(const structures::memtable::memtable_t &memtable) noexcept + -> std::uint32_t +{ + std::uint32_t indexSize{0}; + for (const auto &record : memtable) + { + indexSize += serialization::varint_size(record.m_key.m_key.length()); + indexSize += record.m_key.m_key.length(); + indexSize += sizeof(std::uint32_t); // u32 body_offset + } + return indexSize; +} + +// TODO(lnikon): Use strong types for headerSize, bodySize, bufferSize, as they are consecutive +// u32s and can be mixed up +[[nodiscard]] auto serialize_header( + serialization::buffer_writer_t &writer, + std::uint32_t headerSize, + std::uint32_t bodySize, + std::uint32_t bufferSize +) noexcept -> std::optional +{ + return writer + .write_endian_integer( + serialization::le_uint32_t{SEGMENT_HEADER_MAGIC_NUMBER} + ) // magic_number + .write_endian_integer(serialization::le_uint32_t{SEGMENT_VERSION_NUMBER}) // version + .write_endian_integer(serialization::le_uint32_t{headerSize}) // body_offset + .write_endian_integer(serialization::le_uint32_t{headerSize + bodySize}) // index_ofset + .write_endian_integer(serialization::le_uint32_t{bufferSize}) // size + .error(); +} + +[[nodiscard]] auto serialize_body( + serialization::buffer_writer_t &writer, const structures::memtable::memtable_t &memtable +) noexcept -> std::optional +{ + for (const auto &record : memtable) + { + if (writer.write_varint(record.m_key.m_key.length()) + .write_string(record.m_key.m_key) + .write_varint(record.m_value.m_value.length()) + .write_string(record.m_value.m_value) + .write_endian_integer( + serialization::le_uint64_t{static_cast( + record.m_timestamp.m_value.time_since_epoch().count() + )} + ) + .has_error()) + { + spdlog::debug( + "Failed to serialize memtable record: Error={}, key={}, value={}, " + "sequenceNumber={}, " + "timestamp={}", + magic_enum::enum_name(writer.error().value()), + record.m_key.m_key, + record.m_value.m_value, + record.m_sequenceNumber, + static_cast(record.m_timestamp.m_value.time_since_epoch().count()) + ); + return writer.error(); + } + } + return std::nullopt; +} + +[[nodiscard]] auto serialize_index( + serialization::buffer_writer_t &writer, + const structures::memtable::memtable_t &memtable, + std::uint32_t bodyOffset +) noexcept -> std::optional +{ + std::uint32_t nextIndexEntryOffsetInBody{bodyOffset}; + for (const auto &record : memtable) + { + if (writer.write_varint(record.m_key.m_key.length()) + .write_string(record.m_key.m_key) + .write_endian_integer(serialization::le_uint64_t{nextIndexEntryOffsetInBody}) + .has_error()) + { + spdlog::error( + "Failed to serialize index entry. Error={}, key={}, offset={}", + magic_enum::enum_name(writer.error().value()), + record.m_key.m_key, + nextIndexEntryOffsetInBody + ); + return writer.error(); + } + + nextIndexEntryOffsetInBody += serialization::varint_size(record.m_key.m_key.length()) + + record.m_key.m_key.length() + + serialization::varint_size(record.m_value.m_value.length()) + + record.m_value.m_value.length() + sizeof(std::uint64_t); + } + return std::nullopt; +} + +[[nodiscard]] auto serialize_segment(const structures::memtable::memtable_t &memtable) noexcept + -> std::optional> +{ + // Calculate number of bytes being serialized + const auto headerSize{calculate_header_size_bytes()}; + const auto bodySize{calculate_body_size_bytes(memtable)}; + const auto indexSize{calculate_index_size_bytes(memtable)}; + const auto bufferSize{headerSize + bodySize + indexSize}; + + // Calculate offsets into body and index + const auto bodyOffset{headerSize}; + + // Allocate the buffer + std::vector buffer; + buffer.reserve(bufferSize); + buffer.resize(bufferSize); + + // Serialize into the buffer + serialization::buffer_writer_t writer{buffer}; + ASSERT(writer.bytes_written(), bufferSize); + + if (auto error = serialize_header(writer, headerSize, bodySize, bufferSize)) + { + spdlog::error( + "Failed to serialize segment header. Error={}", + magic_enum::enum_name(writer.error().value()) + ); + return std::nullopt; + } + + // TODO(lnikon): With a change to the binary_writer_t interface can combine these loops into a + // one, need to supply write_pos to the writer with a proper offset for index start, which + // initially should be index_offset=headerSize+memtableSize + if (auto error = serialize_body(writer, memtable); error.has_value()) + { + spdlog::error( + "Failed to serialize segment body. Error={}", + magic_enum::enum_name(writer.error().value()) + ); + return std::nullopt; + } + + if (auto error = serialize_index(writer, memtable, bodyOffset); error.has_value()) + { + spdlog::error( + "Failed to serialize segment index. Error={}", + magic_enum::enum_name(writer.error().value()) + ); + return std::nullopt; + } + + return buffer; +} + +} // namespace + namespace structures::lsmtree::segments::regular_segment { @@ -58,75 +253,48 @@ auto regular_segment_t::record(const hashindex::hashindex_t::offset_t &offset) std::fstream ss{get_path(), std::ios::in | std::fstream::binary}; ss.seekg(offset); - memtable_t::record_t record; - record.read(ss); + // TODO(lnikon): Use binary_buffer_reader_t here + // memtable_t::record_t record; + // record.read(ss); + // return std::make_optional(std::move(record)); - return std::make_optional(std::move(record)); + return std::nullopt; } void regular_segment_t::flush() { - // Skip execution if for some reason the memtable is empty if (!m_memtable.has_value()) { spdlog::warn("Can not flush empty memtable at segment {}", m_path.c_str()); return; } - if (m_memtable->empty()) + const auto &memtable{m_memtable.value()}; + if (memtable.empty()) { spdlog::warn("Can not flush memtable of size 0 at segment {}", m_path.c_str()); return; } - // Serialize memtable into stringstream and build hash index - std::stringstream stringStream; - std::size_t cursor{0}; - const auto &memtable = m_memtable.value(); - for (std::size_t recordIndex{0}; const auto &record : memtable) + auto bufferOpt{serialize_segment(memtable)}; + if (!bufferOpt.has_value()) { - std::size_t ss_before = stringStream.tellp(); - record.write(stringStream); - if (recordIndex++ != m_memtable.value().count()) - { - stringStream << '\n'; - } - const auto length{static_cast(stringStream.tellp()) - ss_before}; - m_hashIndex.emplace(record, cursor); - cursor += length; - } - assert(!m_hashIndex.empty()); - - // Serialize hashindex - const auto hashIndexBlockOffset{stringStream.tellp()}; - for (const auto &[key, offset] : m_hashIndex) - { - stringStream << key.m_key << ' ' << offset; // << '\n'; + spdlog::error("Failed to serialize segment"); + return; } - // Calculate size of the index block - const auto hashIndexBlockSize{stringStream.tellp() - hashIndexBlockOffset}; - - // Get offset into footer section - const auto footerBlockOffset{stringStream.tellp()}; - - // Serialize footer - stringStream << hashIndexBlockOffset << ' ' << hashIndexBlockSize; // << '\n'; - - // Add padding to the footer end - const auto footerPaddingSize{footerSize - (stringStream.tellp() - footerBlockOffset)}; - stringStream << std::string(footerPaddingSize, ' '); // << '\n'; - - // Flush the segment into the disk - std::fstream stream{get_path(), std::fstream::binary | std::fstream::trunc | std::fstream::out}; - if (!stream.is_open()) + std::ofstream binaryStream(get_path(), std::ios::binary); + if (!binaryStream.is_open()) { - throw std::runtime_error("unable to flush segment for path " + m_path.string()); + spdlog::error( + "Unable to open binary stream to write segment. Path={}", get_path().string() + ); + return; } - assert(!stringStream.str().empty()); - stream << stringStream.str(); - stream.flush(); + binaryStream.write( + reinterpret_cast(bufferOpt.value().data()), bufferOpt.value().size() + ); } void regular_segment_t::remove_from_disk() const noexcept @@ -248,7 +416,9 @@ void regular_segment_t::restore_index() // bytesRead += end - start; bytesRead += line.size() + 1; - m_hashIndex.emplace(structures::lsmtree::record_t{key_t{key}, value_t{}}, offset); + m_hashIndex.emplace( + structures::lsmtree::record_t{key_t{key}, value_t{}, sequence_number_t{}}, offset + ); } if (m_hashIndex.empty()) diff --git a/src/structures/lsmtree/segments/segment_serializer.cpp b/src/structures/lsmtree/segments/segment_serializer.cpp new file mode 100644 index 0000000..e69de29 diff --git a/src/structures/memtable/CMakeLists.txt b/src/structures/memtable/CMakeLists.txt index 88892d3..9c4fe1f 100644 --- a/src/structures/memtable/CMakeLists.txt +++ b/src/structures/memtable/CMakeLists.txt @@ -16,7 +16,6 @@ target_link_libraries( gtest::gtest spdlog::spdlog fmt::fmt - LSMTree MemTable ) diff --git a/src/structures/memtable/memtable.cpp b/src/structures/memtable/memtable.cpp index 03c9cf9..47bc50a 100644 --- a/src/structures/memtable/memtable.cpp +++ b/src/structures/memtable/memtable.cpp @@ -55,49 +55,33 @@ auto memtable_t::record_t::value_t::operator==(const memtable_t::record_t::value return m_value == other.m_value; } -// ---------------------------------------------------- -// memtable_t::record_t::timestamp_t -// ---------------------------------------------------- -memtable_t::record_t::timestamp_t::timestamp_t() - : m_value{clock_t::now()} -{ -} - -auto memtable_t::record_t::timestamp_t::operator<(const timestamp_t &other) const -> bool -{ - return m_value < other.m_value; -} - -void memtable_t::record_t::timestamp_t::swap(timestamp_t &lhs, timestamp_t &rhs) -{ - using std::swap; - swap(lhs.m_value, rhs.m_value); -} - // --------------------------------------- // memtable_t::record_t // --------------------------------------- -memtable_t::record_t::record_t(memtable_t::record_t::key_t key, memtable_t::record_t::value_t value) +memtable_t::record_t::record_t(key_t key, value_t value, sequence_number_t sequenceNumber) : m_key(std::move(key)), - m_value(std::move(value)) + m_value(std::move(value)), + m_sequenceNumber(sequenceNumber) { } auto memtable_t::record_t::operator<(const memtable_t::record_t &record) const -> bool { - // return m_key < record.m_key && m_timestamp < record.m_timestamp; - return (m_key != record.m_key) ? m_key < record.m_key : !(m_timestamp < record.m_timestamp); + if (m_key != record.m_key) + { + return m_key < record.m_key; + } + return m_sequenceNumber < record.m_sequenceNumber; } auto memtable_t::record_t::operator>(const memtable_t::record_t &record) const -> bool { - return !(m_key < record.m_key); + return record < *this; } auto memtable_t::record_t::operator==(const record_t &record) const -> bool { - spdlog::info("memtable::==: this.{} record.{}", m_key.m_key, record.m_key.m_key); - return m_key == record.m_key; + return m_key == record.m_key && m_sequenceNumber == record.m_sequenceNumber; } auto memtable_t::record_t::size() const -> std::size_t diff --git a/src/structures/memtable/memtable_test.cpp b/src/structures/memtable/memtable_test.cpp index 6f8ad6d..1667b5e 100644 --- a/src/structures/memtable/memtable_test.cpp +++ b/src/structures/memtable/memtable_test.cpp @@ -8,14 +8,15 @@ using structures::memtable::memtable_t; using record_t = memtable::memtable_t::record_t; using record_key_t = structures::memtable::memtable_t::record_t::key_t; using record_value_t = structures::memtable::memtable_t::record_t::value_t; +using record_sequence_number_t = structures::memtable::memtable_t::record_t::sequence_number_t; TEST(MemTableTest, EmplaceAndFind) { memtable_t mt; - mt.emplace(record_t{record_key_t{"B"}, record_value_t{"123"}}); - mt.emplace(record_t{record_key_t{"A"}, record_value_t{"-12"}}); - mt.emplace(record_t{record_key_t{"Z"}, record_value_t{"34.44"}}); - mt.emplace(record_t{record_key_t{"C"}, record_value_t{"Hello"}}); + mt.emplace(record_t{record_key_t{"B"}, record_value_t{"123"}, record_sequence_number_t{0}}); + mt.emplace(record_t{record_key_t{"A"}, record_value_t{"-12"}, record_sequence_number_t{0}}); + mt.emplace(record_t{record_key_t{"Z"}, record_value_t{"34.44"}, record_sequence_number_t{0}}); + mt.emplace(record_t{record_key_t{"C"}, record_value_t{"Hello"}, record_sequence_number_t{0}}); auto record = mt.find(record_key_t{"C"}); EXPECT_EQ(record->m_key, record_key_t{"C"}); @@ -32,7 +33,7 @@ TEST(MemTableTest, CheckRecordSize) record_key_t k{"B"}; record_value_t v{"123"}; - mt.emplace(record_t{k, v}); + mt.emplace(record_t{k, v, 0}); auto record = mt.find(record_key_t{"B"}); EXPECT_NE(record, std::nullopt); @@ -47,7 +48,7 @@ TEST(MemTableTest, CheckRecordSize) record_key_t k{"B"}; record_value_t v{"123"}; - mt.emplace(record_t{k, v}); + mt.emplace(record_t{k, v, 0}); auto record = mt.find(record_key_t{"B"}); EXPECT_NE(record, std::nullopt); @@ -62,7 +63,7 @@ TEST(MemTableTest, CheckRecordSize) record_key_t k{"B"}; record_value_t v{"123.456"}; - auto record = record_t{k, v}; + auto record = record_t{k, v, 0}; mt.emplace(record); auto recordOpt = mt.find(k); @@ -80,9 +81,9 @@ TEST(MemTableTest, CheckSize) memtable_t mt; auto k1 = record_key_t{"B"}, k2 = record_key_t{"A"}, k3 = record_key_t{"Z"}; auto v1 = record_value_t{"123"}, v2 = record_value_t{"34.44"}, v3 = record_value_t{"Hello"}; - mt.emplace(record_t{k1, v1}); - mt.emplace(record_t{k2, v2}); - mt.emplace(record_t{k3, v3}); + mt.emplace(record_t{k1, v1, 0}); + mt.emplace(record_t{k2, v2, 0}); + mt.emplace(record_t{k3, v3, 0}); // TODO: Not sure if this a good/correct way to check MemTable::Size() :) EXPECT_EQ(mt.size(), k1.size() + v1.size() + k2.size() + v2.size() + k3.size() + v3.size()); @@ -91,10 +92,10 @@ TEST(MemTableTest, CheckSize) TEST(MemTableTest, CheckCount) { memtable_t mt; - mt.emplace(record_t{record_key_t{"B"}, record_value_t{"123"}}); - mt.emplace(record_t{record_key_t{"A"}, record_value_t{"-12"}}); - mt.emplace(record_t{record_key_t{"Z"}, record_value_t{"z`34.44"}}); - mt.emplace(record_t{record_key_t{"C"}, record_value_t{"Hello"}}); + mt.emplace(record_t{record_key_t{"B"}, record_value_t{"123"}, record_sequence_number_t{0}}); + mt.emplace(record_t{record_key_t{"A"}, record_value_t{"-12"}, record_sequence_number_t{0}}); + mt.emplace(record_t{record_key_t{"Z"}, record_value_t{"z`34.44"}, record_sequence_number_t{0}}); + mt.emplace(record_t{record_key_t{"C"}, record_value_t{"Hello"}, record_sequence_number_t{0}}); EXPECT_EQ(mt.count(), 4); }