feat(coredns): use txt-owner-id to strictly separated external-dns instances (#5921)

* feat(coredns): use managed-by to separate records

Signed-off-by: Jan Jansen <jan.jansen@gdata.de>

* feat(coredns): use txt-owner-id to strictly separated external-dns instances

Signed-off-by: Jan Jansen <jan.jansen@gdata.de>

* fix tests

Signed-off-by: Jan Jansen <jan.jansen@gdata.de>

* fix reviewer comments

Signed-off-by: Jan Jansen <jan.jansen@gdata.de>

* answer review comments

* fix deletion behavior and remove extra function

* fix markdown

* fix tests again

---------

Signed-off-by: Jan Jansen <jan.jansen@gdata.de>

Author: farodin91

controller/execute.go

@@ -248,7 +248,7 @@ func buildProvider(
 	case "dnsimple":
 		p, err = dnsimple.NewDnsimpleProvider(domainFilter, zoneIDFilter, cfg.DryRun)
 	case "coredns", "skydns":
-		p, err = coredns.NewCoreDNSProvider(domainFilter, cfg.CoreDNSPrefix, cfg.DryRun)
+		p, err = coredns.NewCoreDNSProvider(domainFilter, cfg.CoreDNSPrefix, cfg.TXTOwnerID, cfg.CoreDNSStrictlyOwned, cfg.DryRun)
 	case "exoscale":
 		p, err = exoscale.NewExoscaleProvider(
 			cfg.ExoscaleAPIEnvironment,

docs/flags.md

@@ -97,6 +97,7 @@
 | `--cloudflare-region-key=""` | When using the Cloudflare provider, specify the default region for Regional Services. Any value other than an empty string will enable the Regional Services feature (optional) |
 | `--cloudflare-record-comment=""` | When using the Cloudflare provider, specify the comment for the DNS records (default: '') |
 | `--coredns-prefix="/skydns/"` | When using the CoreDNS provider, specify the prefix name |
+| `--[no-]coredns-strictly-owned` | When using the CoreDNS provider, store and filter strictly by txt-owner-id using an extra field inside of the etcd service (default: false) |
 | `--akamai-serviceconsumerdomain=""` | When using the Akamai provider, specify the base URL (required when --provider=akamai and edgerc-path not specified) |
 | `--akamai-client-token=""` | When using the Akamai provider, specify the client token (required when --provider=akamai and edgerc-path not specified) |
 | `--akamai-client-secret=""` | When using the Akamai provider, specify the client secret (required when --provider=akamai and edgerc-path not specified) |

docs/tutorials/coredns.md

@@ -2,6 +2,47 @@
 
 - [Documentation](https://coredns.io/)
 
+## Multi cluster support options
+
+The CoreDNS provider allows records from different CoreDNS providers to be separated in a single etcd
+by activating the setting `--coredns-strictly-owned` flag and set `txt-owner-id`. It will prevent any
+override (update/create/delete) of records by a different owner and prevent loading of records by a
+different owner.
+
+Flow:
+
+```mermaid
+graph TD
+  subgraph ETCD
+    store--> E(services from Cluster A)
+    store--> F(services from Cluster B)
+    store--> G(services from someone else)
+  end
+  subgraph Cluster A
+  A(external-dns with stictly-owned)
+  end
+  A --> E
+  subgraph Cluster B
+  B(external-dns with stictly-owned)
+  end
+  B --> F
+  store --> CoreDNS
+```
+
+This features works directly without any change to CoreDNS. CoreDNS will ignore this field inside the etcd record.
+
+### Other entries inside etcd
+
+Service entries in etcd without an `ownedby` field will be filtered out by the provider if `strictly-owned` is activated.
+Warning: If you activate `strictly-owned` afterwards, these entries will be ignored as the `ownedby` field is empty.
+
+### Ways to migrate to a multi cluster setup
+
+Ways:
+
+1. Add the correct owner to all services inside etcd by adding the field `ownedby` to the JSON.
+2. Remove all services and allow them to be required again after restarting the provider. (Possible downtime.)
+
 ## Specific service annotation options
 
 ### Groups

endpoint/endpoint_test.go

@@ -583,7 +583,7 @@ func TestIsOwnedBy(t *testing.T) {
 				Labels: tt.fields.Labels,
 			}
 			if got := e.IsOwnedBy(tt.args.ownerID); got != tt.want {
-				t.Errorf("Endpoint.IsOwnedBy() = %v, want %v", got, tt.want)
+				t.Errorf("Endpoint.isOwnedBy() = %v, want %v", got, tt.want)
 			}
 		})
 	}

pkg/apis/externaldns/types.go

@@ -119,6 +119,7 @@ type Config struct {
 	CloudflareRegionalServices                    bool
 	CloudflareRegionKey                           string
 	CoreDNSPrefix                                 string
+	CoreDNSStrictlyOwned                          bool
 	AkamaiServiceConsumerDomain                   string
 	AkamaiClientToken                             string
 	AkamaiClientSecret                            string
@@ -269,6 +270,7 @@ var defaultConfig = &Config{
 	Compatibility:                "",
 	ConnectorSourceServer:        "localhost:8080",
 	CoreDNSPrefix:                "/skydns/",
+	CoreDNSStrictlyOwned:         false,
 	CRDSourceAPIVersion:          "externaldns.k8s.io/v1alpha1",
 	CRDSourceKind:                "DNSEndpoint",
 	DefaultTargets:               []string{},
@@ -702,6 +704,7 @@ func bindFlags(b FlagBinder, cfg *Config) {
 	b.StringVar("cloudflare-record-comment", "When using the Cloudflare provider, specify the comment for the DNS records (default: '')", "", &cfg.CloudflareDNSRecordsComment)
 
 	b.StringVar("coredns-prefix", "When using the CoreDNS provider, specify the prefix name", defaultConfig.CoreDNSPrefix, &cfg.CoreDNSPrefix)
+	b.BoolVar("coredns-strictly-owned", "When using the CoreDNS provider, store and filter strictly by txt-owner-id using an extra field inside of the etcd service (default: false)", defaultConfig.CoreDNSStrictlyOwned, &cfg.CoreDNSStrictlyOwned)
 	b.StringVar("akamai-serviceconsumerdomain", "When using the Akamai provider, specify the base URL (required when --provider=akamai and edgerc-path not specified)", defaultConfig.AkamaiServiceConsumerDomain, &cfg.AkamaiServiceConsumerDomain)
 	b.StringVar("akamai-client-token", "When using the Akamai provider, specify the client token (required when --provider=akamai and edgerc-path not specified)", defaultConfig.AkamaiClientToken, &cfg.AkamaiClientToken)
 	b.StringVar("akamai-client-secret", "When using the Akamai provider, specify the client secret (required when --provider=akamai and edgerc-path not specified)", defaultConfig.AkamaiClientSecret, &cfg.AkamaiClientSecret)

provider/coredns/coredns.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
+	"go.etcd.io/etcd/api/v3/mvccpb"
 	etcdcv3 "go.etcd.io/etcd/client/v3"
 
 	"sigs.k8s.io/external-dns/pkg/tlsutils"
@@ -84,10 +85,15 @@ type Service struct {
 
 	// Etcd key where we found this service and ignored from json un-/marshaling
 	Key string `json:"-"`
+
+	// OwnedBy is used to prevent service to be added by different external-dns (only used by external-dns)
+	OwnedBy string `json:"ownedby,omitempty"`
 }
 
 type etcdClient struct {
-	client *etcdcv3.Client
+	client        *etcdcv3.Client
+	ownerID       string
+	strictlyOwned bool
 }
 
 var _ coreDNSClient = etcdClient{}
@@ -106,11 +112,21 @@ func (c etcdClient) GetServices(ctx context.Context, prefix string) ([]*Service,
 	var svcs []*Service
 	bx := make(map[Service]bool)
 	for _, n := range r.Kvs {
-		svc := new(Service)
-		if err := json.Unmarshal(n.Value, svc); err != nil {
-			return nil, fmt.Errorf("%s: %w", n.Key, err)
+		svc, err := c.unmarshalService(n)
+		if err != nil {
+			return nil, err
+		}
+		if c.strictlyOwned && svc.OwnedBy != c.ownerID {
+			continue
+		}
+		b := Service{
+			Host:     svc.Host,
+			Port:     svc.Port,
+			Priority: svc.Priority,
+			Weight:   svc.Weight,
+			Text:     svc.Text,
+			Key:      string(n.Key),
 		}
-		b := Service{Host: svc.Host, Port: svc.Port, Priority: svc.Priority, Weight: svc.Weight, Text: svc.Text, Key: string(n.Key)}
 		if _, ok := bx[b]; ok {
 			// skip the service if already added to service list.
 			// the same service might be found in multiple etcd nodes.
@@ -132,6 +148,25 @@ func (c etcdClient) SaveService(ctx context.Context, service *Service) error {
 	ctx, cancel := context.WithTimeout(ctx, etcdTimeout)
 	defer cancel()
 
+	// check only for empty OwnedBy
+	if c.strictlyOwned && service.OwnedBy != c.ownerID {
+		r, err := c.client.Get(ctx, service.Key)
+		if err != nil {
+			return fmt.Errorf("etcd get %q: %w", service.Key, err)
+		}
+		// Key missing -> treat as owned (safe to create)
+		if r != nil && len(r.Kvs) != 0 {
+			svc, err := c.unmarshalService(r.Kvs[0])
+			if err != nil {
+				return fmt.Errorf("failed to unmarshal value for key %q: %w", service.Key, err)
+			}
+			if svc.OwnedBy != c.ownerID {
+				return fmt.Errorf("key %q is not owned by this provider", service.Key)
+			}
+		}
+		service.OwnedBy = c.ownerID
+	}
+
 	value, err := json.Marshal(&service)
 	if err != nil {
 		return err
@@ -148,8 +183,38 @@ func (c etcdClient) DeleteService(ctx context.Context, key string) error {
 	ctx, cancel := context.WithTimeout(ctx, etcdTimeout)
 	defer cancel()
 
-	_, err := c.client.Delete(ctx, key, etcdcv3.WithPrefix())
-	return err
+	if c.strictlyOwned {
+		rs, err := c.client.Get(ctx, key, etcdcv3.WithPrefix())
+		if err != nil {
+			return err
+		}
+		for _, r := range rs.Kvs {
+			svc, err := c.unmarshalService(r)
+			if err != nil {
+				return err
+			}
+			if svc.OwnedBy != c.ownerID {
+				continue
+			}
+
+			_, err = c.client.Delete(ctx, string(r.Key))
+			if err != nil {
+				return err
+			}
+		}
+		return err
+	} else {
+		_, err := c.client.Delete(ctx, key, etcdcv3.WithPrefix())
+		return err
+	}
+}
+
+func (c etcdClient) unmarshalService(n *mvccpb.KeyValue) (*Service, error) {
+	svc := new(Service)
+	if err := json.Unmarshal(n.Value, svc); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal %q: %w", n.Key, err)
+	}
+	return svc, nil
 }
 
 // builds etcd client config depending on connection scheme and TLS parameters
@@ -183,7 +248,7 @@ func getETCDConfig() (*etcdcv3.Config, error) {
 }
 
 // the newETCDClient is an etcd client constructor
-func newETCDClient() (coreDNSClient, error) {
+func newETCDClient(ownerID string, strictlyOwned bool) (coreDNSClient, error) {
 	cfg, err := getETCDConfig()
 	if err != nil {
 		return nil, err
@@ -192,12 +257,12 @@ func newETCDClient() (coreDNSClient, error) {
 	if err != nil {
 		return nil, err
 	}
-	return etcdClient{c}, nil
+	return etcdClient{c, ownerID, strictlyOwned}, nil
 }
 
 // NewCoreDNSProvider is a CoreDNS provider constructor
-func NewCoreDNSProvider(domainFilter *endpoint.DomainFilter, prefix string, dryRun bool) (provider.Provider, error) {
-	client, err := newETCDClient()
+func NewCoreDNSProvider(domainFilter *endpoint.DomainFilter, prefix, ownerID string, strictlyOwned, dryRun bool) (provider.Provider, error) {
+	client, err := newETCDClient(ownerID, strictlyOwned)
 	if err != nil {
 		return nil, err
 	}