feature: Adding apipa nic support for swiftv2 windows

cni/netconfig.go

@@ -72,6 +72,7 @@ type NetworkConfig struct {
 	EnableExactMatchForPodName    bool            `json:"enableExactMatchForPodName,omitempty"`
 	DisableHairpinOnHostInterface bool            `json:"disableHairpinOnHostInterface,omitempty"`
 	DisableIPTableLock            bool            `json:"disableIPTableLock,omitempty"`
+	DisableAsyncDelete            bool            `json:"disableAsyncDelete,omitempty"`
 	CNSUrl                        string          `json:"cnsurl,omitempty"`
 	ExecutionMode                 string          `json:"executionMode,omitempty"`
 	IPAM                          IPAM            `json:"ipam,omitempty"`

cni/network/invoker_cns.go

@@ -165,7 +165,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro
 		}
 
 		logger.Info("Received info for pod",
-			zap.Any("ipInfo", info),
+			zap.Any("ipInfo", response.PodIPInfo[i]),
 			zap.Any("podInfo", podInfo))
 
 		//nolint:exhaustive // ignore exhaustive types check
@@ -192,6 +192,11 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro
 			if err := addBackendNICToResult(&info, &addResult, key); err != nil {
 				return IPAMAddResult{}, err
 			}
+		case cns.ApipaNIC:
+			if err := configureApipaAddResult(&addResult, &response.PodIPInfo[i], key); err != nil {
+				return IPAMAddResult{}, err
+			}
+
 		case cns.InfraNIC, "":
 			// if we change from legacy cns, the nicType will be empty, so we assume it is infra nic
 			info.nicType = cns.InfraNIC
@@ -508,6 +513,32 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p
 	return nil
 }
 
+func configureApipaAddResult(addResult *IPAMAddResult, info *cns.PodIpInfo, key string) error {
+	ip, ipnet, err := info.PodIPConfig.GetIPNet()
+	if ip == nil {
+		return errors.Wrap(err, "GetIPNet failed while configuring apipa AddResult")
+	}
+
+	addResult.interfaceInfo[key] = network.InterfaceInfo{
+		IPConfigs: []*network.IPConfig{
+			{
+				Address: net.IPNet{
+					IP:   ip,
+					Mask: ipnet.Mask,
+				},
+				Gateway: net.ParseIP(info.NetworkContainerPrimaryIPConfig.GatewayIPAddress),
+			},
+		},
+		NICType:                    info.NICType,
+		SkipDefaultRoutes:          true,
+		NetworkContainerID:         info.NetworkContainerID,
+		AllowHostToNCCommunication: info.AllowHostToNCCommunication,
+		AllowNCToHostCommunication: info.AllowNCToHostCommunication,
+	}
+
+	return nil
+}
+
 func addBackendNICToResult(info *IPResultInfo, addResult *IPAMAddResult, key string) error {
 	macAddress, err := net.ParseMAC(info.macAddress)
 	if err != nil {

cni/network/network.go

@@ -52,6 +52,7 @@ const (
 	ipv4FullMask          = 32
 	ipv6FullMask          = 128
 	ibInterfacePrefix     = "ib"
+	apipaInterfacePrefix  = "apipa"
 )
 
 // CNI Operation Types
@@ -643,6 +644,8 @@ func (plugin *NetPlugin) findMasterInterface(opt *createEpInfoOpt) string {
 		// when the VF is dismounted, this interface will go away
 		// return an unique interface name to containerd
 		return ibInterfacePrefix + strconv.Itoa(opt.endpointIndex)
+	case cns.ApipaNIC:
+		return apipaInterfacePrefix + strconv.Itoa(opt.endpointIndex)
 	default:
 		return ""
 	}
@@ -757,8 +760,11 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn
 		IPAddresses: addresses,
 		MacAddress:  opt.ifInfo.MacAddress,
 		// the following is used for creating an external interface if we can't find an existing network
-		HostSubnetPrefix: opt.ifInfo.HostSubnetPrefix.String(),
-		PnPID:            opt.ifInfo.PnPID,
+		HostSubnetPrefix:         opt.ifInfo.HostSubnetPrefix.String(),
+		PnPID:                    opt.ifInfo.PnPID,
+		NetworkContainerID:       opt.ifInfo.NetworkContainerID,
+		AllowInboundFromHostToNC: opt.ifInfo.AllowHostToNCCommunication,
+		AllowInboundFromNCToHost: opt.ifInfo.AllowNCToHostCommunication,
 	}
 
 	if err = addSubnetToEndpointInfo(*opt.ifInfo, &endpointInfo); err != nil {
@@ -1072,7 +1078,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error {
 		epInfos, err = plugin.nm.GetEndpointState(networkID, args.ContainerID)
 		// if stateless CNI fail to get the endpoint from CNS for any reason other than  Endpoint Not found
 		if err != nil {
-			if errors.Is(err, network.ErrConnectionFailure) {
+			if errors.Is(err, network.ErrConnectionFailure) && !nwCfg.DisableAsyncDelete {
 				logger.Info("failed to connect to CNS", zap.String("containerID", args.ContainerID), zap.Error(err))
 				addErr := fsnotify.AddFile(args.ContainerID, args.ContainerID, watcherPath)
 				logger.Info("add containerid file for Asynch delete", zap.String("containerID", args.ContainerID), zap.Error(addErr))
@@ -1152,10 +1158,10 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error {
 			}
 		}
 	}
-	logger.Info("Deleting the state from the cni statefile")
+	logger.Info("Deleting endpoint state from statefile")
 	err = plugin.nm.DeleteState(epInfos)
 	if err != nil {
-		return plugin.RetriableError(fmt.Errorf("failed to save state: %w", err))
+		return plugin.RetriableError(fmt.Errorf("failed to delete state: %w", err))
 	}
 
 	return err

cni/network/network_windows.go

@@ -75,10 +75,13 @@ func (plugin *NetPlugin) getNetworkName(netNs string, interfaceInfo *network.Int
 	// Swiftv2 L1VH Network Name
 	swiftv2NetworkNamePrefix := "azure-"
 	if interfaceInfo != nil && (interfaceInfo.NICType == cns.NodeNetworkInterfaceFrontendNIC || interfaceInfo.NICType == cns.BackendNIC) {
-		logger.Info("swiftv2", zap.String("network name", interfaceInfo.MacAddress.String()))
 		return swiftv2NetworkNamePrefix + interfaceInfo.MacAddress.String(), nil
 	}
 
+	if interfaceInfo != nil && interfaceInfo.NICType == cns.ApipaNIC {
+		return swiftv2NetworkNamePrefix + apipaInterfacePrefix, nil
+	}
+
 	// For singletenancy, the network name is simply the nwCfg.Name
 	if !nwCfg.MultiTenancy {
 		return nwCfg.Name, nil

cns/Dockerfile

@@ -38,4 +38,4 @@ FROM --platform=windows/${ARCH} mcr.microsoft.com/oss/kubernetes/windows-host-pr
 FROM hpc as windows
 COPY --from=builder /go/bin/azure-cns /azure-cns.exe
 ENTRYPOINT ["azure-cns.exe"]
-EXPOSE 10090
+EXPOSE 10090

cns/NetworkContainerContract.go

@@ -93,6 +93,9 @@ const (
 	NodeNetworkInterfaceFrontendNIC NICType = "FrontendNIC"
 	// NodeNetworkInterfaceBackendNIC is the new name for BackendNIC
 	NodeNetworkInterfaceBackendNIC NICType = "BackendNIC"
+
+	// ApipaNIC is used for internal communication between host and container
+	ApipaNIC NICType = "ApipaNIC"
 )
 
 // ChannelMode :- CNS channel modes
@@ -516,6 +519,12 @@ type PodIpInfo struct {
 	PnPID string
 	// Default Deny ACL's to configure on HNS endpoints for Swiftv2 window nodes
 	EndpointPolicies []policy.Policy
+	// This flag is in effect only if nic type is apipa. This allows connection originating from host to container via apipa nic and not other way.
+	AllowHostToNCCommunication bool
+	// This flag is in effect only if nic type is apipa. This allows connection originating from container to host via apipa nic and not other way.
+	AllowNCToHostCommunication bool
+	// NetworkContainerID is the ID of the network container to which this Pod IP belongs
+	NetworkContainerID string
 }
 
 type HostIPInfo struct {

cns/client/client.go

@@ -1100,3 +1100,35 @@ func (c *Client) UpdateEndpoint(ctx context.Context, endpointID string, ipInfo m
 
 	return &response, nil
 }
+
+func (c *Client) DeleteEndpointState(ctx context.Context, endpointID string) (*cns.Response, error) {
+	// build the request
+	u := c.routes[cns.EndpointAPI]
+	uString := u.String() + endpointID
+	req, err := http.NewRequestWithContext(ctx, http.MethodDelete, uString, http.NoBody)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build request")
+	}
+	req.Header.Set(headerContentType, contentTypeJSON)
+	res, err := c.client.Do(req)
+	if err != nil {
+		return nil, &ConnectionFailureErr{cause: err}
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("http response %d", res.StatusCode)
+	}
+
+	var response cns.Response
+	err = json.NewDecoder(res.Body).Decode(&response)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to decode CNS Response")
+	}
+
+	if response.ReturnCode != 0 {
+		return nil, errors.New(response.Message)
+	}
+
+	return &response, nil
+}

cns/endpointmanager/endpointmanager_windows.go

@@ -16,6 +16,7 @@ func (em *EndpointManager) ReleaseIPs(ctx context.Context, ipconfigreq cns.IPCon
 	if err := em.deleteEndpoint(ctx, ipconfigreq.InfraContainerID); err != nil {
 		logger.Errorf("failed to remove HNS endpoint %s", err.Error())
 	}
+
 	return errors.Wrap(em.cli.ReleaseIPs(ctx, ipconfigreq), "failed to release IP from CNS")
 }
 

cns/restserver/ipam.go

@@ -149,14 +149,33 @@ func (service *HTTPRestService) requestIPConfigHandlerHelperStandalone(ctx conte
 
 	// assign NICType and MAC Address for SwiftV2. we assume that there won't be any SwiftV1 NCs here
 	podIPInfoList := make([]cns.PodIpInfo, 0, len(resp))
+	apipaIndex := -1
 	for i := range resp {
 		podIPInfo := cns.PodIpInfo{
 			PodIPConfig:                     resp[i].IPConfiguration.IPSubnet,
 			MacAddress:                      resp[i].NetworkInterfaceInfo.MACAddress,
 			NICType:                         resp[i].NetworkInterfaceInfo.NICType,
 			NetworkContainerPrimaryIPConfig: resp[i].IPConfiguration,
+			NetworkContainerID:              resp[i].NetworkContainerID,
 		}
 		podIPInfoList = append(podIPInfoList, podIPInfo)
+		if resp[i].AllowHostToNCCommunication || resp[i].AllowNCToHostCommunication {
+			apipaIndex = i
+			break
+		}
+	}
+
+	if apipaIndex != -1 {
+		apipaPodIPInfo := cns.PodIpInfo{
+			PodIPConfig:                     resp[apipaIndex].LocalIPConfiguration.IPSubnet,
+			NICType:                         cns.ApipaNIC,
+			NetworkContainerPrimaryIPConfig: resp[apipaIndex].LocalIPConfiguration,
+			SkipDefaultRoutes:               true,
+			AllowHostToNCCommunication:      resp[apipaIndex].AllowHostToNCCommunication,
+			AllowNCToHostCommunication:      resp[apipaIndex].AllowNCToHostCommunication,
+			NetworkContainerID:              resp[apipaIndex].NetworkContainerID,
+		}
+		podIPInfoList = append(podIPInfoList, apipaPodIPInfo)
 	}
 
 	ipConfigsResp := &cns.IPConfigsResponse{
@@ -1123,11 +1142,76 @@ func (service *HTTPRestService) EndpointHandlerAPI(w http.ResponseWriter, r *htt
 		service.GetEndpointHandler(w, r)
 	case http.MethodPatch:
 		service.UpdateEndpointHandler(w, r)
+	case http.MethodDelete:
+		service.DeleteEndpointStateHandler(w, r)
 	default:
 		logger.Errorf("[EndpointHandlerAPI] EndpointHandler API expect http Get or Patch method")
 	}
 }
 
+func (service *HTTPRestService) DeleteEndpointStateHandler(w http.ResponseWriter, r *http.Request) {
+	opName := "DeleteEndpointStateHandler"
+	logger.Printf("[DeleteEndpointStateHandler] DeleteEndpointState for %s", r.URL.Path) //nolint:staticcheck // reason: using deprecated call until migration to new API
+	endpointID := strings.TrimPrefix(r.URL.Path, cns.EndpointPath)
+
+	if service.EndpointStateStore == nil {
+		response := cns.Response{
+			ReturnCode: types.NilEndpointStateStore,
+			Message:    "[DeleteEndpointStateHandler] EndpointStateStore is not initialized",
+		}
+		err := common.Encode(w, &response)
+		logger.Response(opName, response, response.ReturnCode, err) //nolint:staticcheck // reason: using deprecated call until migration to new API
+		return
+	}
+
+	// Delete the endpoint from state
+	err := service.DeleteEndpointStateHelper(endpointID)
+	if err != nil {
+		response := cns.Response{
+			ReturnCode: types.UnexpectedError,
+			Message:    fmt.Sprintf("[DeleteEndpointStateHandler] Failed to delete endpoint state for %s with error: %s", endpointID, err.Error()),
+		}
+
+		if errors.Is(err, ErrEndpointStateNotFound) {
+			response.ReturnCode = types.NotFound
+		}
+
+		err = common.Encode(w, &response)
+		logger.Response(opName, response, response.ReturnCode, err) //nolint:staticcheck // reason: using deprecated call until migration to new API
+		return
+	}
+
+	response := cns.Response{
+		ReturnCode: types.Success,
+		Message:    "[DeleteEndpointStateHandler] Endpoint state deleted successfully",
+	}
+	err = common.Encode(w, &response)
+	logger.Response(opName, response, response.ReturnCode, err) //nolint:staticcheck // reason: using deprecated call until migration to new API
+}
+
+func (service *HTTPRestService) DeleteEndpointStateHelper(endpointID string) error {
+	if service.EndpointStateStore == nil {
+		return ErrStoreEmpty
+	}
+	logger.Printf("[deleteEndpointState] Deleting Endpoint state from state file %s", endpointID) //nolint:staticcheck // reason: using deprecated call until migration to new API
+	_, endpointExist := service.EndpointState[endpointID]
+	if !endpointExist {
+		logger.Printf("[deleteEndpointState] endpoint could not be found in the statefile %s", endpointID) //nolint:staticcheck // reason: using deprecated call until migration to new API
+		return fmt.Errorf("[deleteEndpointState] endpoint %s: %w", endpointID, ErrEndpointStateNotFound)
+	}
+
+	// Delete the endpoint from the state
+	delete(service.EndpointState, endpointID)
+
+	// Write the updated state back to the store
+	err := service.EndpointStateStore.Write(EndpointStoreKey, service.EndpointState)
+	if err != nil {
+		return fmt.Errorf("[deleteEndpointState] failed to write endpoint state to store: %w", err)
+	}
+	logger.Printf("[deleteEndpointState] successfully deleted endpoint %s from state file", endpointID) //nolint:staticcheck // reason: using deprecated call until migration to new API
+	return nil
+}
+
 // GetEndpointHandler handles the incoming GetEndpoint requests with http Get method
 func (service *HTTPRestService) GetEndpointHandler(w http.ResponseWriter, r *http.Request) {
 	opName := "getEndpointState"
@@ -1313,6 +1397,11 @@ func updateIPInfoMap(iPInfo map[string]*IPInfo, interfaceInfo *IPInfo, ifName, e
 		iPInfo[ifName].MacAddress = interfaceInfo.MacAddress
 		logger.Printf("[updateEndpoint] update the endpoint %s with MacAddress  %s", endpointID, interfaceInfo.MacAddress)
 	}
+
+	if interfaceInfo.NetworkContainerID != "" {
+		iPInfo[ifName].NetworkContainerID = interfaceInfo.NetworkContainerID
+		logger.Printf("[updateEndpoint] update the endpoint %s with NetworkContainerID  %s", endpointID, interfaceInfo.NetworkContainerID) //nolint
+	}
 }
 
 // verifyUpdateEndpointStateRequest verify the CNI request body for the UpdateENdpointState API

cns/restserver/ipam_test.go

@@ -2176,11 +2176,12 @@ func TestStatelessCNIStateFile(t *testing.T) {
 	endpointInfo2ContainerID := "1b4917617e15d24dc495e407d8eb5c88e4406e58fa209e4eb75a2c2fb7045eea"
 	endpointInfo2 := &EndpointInfo{IfnameToIPMap: make(map[string]*IPInfo)}
 	endpointInfo2.IfnameToIPMap["eth2"] = &IPInfo{
-		IPv4:          nil,
-		NICType:       cns.DelegatedVMNIC,
-		HnsEndpointID: "5c15cccc-830a-4dff-81f3-4b1e55cb7dcb",
-		HnsNetworkID:  "5c0712cd-824c-4898-b1c0-2fcb16ede4fb",
-		MacAddress:    "7c:1e:52:06:d3:4b",
+		IPv4:               nil,
+		NICType:            cns.DelegatedVMNIC,
+		HnsEndpointID:      "5c15cccc-830a-4dff-81f3-4b1e55cb7dcb",
+		HnsNetworkID:       "5c0712cd-824c-4898-b1c0-2fcb16ede4fb",
+		MacAddress:         "7c:1e:52:06:d3:4b",
+		NetworkContainerID: testNCID,
 	}
 	// test cases
 	tests := []struct {

cns/restserver/restserver.go

@@ -126,13 +126,14 @@ type EndpointInfo struct {
 }
 
 type IPInfo struct {
-	IPv4          []net.IPNet
-	IPv6          []net.IPNet `json:",omitempty"`
-	HnsEndpointID string      `json:",omitempty"`
-	HnsNetworkID  string      `json:",omitempty"`
-	HostVethName  string      `json:",omitempty"`
-	MacAddress    string      `json:",omitempty"`
-	NICType       cns.NICType
+	IPv4               []net.IPNet
+	IPv6               []net.IPNet `json:",omitempty"`
+	HnsEndpointID      string      `json:",omitempty"`
+	HnsNetworkID       string      `json:",omitempty"`
+	HostVethName       string      `json:",omitempty"`
+	MacAddress         string      `json:",omitempty"`
+	NetworkContainerID string      `json:",omitempty"`
+	NICType            cns.NICType
 }
 
 type GetHTTPServiceDataResponse struct {

network/endpoint.go

@@ -128,17 +128,20 @@ type RouteInfo struct {
 
 // InterfaceInfo contains information for secondary interfaces
 type InterfaceInfo struct {
-	Name              string
-	MacAddress        net.HardwareAddr
-	IPConfigs         []*IPConfig
-	Routes            []RouteInfo
-	DNS               DNSInfo
-	NICType           cns.NICType
-	SkipDefaultRoutes bool
-	HostSubnetPrefix  net.IPNet // Move this field from ipamAddResult
-	NCResponse        *cns.GetNetworkContainerResponse
-	PnPID             string
-	EndpointPolicies  []policy.Policy
+	Name                       string
+	MacAddress                 net.HardwareAddr
+	IPConfigs                  []*IPConfig
+	Routes                     []RouteInfo
+	DNS                        DNSInfo
+	NICType                    cns.NICType
+	SkipDefaultRoutes          bool
+	HostSubnetPrefix           net.IPNet // Move this field from ipamAddResult
+	NCResponse                 *cns.GetNetworkContainerResponse
+	PnPID                      string
+	EndpointPolicies           []policy.Policy
+	NetworkContainerID         string
+	AllowNCToHostCommunication bool
+	AllowHostToNCCommunication bool
 }
 
 type IPConfig struct {