Speeds up ASG syncing and Querying

docs/07-network-policy-database-overview.md

@@ -26,10 +26,10 @@ This document is intended to help people who are poking around the `network_poli
 
 ## <a name="access-db"></a> How to access an internal database
 1. Bosh ssh onto the VM where the `policy-server` is running. You can figure out what machine by running `bosh is --ps | grep policy-server`.
-2. Grab the mysql config. 
+2. Grab the mysql config.
    ```
    $ cat /var/vcap/jobs/policy-server/config/policy-server.json | grep \"database\" -A 11
-   
+
     "database": {
        "type": "mysql",
        "user": "USER",
@@ -43,7 +43,7 @@ This document is intended to help people who are poking around the `network_poli
        "skip_hostname_validation": false
     },
    ```
-   
+
 3. Bosh ssh onto the database VM.
 4. Connect to the mysql instance.
    ```
@@ -61,7 +61,11 @@ Below are all of the tables in the `network_policy` database.
 | gorp_migrations  | Record of which migrations have been run. |
 | groups  | List of all apps that are either the source or destination of a network policy. |
 | policies  | List of source apps and destination metadata for network policies. |
-
+| policies_info | A single row indicating the last update time of any network policy, used to save on DB queries from vxlan-policy-agent |
+| security_groups | Lists all security groups defined in CAPI. This is populated by policy-server-asg-syncer, and is not a source of truth. |
+| security_groups_info | A single row indicating the last update time of any security group info, used to save on DB queries from vxlan-policy-agent |
+| running_security_groups_spaces | A join table associating security groups to the spaces they are bound to for running lifecycle workloads. |
+| staging_security_groups_spaces | A join table associating security groups to the spaes they are bound to for staging lifecycle workloads. |
 
 The following tables were related to dynamic egress, which has been removed
 from the codebase. These tables should no longer present in your database as of
@@ -80,7 +84,7 @@ v3.6.0.
 
 ## <a name="network-policy-tables"></a> Network Policy Related Tables
 
-There are three tables related to cf networking policies: policies, groups, and destinations. 
+There are four tables related to cf networking policies: policies, groups, destinations, and policies_info.
 
 ### <a name="groups-table"></a> Groups
 
@@ -152,11 +156,25 @@ mysql> describe policies;
 | group_id | This is the id for the group table entry that represents the source app. |
 | destination_id | This is the id for the destinations table entry that represents the destination metadata. |
 
+### <a name="policies_info"></a> policies_info
+This table is a single row with a single value that represents the last updated timestamp of policy data,
+to allow VXLAN Policy Agent to short-circuit its sync loop if no changes were made.
+
+```
+mysql> describe policies_info;
++--------------+--------------+------+-----+----------------------+-------------------+
+| Field        | Type         | Null | Key | Default              | Extra             |
++--------------+--------------+------+-----+----------------------+-------------------+
+| id           | int          | NO   | PRI | NULL                 | auto_increment    |
+| last_updated | timestamp(6) | NO   |     | CURRENT_TIMESTAMP(6) | DEFAULT_GENERATED |
++--------------+--------------+------+-----+----------------------+-------------------+
+```
+
 
 ## <a name="network-policy-example"></a> Networking Policy Example
 
-In this example: 
-* There is a network policy from AppA to AppB. 
+In this example:
+* There is a network policy from AppA to AppB.
 * AppA has guid `2ffe4b0f-b03c-48bb-a4fa-bf22657d34a2`
 * AppB has guid `5346072e-7265-45f9-b70a-80c42e3f13ae`
 
@@ -181,14 +199,117 @@ mysql> select * from policies;           mysql> select * from destinations;
                  |  2 | 5346072e-7265-45f9-b70a-80c42e3f13ae | app  <--+
                  |  3 | NULL                                 | app  |
                  +----+--------------------------------------+------+
+```
+
+## <a name="security-groups-tables"</a> Security Group Related Tables
+
+There are four tables storing information about security groups: security_groups, running_security_groups_spaces,
+staging_security_groups_spaces, and security_groups_info.
+
+
+### <a name="security-groups-table"></a> security_groups
+This table stores a copy of all security groups found in CAPI, so vxlan-policy-agent can query
+policy-server-internal for this information, rather than overwhelm CAPI with requests. Its data is
+synced and updated via the policy-server-asg-syncer process, and is not a source of truth for ASG data.
+
+```
+mysql> describe security_groups;
++-----------------+--------------+------+-----+---------+----------------+
+| Field           | Type         | Null | Key | Default | Extra          |
++-----------------+--------------+------+-----+---------+----------------+
+| id              | bigint       | NO   | PRI | NULL    | auto_increment |
+| guid            | varchar(36)  | NO   | UNI | NULL    |                |
+| name            | varchar(255) | NO   |     | NULL    |                |
+| rules           | mediumtext   | YES  |     | NULL    |                |
+| staging_default | tinyint(1)   | YES  | MUL | 0       |                |
+| running_default | tinyint(1)   | YES  | MUL | 0       |                |
+| staging_spaces  | json         | YES  |     | NULL    |                |
+| running_spaces  | json         | YES  |     | NULL    |                |
++-----------------+--------------+------+-----+---------+----------------+
+```
 
+| Field | Note |
+|---|---|
+| id | An internal id for each record |
+| guid | The CAPI GUID of the security group |
+| name | The name of the security group as it appears in CAPI |
+| hash | A SHA256 hash of the ASG data, used to check whether records need updating during policy-server-asg-syncer polls |
+| rules | The rules (in JSON) associated with the ASG defined in CAPI |
+| staging_default | Whether or not this is a globally bound security group for `staging` lifecycles |
+| running_default | Whether or not this is a globally bound security group for `running` lifecycles |
+| staging_spaces | A json list of CAPI guids for all spaces this security group is bound to for the `staging` lifecycle. This column duplicates data in the `staging_security_groups_spaces` table, but is already in JSON format so we pull it out for faster data presentation when serving queries from VXLAN Policy Agent, while filtering via the `staging_security_groups_spaces` table. |
+| running_spaces | A json list of CAPI guids for all spaces this security group is bound to for the `running` lifecycle. This column duplicates data in the `running_security_groups_spaces` table, but is already in JSON format so we pull it out for faster data presentation when serving queries from VXLAN Policy Agent, while filtering via the `running_security_groups_spaces` table. |
+
+### <a name="running-security-groups-spaces-table"></a> running_security_groups_spaces
+This table is a join table to enable faster querying of security_groups when filtering by
+running_space guids. It is used by the BySpaceGuids() store function, when returning lists
+of ASGs for a given set of space guids. Querying the space associations directly in the security_groups
+table results in unindexed queries, and giant full-table scans which topple databases with thousands of
+ASGs. Adding this table enables indexed lookups of space guids to find the security group they're bound to,
+drasticly speeding up query times for VXLAN Policy Agent requests.
+
+It is synced and updated via the policy-server-asg-syncer process, and is not a source of
+truth for ASG data.
 
 ```
+mysql> describe running_security_groups_spaces;
++---------------------+--------------+------+-----+---------+-------+
+| Field               | Type         | Null | Key | Default | Extra |
++---------------------+--------------+------+-----+---------+-------+
+| space_guid          | varchar(255) | NO   | PRI | NULL    |       |
+| security_group_guid | varchar(255) | NO   | PRI | NULL    |       |
++---------------------+--------------+------+-----+---------+-------+
+```
+
+| Field | Note|
+|---|---|
+| space_guid | This value is the CAPI guid for the space bound to a given security group via the `running` app lifecycle |
+| security_group_guid | This value is the CAPI guid for the security group bound to a given space via the `running` app lifecycle |
 
 
+### <a name="staging-security-groups-spaces-table"></a> staging_security_groups_spaces
+This table is a join table to enable faster querying of security_groups when filtering by
+staging_space guids. It is used by the BySpaceGuids() store function, when returning lists
+of ASGs for a given set of space guids. Querying the space associations directly in the security_groups
+table results in unindexed queries, and giant full-table scans which topple databases with thousands of
+ASGs. Adding this table enables indexed lookups of space guids to find the security group they're bound to,
+drasticly speeding up query times for VXLAN Policy Agent requests.
+
+It is synced and updated via the policy-server-asg-syncer process, and is not a source of
+truth for ASG data.
+
+```
+mysql> describe staging_security_groups_spaces;
++---------------------+--------------+------+-----+---------+-------+
+| Field               | Type         | Null | Key | Default | Extra |
++---------------------+--------------+------+-----+---------+-------+
+| space_guid          | varchar(255) | NO   | PRI | NULL    |       |
+| security_group_guid | varchar(255) | NO   | PRI | NULL    |       |
++---------------------+--------------+------+-----+---------+-------+
+```
+
+| Field | Note|
+|---|---|
+| space_guid | This value is the CAPI guid for the space bound to a given security group via the `staging` app lifecycle |
+| security_group_guid | This value is the CAPI guid for the security group bound to a given space via the `staging` app lifecycle |
+
+### <a name="security_groups_info"></a> security_groups_info
+This table is a single row with a single value that represents the last updated timestamp of security group data,
+to allow VXLAN Policy Agent to short-circuit its sync loop if no changes were made.
+
+```
+mysql> describe security_groups_info;
++--------------+--------------+------+-----+----------------------+-------------------+
+| Field        | Type         | Null | Key | Default              | Extra             |
++--------------+--------------+------+-----+----------------------+-------------------+
+| id           | int          | NO   | PRI | NULL                 | auto_increment    |
+| last_updated | timestamp(6) | NO   |     | CURRENT_TIMESTAMP(6) | DEFAULT_GENERATED |
++--------------+--------------+------+-----+----------------------+-------------------+
+```
+
 ## <a name="migrations-tables"></a> Migration Related Tables
 
-There are two tables related to migraitons: gorp_migrations and gorp_lock. 
+There are two tables related to migrations: gorp_migrations and gorp_lock.
 
 ### <a name="gorp-mirations-table"></a> gorp_migrations
 This table tracks what database migrations have been applied.

src/code.cloudfoundry.org/cf-pusher/cmd/multispace-pusher/main.go

@@ -29,6 +29,7 @@ type Config struct {
 	TotalSpaces                         int  `json:"total_spaces"`
 	AppsPerSpace                        int  `json:"apps_per_space"`
 	SkipASGCreation                     bool `json:"skip_asg_creation"`
+	SkipSpaceCreation                   bool `json:"skip_space_creation"`
 }
 
 type ConcurrentSpaceSetup struct {
@@ -60,9 +61,9 @@ func main() {
 	if !config.SkipASGCreation {
 		// Create global asgs
 		createGlobalASGs(config)
-		// Create a bunch of bindable ASGs
-		asgs = createASGs(config.TotalASGs-config.GlobalASGs, config.ASGSize, config.Prefix, globalAdapter)
 	}
+	// Create a bunch of bindable ASGs
+	asgs = createASGs(config.TotalASGs-config.GlobalASGs, config.ASGSize, config.Prefix, globalAdapter, config.SkipASGCreation)
 	spaces = createSpacesConcurrently(config)
 
 	orgName := fmt.Sprintf("%s-org", config.Prefix)
@@ -94,30 +95,34 @@ func createSpacesConcurrently(config Config) []string {
 	sem := make(chan bool, config.Concurrency)
 	var spaceNames []string
 	for i := 0; i < config.TotalSpaces; i++ {
-		sem <- true
 		setup := generateConcurrentSpaceSetup(i, config)
 		spaceNames = append(spaceNames, setup.OrgSpaceCreator.Space)
-		go func(s *ConcurrentSpaceSetup, c Config, index int) {
-			defer func() { <-sem }()
-
-			// Connect to the api with this adapter
-			if err := s.ApiConnector.Connect(); err != nil {
-				log.Fatalf("connecting to api: %s", err)
-			}
+		if !config.SkipSpaceCreation {
+			sem <- true
+			go func(s *ConcurrentSpaceSetup, c Config, index int) {
+				defer func() { <-sem }()
+
+				// Connect to the api with this adapter
+				if err := s.ApiConnector.Connect(); err != nil {
+					log.Fatalf("connecting to api: %s", err)
+				}
 
-			// Create and target the space
-			if err := s.OrgSpaceCreator.Create(); err != nil {
-				log.Fatalf("creating org and space: %s", err)
-			}
+				// Create and target the space
+				if err := s.OrgSpaceCreator.Create(); err != nil {
+					log.Fatalf("creating org and space: %s", err)
+				}
 
-			// Push apps for this space
-			if err := s.AppPusher.Push(); err != nil {
-				log.Printf("Got an error while pushing proxy apps: %s", err)
-			}
-		}(setup, config, i)
+				// Push apps for this space
+				if err := s.AppPusher.Push(); err != nil {
+					log.Printf("Got an error while pushing proxy apps: %s", err)
+				}
+			}(setup, config, i)
+		}
 	}
-	for i := 0; i < cap(sem); i++ {
-		sem <- true
+	if !config.SkipSpaceCreation {
+		for i := 0; i < cap(sem); i++ {
+			sem <- true
+		}
 	}
 	return spaceNames
 }
@@ -268,27 +273,29 @@ func bindASGToThisSpace(asg string, orgName, spaceName string, adapter *cf_cli_a
 	}
 }
 
-func createASGs(howMany, asgSize int, prefix string, adapter *cf_cli_adapter.Adapter) []string {
+func createASGs(howMany, asgSize int, prefix string, adapter *cf_cli_adapter.Adapter, skipASGCreation bool) []string {
 	var asgNames []string
 	for i := 0; i < howMany; i++ {
 		asgName := fmt.Sprintf("%s-many-%d-asg", prefix, i)
 		asgNames = append(asgNames, asgName)
-		asgContent := testsupport.BuildASG(asgSize)
-		asgFile, err := testsupport.CreateTempFile(asgContent)
-		if err != nil {
-			log.Fatalf("creating asg file: %s", err)
-		}
-
-		// check ASG and create if not OK
-		asgChecker := cf_command.ASGChecker{Adapter: adapter}
-		asgErr := asgChecker.CheckASG(asgName, asgContent)
-		if asgErr != nil {
-			// install ASG
-			if err := adapter.DeleteSecurityGroup(asgName); err != nil {
-				log.Fatalf("deleting security group: %s", err)
+		if !skipASGCreation {
+			asgContent := testsupport.BuildASG(asgSize)
+			asgFile, err := testsupport.CreateTempFile(asgContent)
+			if err != nil {
+				log.Fatalf("creating asg file: %s", err)
 			}
-			if err := adapter.CreateSecurityGroup(asgName, asgFile); err != nil {
-				log.Fatalf("creating security group: %s", err)
+
+			// check ASG and create if not OK
+			asgChecker := cf_command.ASGChecker{Adapter: adapter}
+			asgErr := asgChecker.CheckASG(asgName, asgContent)
+			if asgErr != nil {
+				// install ASG
+				if err := adapter.DeleteSecurityGroup(asgName); err != nil {
+					log.Fatalf("deleting security group: %s", err)
+				}
+				if err := adapter.CreateSecurityGroup(asgName, asgFile); err != nil {
+					log.Fatalf("creating security group: %s", err)
+				}
 			}
 		}
 	}

src/code.cloudfoundry.org/policy-server/cmd/policy-server-asg-syncer/main.go

@@ -73,7 +73,8 @@ func main() {
 	}
 
 	securityGroupsStore := &store.SGStore{
-		Conn: connectionPool,
+		Logger: logger.Session("security-groups-store"),
+		Conn:   connectionPool,
 	}
 
 	metricsSender := &metrics.MetricsSender{

src/code.cloudfoundry.org/policy-server/cmd/policy-server-internal/main.go

@@ -80,7 +80,8 @@ func main() {
 	)
 
 	securityGroupsStore := &store.SGStore{
-		Conn: connectionPool,
+		Logger: logger.Session("security-groups-store"),
+		Conn:   connectionPool,
 	}
 
 	tagDataStore := store.NewTagStore(connectionPool, &store.GroupTable{}, conf.TagLength)

src/code.cloudfoundry.org/policy-server/store/helpers/helpers.go

@@ -35,19 +35,3 @@ func RebindForSQLDialect(query, dialect string) string {
 	}
 	return strings.Join(strParts, "")
 }
-
-func RebindForSQLDialectAndMark(query, dialect, mark string) string {
-	if dialect != Postgres && dialect != MySQL {
-		panic(fmt.Sprintf("Unrecognized DB dialect '%s'", dialect))
-	}
-
-	if dialect == MySQL {
-		return strings.ReplaceAll(query, mark, "?")
-	}
-
-	strParts := strings.Split(query, mark)
-	for i := 1; i < len(strParts); i++ {
-		strParts[i-1] = fmt.Sprintf("%s$%d", strParts[i-1], i)
-	}
-	return strings.Join(strParts, "")
-}

src/code.cloudfoundry.org/policy-server/store/migrations/migrations.go

@@ -472,4 +472,40 @@ var MigrationsToPerform = PolicyServerMigrations{
 		Id: "91",
 		Up: migration_v0091,
 	},
+	PolicyServerMigration{
+		Id: "92",
+		Up: migration_v0092,
+	},
+	PolicyServerMigration{
+		Id: "93",
+		Up: migration_v0093,
+	},
+	PolicyServerMigration{
+		Id: "94",
+		Up: migration_v0094,
+	},
+	PolicyServerMigration{
+		Id: "95",
+		Up: migration_v0095,
+	},
+	PolicyServerMigration{
+		Id: "96",
+		Up: migration_v0096,
+	},
+	PolicyServerMigration{
+		Id: "97",
+		Up: migration_v0097,
+	},
+	PolicyServerMigration{
+		Id: "98",
+		Up: migration_v0098,
+	},
+	PolicyServerMigration{
+		Id: "99",
+		Up: migration_v0099,
+	},
+	PolicyServerMigration{
+		Id: "100",
+		Up: migration_v0100,
+	},
 }