Align tests and schemas

actions/networking/verify.go

@@ -4,8 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
-	"strconv"
 	"strings"
+	"testing"
 
 	"github.com/rancher/shepherd/clients/rancher"
 	v1 "github.com/rancher/shepherd/clients/rancher/v1"
@@ -15,6 +15,7 @@ import (
 	"github.com/rancher/shepherd/extensions/sshkeys"
 	"github.com/rancher/tests/actions/clusters"
 	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
 	"golang.org/x/crypto/ssh"
 	corev1 "k8s.io/api/core/v1"
 )
@@ -86,56 +87,68 @@ func VerifyNetworkPolicy(client *rancher.Client, clusterID string, namespaceName
 	return nil
 }
 
-// VerifyNodePortConnectivity verifies that the node port is accessible by curling the worker node external IP
-func VerifyNodePortConnectivity(client *rancher.Client, clusterID string, nodePort int, workloadName string) error {
-	steveClient, err := client.Steve.ProxyDownstream(clusterID)
+// verifyConnectivityFromWorkerNodes verifies if any worker node in the cluster is able to access the provided ip:port
+// and retrieve the expected content from name.html.
+func verifyConnectivityFromWorkerNodes(client *rancher.Client, clusterID string, ip string, port int, workloadName string) error {
+	query, err := url.ParseQuery(clusters.LabelWorker)
 	if err != nil {
 		return err
 	}
 
-	query, err := url.ParseQuery(clusters.LabelWorker)
+	steveClient, err := client.Steve.ProxyDownstream(clusterID)
 	if err != nil {
-		return fmt.Errorf("failed to build worker node query: %w", err)
+		return err
 	}
 
 	nodeList, err := steveClient.SteveType(stevetypes.Node).List(query)
 	if err != nil {
-		return fmt.Errorf("failed to list worker nodes: %w", err)
+		return err
 	}
 
 	if len(nodeList.Data) == 0 {
 		return errors.New("no worker nodes found")
 	}
 
 	for _, machine := range nodeList.Data {
-		newNode := &corev1.Node{}
-		err = v1.ConvertToK8sType(machine.JSONResp, newNode)
+		sshNode, err := sshkeys.GetSSHNodeFromMachine(client, &machine)
 		if err != nil {
-			return fmt.Errorf("failed to convert node %s: %w", machine.Name, err)
-		}
-
-		nodeIP := kubeapinodes.GetNodeIP(newNode, corev1.NodeExternalIP)
-		if nodeIP == "" {
-			nodeIP = kubeapinodes.GetNodeIP(newNode, corev1.NodeInternalIP)
+			logrus.Debugf("Could not SSH into worker node %s: %s", machine.Name, err.Error())
+			continue
 		}
 
-		logrus.Debugf("Curling node port %d on node %s (%s)", nodePort, machine.Name, nodeIP)
-		execCmd := []string{"curl", fmt.Sprintf("%s:%s/name.html", nodeIP, strconv.Itoa(nodePort))}
-		log, err := kubectl.Command(client, nil, clusterID, execCmd, "")
-		if err != nil {
-			return fmt.Errorf("curl command failed on node %s: %w", machine.Name, err)
+		logrus.Debugf("Curling '%s:%d/name.html' from node %s", ip, port, machine.Name)
+		log, err := sshNode.ExecuteCommand(fmt.Sprintf("curl -s %s:%d/name.html", ip, port))
+		if err != nil && !errors.Is(err, &ssh.ExitMissingError{}) {
+			logrus.Debugf("Curl failed on node %s: %v", machine.Name, err)
+			continue
 		}
 
-		if strings.Contains(log, workloadName) {
+		if strings.Contains(log, workloadName) { // This should be one of the pod's names.
 			return nil
+		} else {
+			logrus.Debugf("Curl result %s doesn't contain expected content '%s'", log, workloadName)
 		}
 	}
 
-	return fmt.Errorf("unable to access node port %d for workload %s", nodePort, workloadName)
+	return fmt.Errorf("Unable to connect to %s:%d/name.html from any worker node", ip, port)
 }
 
-// VerifyHostPortConnectivity verifies that the host port is accessible on worker nodes by SSHing directly into each node
-func VerifyHostPortConnectivity(client *rancher.Client, clusterID string, hostPort int, workloadName string) error {
+func verifyConnectivityFromPod(client *rancher.Client, clusterID string, ip string, port int, workloadName string) error {
+	execCmd := []string{"curl", "-s", fmt.Sprintf("%s:%d/name.html", ip, port)}
+	log, err := kubectl.Command(client, nil, clusterID, execCmd, "")
+	if err != nil {
+		return err
+	}
+
+	if !strings.Contains(log, workloadName) { // This should be one of the pod's names.
+		return fmt.Errorf("Curl result %s doesn't include the workload name %s", log, workloadName)
+	}
+
+	return nil
+}
+
+// VerifyNodePortConnectivity verifies that the node port is accessible by curling the worker node external IP
+func VerifyNodePortConnectivity(client *rancher.Client, clusterID string, nodePort int, workloadName string) error {
 	steveClient, err := client.Steve.ProxyDownstream(clusterID)
 	if err != nil {
 		return err
@@ -156,28 +169,55 @@ func VerifyHostPortConnectivity(client *rancher.Client, clusterID string, hostPo
 	}
 
 	for _, machine := range nodeList.Data {
-		sshNode, err := sshkeys.GetSSHNodeFromMachine(client, &machine)
+		newNode := &corev1.Node{}
+		err = v1.ConvertToK8sType(machine.JSONResp, newNode)
 		if err != nil {
-			logrus.Debugf("Could not SSH into worker node %s, skipping: %v", machine.Name, err)
-			continue
+			return fmt.Errorf("failed to convert node %s: %w", machine.Name, err)
 		}
 
-		logrus.Debugf("Curling host port %d on worker node %s", hostPort, machine.Name)
-		output, err := sshNode.ExecuteCommand(fmt.Sprintf("curl localhost:%d/name.html", hostPort))
-		if err != nil {
-			continue
+		nodeIP := kubeapinodes.GetNodeIP(newNode, corev1.NodeExternalIP)
+		if nodeIP == "" {
+			nodeIP = kubeapinodes.GetNodeIP(newNode, corev1.NodeInternalIP)
 		}
 
-		if strings.Contains(output, workloadName) {
-			return nil
-		}
+		logrus.Debugf("Curling node port %d on node %s (%s)", nodePort, machine.Name, nodeIP)
+		verifyConnectivityFromPod(client, clusterID, nodeIP, nodePort, workloadName)
 	}
 
-	return fmt.Errorf("unable to access host port %d for workload %s on any worker node", hostPort, workloadName)
+	return fmt.Errorf("unable to access node port %d for workload %s", nodePort, workloadName)
+}
+
+// VerifyLoadBalancerConnectivity verifies that the Load Balancer service is accessible by curling its IP:port.
+// This includes
+func VerifyLoadBalancerConnectivity(t *testing.T, client *rancher.Client, clusterID string, serviceID string, workloadName string) {
+	steveClient, err := client.Steve.ProxyDownstream(clusterID)
+	require.NoError(t, err)
+
+	service, err := steveClient.SteveType(stevetypes.Service).ByID(serviceID)
+	require.NoError(t, err)
+
+	k8sService := &corev1.Service{}
+	err = v1.ConvertToK8sType(service, k8sService)
+	require.NoError(t, err)
+	require.Equal(t, corev1.ServiceTypeLoadBalancer, k8sService.Spec.Type)
+	require.NotEmpty(t, k8sService.Spec.Ports)
+	require.NotEmpty(t, k8sService.Status.LoadBalancer.Ingress)
+
+	port := k8sService.Spec.Ports[0].Port
+	ip := k8sService.Status.LoadBalancer.Ingress[0].IP
+	t.Logf("Testing connectivity with load balancer %s by curling %s:%d/name.html", k8sService.Name, ip, port)
+
+	err = verifyConnectivityFromPod(client, clusterID, ip, int(port), workloadName)
+	require.NoError(t, err)
+}
+
+// VerifyHostPortConnectivity verifies that the host port is accessible on worker nodes by SSHing directly into each node
+func VerifyHostPortConnectivity(client *rancher.Client, clusterID string, hostPort int, workloadName string) error {
+	return verifyConnectivityFromWorkerNodes(client, clusterID, "localhost", hostPort, workloadName)
 }
 
 // VerifyClusterConnectivity verifies that the ClusterIP service is accessible via SSH from a worker node
-func VerifyClusterConnectivity(client *rancher.Client, clusterID string, serviceID string, path string, content string) error {
+func VerifyClusterConnectivity(client *rancher.Client, clusterID string, serviceID string, port int, content string) error {
 	steveClient, err := client.Steve.ProxyDownstream(clusterID)
 	if err != nil {
 		return err
@@ -194,40 +234,5 @@ func VerifyClusterConnectivity(client *rancher.Client, clusterID string, service
 		return err
 	}
 
-	clusterIP := newService.Spec.ClusterIP
-
-	query, err := url.ParseQuery(clusters.LabelWorker)
-	if err != nil {
-		return err
-	}
-
-	nodeList, err := steveClient.SteveType(stevetypes.Node).List(query)
-	if err != nil {
-		return err
-	}
-
-	if len(nodeList.Data) == 0 {
-		return errors.New("no worker nodes found")
-	}
-
-	for _, machine := range nodeList.Data {
-		sshNode, err := sshkeys.GetSSHNodeFromMachine(client, &machine)
-		if err != nil {
-			logrus.Debugf("Could not SSH into worker node %s, trying next node: %v", machine.Name, err)
-			continue
-		}
-
-		logrus.Debugf("Curling cluster IP %s:%s from node %s", clusterIP, path, machine.Name)
-		log, err := sshNode.ExecuteCommand(fmt.Sprintf("curl %s:%s", clusterIP, path))
-		if err != nil && !errors.Is(err, &ssh.ExitMissingError{}) {
-			logrus.Debugf("Curl failed on node %s: %v, trying next node", machine.Name, err)
-			continue
-		}
-
-		if strings.Contains(log, content) {
-			return nil
-		}
-	}
-
-	return fmt.Errorf("unable to connect to the cluster IP %s:%s from any worker node", clusterIP, path)
+	return verifyConnectivityFromWorkerNodes(client, clusterID, newService.Spec.ClusterIP, port, content)
 }

actions/workloads/deployment/verify.go

@@ -384,11 +384,12 @@ func VerifyDeploymentPodScaleDown(client *rancher.Client, clusterID, namespace,
 		return err
 	}
 
-	logrus.Debugf("Updating deployment (%s) replicas from %v to %v", deployment.Name, *deployment.Spec.Replicas, *deployment.Spec.Replicas-1)
 	replicas := int32(*deployment.Spec.Replicas - 1)
 	if replicas < 0 {
 		return errors.New("Can't scale down a deployment with 0 replicas")
 	}
+
+	logrus.Debugf("Updating deployment (%s) replicas from %v to %v", deployment.Name, *deployment.Spec.Replicas, replicas)
 	deployment.Spec.Replicas = &replicas
 
 	deployment, err = extdeploymentsapi.UpdateDeployment(client, clusterID, deployment, true)

validation/charts/appco/schemas/pit_schemas.yaml

@@ -1,4 +1,4 @@
-- projects: [RANCHERINT]
+- projects: [ RANCHERINT ]
   suite: AppCo
   cases:
   - title: "Install in SideCar Mode"

validation/charts/schemas/pit_schemas.yaml

@@ -1,4 +1,4 @@
-- projects: [RANCHERINT]
+- projects: [ RANCHERINT ]
   suite: Charts
   cases:
   - title: "CIS Benchmark Chart Installation and Scan"
@@ -36,7 +36,7 @@
       expectedresult: "The 'System' project is successfully created"
       position: 2
     - action: "Install Rancher Alerting chart"
-      data: "helm install rancher-alerting-drivers --namespace=default"
+      data: "helm install rancher-alerting-drivers --namespace=cattle-monitoring-system"
       expectedresult: "Alerting chart installation starts successfully"
       position: 3
     - action: "Wait for Alerting deployments"
@@ -107,10 +107,10 @@
       expectedresult: "Install payload prepared successfully"
       position: 5
     - action: "Install NeuVector charts"
-      data: "Install neuvector, neuvector-monitor, and neuvector-crd charts"
+      data: "Install neuvector and neuvector-crd charts"
       expectedresult: "All NeuVector charts installed successfully"
       position: 6
-    - action: "Wait for neuvector, neuvector-monitor, and neuvector-crd charts installations to complete"
+    - action: "Wait for neuvector and neuvector-crd charts installations to complete"
       expectedresult: "Charts reports successful installations"
       position: 7
     - action: "Verify NeuVector deployments readiness"

validation/fleet/schemas/pit_schemas.yaml

@@ -15,7 +15,7 @@
       position: 2
     - action: "Deploy a GitRepo object targetting the specified downstream cluster"
       data: "/validation/fleet/schemas/gitrepo.yaml"
-      expectedresult: "the gitRepo itself comes to an active state and the resources defined in the spec are created on the downstream cluster in the fleet-testns namespace"
+      expectedresult: "the gitRepo itself comes to an active state and the resources defined in the spec are created on the downstream cluster in the test namespace"
       position: 3
     custom_field:
       "15": "TestGitRepoDeployment"

validation/longhorn/schemas/pit_schemas.yaml

@@ -147,27 +147,27 @@
       data: ""
       expectedresult: "Manifest prepared with Longhorn PVC template"
       position: 1
-    - action: "Deploy StatefulSet with 3 replicas via Rancher or kubectl"
+    - action: "Deploy StatefulSet with 1 replica via Rancher"
       data: ""
       expectedresult: "StatefulSet deployment begins"
       position: 2
     - action: "Verify each pod gets dedicated persistent volume"
       data: ""
-      expectedresult: "3 volumes created, each attached to respective pod"
+      expectedresult: "1 volume created, attached to respective pod"
       position: 3
     - action: "Write unique test data to each pod's volume"
       data: ""
       expectedresult: "Data successfully written to all volumes"
       position: 4
-    - action: "Scale StatefulSet up to 5 replicas"
+    - action: "Scale StatefulSet up to number of nodes + 1"
       data: ""
       expectedresult: "Scaling operation succeeds"
       position: 5
     - action: "Verify new pods get new dedicated volumes"
       data: ""
-      expectedresult: "2 additional volumes created and attached"
+      expectedresult: "additional volumes equal to the number of nodes created and attached"
       position: 6
-    - action: "Scale StatefulSet down to 2 replicas"
+    - action: "Scale StatefulSet down to 1 replica"
       data: ""
       expectedresult: "Scaling down completes"
       position: 7
@@ -801,7 +801,7 @@
       data: ""
       expectedresult: "Workload uses encrypted volume"
       position: 3
-    - action: "Verify volume shows as encrypted in Longhorn UI"
+    - action: "Verify volume shows as encrypted in Longhorn API"
       data: ""
       expectedresult: "Volume displays encryption status"
       position: 4