Postgres Operator automated upgrade status checks

tjmoore4 · andrewlecuyer · commit b2c49ff19d27 · 2020-04-28T21:42:14.000-05:00
Checks were added for the various PGO cli operations
to validate that the cluster in question is either current to
this release of the Postgres Operator, or has been upgraded to
the current version. This is to ensure that unexpected results
are not achieved due to pgclusters that are out of spec.

A check was also added to ensure the ConfigMap Controller only
syncs configmaps on clusters that have already been upgraded.
diff --git a/apiserver/backrestservice/backrestimpl.go b/apiserver/backrestservice/backrestimpl.go
@@ -125,6 +125,14 @@ func CreateBackup(request *msgs.CreateBackrestBackupRequest, ns, pgouser string)
 			return resp
 		}
 
+		// check if the current cluster is not upgraded to the deployed
+		// Operator version. If not, do not allow the command to complete
+		if cluster.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+			resp.Status.Code = msgs.Error
+			resp.Status.Msg = fmt.Sprintf("%s %s", cluster.Name, msgs.UpgradeError)
+			return resp
+		}
+
 		if cluster.Labels[config.LABEL_BACKREST] != "true" {
 			resp.Status.Code = msgs.Error
 			resp.Status.Msg = clusterName + " does not have pgbackrest enabled"
@@ -472,7 +480,15 @@ func Restore(request *msgs.RestoreRequest, ns, pgouser string) msgs.RestoreRespo
 		return resp
 	}
 
-	//verify that the cluster we are restoring from has backrest enabled
+	// check if the current cluster is not upgraded to the deployed
+	// Operator version. If not, do not allow the command to complete
+	if cluster.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+		resp.Status.Code = msgs.Error
+		resp.Status.Msg = fmt.Sprintf("%s %s", cluster.Name, msgs.UpgradeError)
+		return resp
+	}
+
+	// verify that the cluster we are restoring from has backrest enabled
 	if cluster.Labels[config.LABEL_BACKREST] != "true" {
 		resp.Status.Code = msgs.Error
 		resp.Status.Msg = "can't restore, cluster restoring from does not have backrest enabled"
diff --git a/apiserver/catservice/catimpl.go b/apiserver/catservice/catimpl.go
@@ -55,6 +55,14 @@ func Cat(request *msgs.CatRequest, ns string) msgs.CatResponse {
 		return resp
 	}
 
+	// check if the current cluster is not upgraded to the deployed
+	// Operator version. If not, do not allow the command to complete
+	if cluster.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+		resp.Status.Code = msgs.Error
+		resp.Status.Msg = cluster.Name + msgs.UpgradeError
+		return resp
+	}
+
 	err = validateArgs(request.Args)
 	if err != nil {
 		resp.Status.Code = msgs.Error
diff --git a/apiserver/cloneservice/cloneimpl.go b/apiserver/cloneservice/cloneimpl.go
@@ -32,7 +32,7 @@ import (
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-//  Clone allows a user to clone a cluster into a new deployment
+// Clone allows a user to clone a cluster into a new deployment
 func Clone(request *msgs.CloneRequest, namespace, pgouser string) msgs.CloneResponse {
 	log.Debugf("clone called with ")
 
@@ -67,6 +67,14 @@ func Clone(request *msgs.CloneRequest, namespace, pgouser string) msgs.CloneResp
 		return response
 	}
 
+	// check if the current cluster is not upgraded to the deployed
+	// Operator version. If not, do not allow the command to complete
+	if sourcePgcluster.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+		response.Status.Code = msgs.Error
+		response.Status.Msg = sourcePgcluster.Name + msgs.UpgradeError
+		return response
+	}
+
 	// now, let's ensure the target pgCluster does *not* exist
 	targetPgcluster := crv1.Pgcluster{}
 	targetPgclusterExists, _ := kubeapi.Getpgcluster(apiserver.RESTClient,
diff --git a/apiserver/clusterservice/clusterimpl.go b/apiserver/clusterservice/clusterimpl.go
@@ -74,6 +74,14 @@ func DeleteCluster(name, selector string, deleteData, deleteBackups bool, ns, pg
 
 	for _, cluster := range clusterList.Items {
 
+		// check if the current cluster is not upgraded to the deployed
+		// Operator version. If not, do not allow the command to complete
+		if cluster.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+			response.Status.Code = msgs.Error
+			response.Status.Msg = cluster.Name + msgs.UpgradeError
+			return response
+		}
+
 		log.Debugf("deleting cluster %s", cluster.Spec.Name)
 		taskName := cluster.Spec.Name + "-rmdata"
 		log.Debugf("creating taskName %s", taskName)
diff --git a/apiserver/clusterservice/scaleimpl.go b/apiserver/clusterservice/scaleimpl.go
@@ -67,6 +67,14 @@ func ScaleCluster(name, replicaCount, storageConfig, nodeLabel,
 		return response
 	}
 
+	// check if the current cluster is not upgraded to the deployed
+	// Operator version. If not, do not allow the command to complete
+	if cluster.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+		response.Status.Code = msgs.Error
+		response.Status.Msg = cluster.Name + msgs.UpgradeError
+		return response
+	}
+
 	spec := crv1.PgreplicaSpec{}
 
 	//refer to the cluster's replica storage setting by default
diff --git a/apiserver/failoverservice/failoverimpl.go b/apiserver/failoverservice/failoverimpl.go
@@ -40,13 +40,21 @@ func CreateFailover(request *msgs.CreateFailoverRequest, ns, pgouser string) msg
 	resp.Status.Msg = ""
 	resp.Results = make([]string, 0)
 
-	_, err = validateClusterName(request.ClusterName, ns)
+	cluster, err := validateClusterName(request.ClusterName, ns)
 	if err != nil {
 		resp.Status.Code = msgs.Error
 		resp.Status.Msg = err.Error()
 		return resp
 	}
 
+	// check if the current cluster is not upgraded to the deployed
+	// Operator version. If not, do not allow the command to complete
+	if cluster.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+		resp.Status.Code = msgs.Error
+		resp.Status.Msg = cluster.Name + msgs.UpgradeError
+		return resp
+	}
+
 	if request.Target != "" {
 		_, err = isValidFailoverTarget(request.Target, request.ClusterName, ns)
 		if err != nil {
@@ -166,6 +174,7 @@ func validateClusterName(clusterName, ns string) (*crv1.Pgcluster, error) {
 	cluster := crv1.Pgcluster{}
 	found, err := kubeapi.Getpgcluster(apiserver.RESTClient,
 		&cluster, clusterName, ns)
+
 	if !found {
 		return &cluster, errors.New("no cluster found named " + clusterName)
 	}
diff --git a/apiserver/loadservice/loadimpl.go b/apiserver/loadservice/loadimpl.go
@@ -142,6 +142,15 @@ func Load(request *msgs.LoadRequest, ns, pgouser string) msgs.LoadResponse {
 				resp.Status.Msg = err.Error()
 				return resp
 			}
+
+			// check if the current cluster is not upgraded to the deployed
+			// Operator version. If not, do not allow the command to complete
+			if cl.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+				resp.Status.Code = msgs.Error
+				resp.Status.Msg = cl.Name + msgs.UpgradeError
+				return resp
+			}
+
 			clusterList.Items = append(clusterList.Items, cl)
 		}
 	}
diff --git a/apiserver/pgbouncerservice/pgbouncerimpl.go b/apiserver/pgbouncerservice/pgbouncerimpl.go
@@ -79,6 +79,15 @@ func CreatePgbouncer(request *msgs.CreatePgbouncerRequest, ns, pgouser string) m
 	}
 
 	for _, cluster := range clusterList.Items {
+
+		// check if the current cluster is not upgraded to the deployed
+		// Operator version. If not, do not allow the command to complete
+		if cluster.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+			resp.Status.Code = msgs.Error
+			resp.Status.Msg = cluster.Name + msgs.UpgradeError
+			return resp
+		}
+
 		log.Debugf("adding pgbouncer to cluster [%s]", cluster.Name)
 
 		resources := v1.ResourceList{}
diff --git a/apiserver/pgdumpservice/pgdumpimpl.go b/apiserver/pgdumpservice/pgdumpimpl.go
@@ -111,6 +111,14 @@ func CreatepgDump(request *msgs.CreatepgDumpBackupRequest, ns string) msgs.Creat
 			return resp
 		}
 
+		// check if the current cluster is not upgraded to the deployed
+		// Operator version. If not, do not allow the command to complete
+		if cluster.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+			resp.Status.Code = msgs.Error
+			resp.Status.Msg = cluster.Name + msgs.UpgradeError
+			return resp
+		}
+
 		RemovePgDumpJob(clusterName+pgDumpJobExtension, ns)
 
 		result := crv1.Pgtask{}
diff --git a/apiserver/reloadservice/reloadimpl.go b/apiserver/reloadservice/reloadimpl.go
@@ -85,6 +85,14 @@ func Reload(request *msgs.ReloadRequest, ns, username string) msgs.ReloadRespons
 			return resp
 		}
 
+		// check if the current cluster is not upgraded to the deployed
+		// Operator version. If not, do not allow the command to complete
+		if cluster.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+			resp.Status.Code = msgs.Error
+			resp.Status.Msg = cluster.Name + msgs.UpgradeError
+			return resp
+		}
+
 		var podList *v1.PodList
 		selector := config.LABEL_SERVICE_NAME + "=" + cluster.Spec.Name
 		podList, err = kubeapi.GetPods(apiserver.Clientset, selector, ns)
diff --git a/apiserver/scheduleservice/scheduleimpl.go b/apiserver/scheduleservice/scheduleimpl.go
@@ -20,10 +20,9 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/crunchydata/postgres-operator/apiserver/backupoptions"
-
 	crv1 "github.com/crunchydata/postgres-operator/apis/crunchydata.com/v1"
 	"github.com/crunchydata/postgres-operator/apiserver"
+	"github.com/crunchydata/postgres-operator/apiserver/backupoptions"
 	msgs "github.com/crunchydata/postgres-operator/apiservermsgs"
 	"github.com/crunchydata/postgres-operator/config"
 	"github.com/crunchydata/postgres-operator/kubeapi"
@@ -144,6 +143,14 @@ func CreateSchedule(request *msgs.CreateScheduleRequest, ns string) msgs.CreateS
 	log.Debug("Making schedules")
 	var schedules []*PgScheduleSpec
 	for _, cluster := range clusterList.Items {
+
+		// check if the current cluster is not upgraded to the deployed
+		// Operator version. If not, do not allow the command to complete
+		if cluster.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+			sr.Response.Status.Code = msgs.Error
+			sr.Response.Status.Msg = cluster.Name + msgs.UpgradeError
+			return *sr.Response
+		}
 		switch sr.Request.ScheduleType {
 		case "pgbackrest":
 			schedule := sr.createBackRestSchedule(&cluster, ns)
diff --git a/apiserver/userservice/userimpl.go b/apiserver/userservice/userimpl.go
@@ -25,6 +25,7 @@ import (
 	crv1 "github.com/crunchydata/postgres-operator/apis/crunchydata.com/v1"
 	"github.com/crunchydata/postgres-operator/apiserver"
 	msgs "github.com/crunchydata/postgres-operator/apiservermsgs"
+	"github.com/crunchydata/postgres-operator/config"
 	"github.com/crunchydata/postgres-operator/kubeapi"
 	"github.com/crunchydata/postgres-operator/util"
 
@@ -203,6 +204,14 @@ func CreateUser(request *msgs.CreateUserRequest, pgouser string) msgs.CreateUser
 			continue
 		}
 
+		// check if the current cluster is not upgraded to the deployed
+		// Operator version. If not, do not allow the command to complete
+		if cluster.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+			response.Status.Code = msgs.Error
+			response.Status.Msg = cluster.Spec.ClusterName + msgs.UpgradeError
+			return response
+		}
+
 		// build up the SQL clause that will be executed.
 		sql := sqlCreateRole
 
diff --git a/controller/configmap/synchandler.go b/controller/configmap/synchandler.go
@@ -59,6 +59,13 @@ func (c *Controller) handleConfigMapSync(key string) error {
 		return err
 	}
 
+	// if an upgrade is pending for the cluster, then don't attempt to sync and just return
+	if cluster.GetAnnotations()[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {
+		log.Debugf("ConfigMap Controller: syncing of configMap %s (namespace %s) disabled pending the "+
+			"upgrade of cluster %s", configMapName, namespace, clusterName)
+		return nil
+	}
+
 	// disable syncing when the cluster isn't currently initialized
 	if cluster.Status.State != crv1.PgclusterStateInitialized {
 		return nil

Original file line number	Diff line number	Diff line change
`@@ -142,6 +142,15 @@ func Load(request *msgs.LoadRequest, ns, pgouser string) msgs.LoadResponse {`
`142`	`142`	`resp.Status.Msg = err.Error()`
`143`	`143`	`return resp`
`144`	`144`	`}`
	`145`	`+`
	`146`	`+ // check if the current cluster is not upgraded to the deployed`
	`147`	`+ // Operator version. If not, do not allow the command to complete`
	`148`	`+ if cl.Annotations[config.ANNOTATION_IS_UPGRADED] == config.ANNOTATIONS_FALSE {`
	`149`	`+ resp.Status.Code = msgs.Error`
	`150`	`+ resp.Status.Msg = cl.Name + msgs.UpgradeError`
	`151`	`+ return resp`
	`152`	`+ }`
	`153`	`+`
`145`	`154`	`clusterList.Items = append(clusterList.Items, cl)`
`146`	`155`	`}`
`147`	`156`	`}`