Replaces SSH with kubectl for communicating between

andrewlecuyer · andrewlecuyer · commit 11d34b68d56b · 2019-11-12T00:41:16.000-06:00
the pgBackRest dedicated repository host and the primary
PG database.  Specifically, a script has been created that
translates any SSH commands issued by pgBackRest into
the proper kubectl command.  pgBackRest is then
configured to call this script when SSH is required, instead
of calling SSH directly.  This allows kubectl to be utilized
as a drop-in replacement for SSH, and therefore prevents
the need to install and run SSH on any containers created
by the PGO (for instance, SSH no longer needs to be
enabled on the dedicated repository host nor the PG
primary pod).

However, it should be noted that SSH can still be enabled
in any backrest container if necessary while the kubectl
solution is fully verified and validated.  Therefore, at this
time SSH is still installed installed in any backrest containers.
Additionally, kubectl and the associated yum repo are now
installed in any backrest containers to support the use of
kubectl instead of SSH.
diff --git a/bin/pgo-backrest-repo/pgo-backrest-repo.sh b/bin/pgo-backrest-repo/pgo-backrest-repo.sh
@@ -14,21 +14,21 @@
 # limitations under the License.
 
 
-function trap_sigterm() {
+function trap_sigterm_tail() {
 	echo "Signal trap triggered, beginning shutdown.." 
-	killall sshd
+	killall tail
 }
 
-trap 'trap_sigterm' SIGINT SIGTERM
+function trap_sigterm_sshd() {
+	echo "Signal trap triggered, beginning shutdown.." 
+	killall sshd
+}
 
-CONFIG=/sshd
 REPO=/backrestrepo
 
 echo "PGBACKREST env vars are set to:"
 set | grep PGBACKREST
 
-echo "CONFIG is.."
-ls $CONFIG
 echo "REPO is ..."
 ls $REPO
 
@@ -37,14 +37,32 @@ if [ ! -d $PGBACKREST_REPO_PATH ]; then
 	mkdir -p $PGBACKREST_REPO_PATH
 fi
 
-mkdir ~/.ssh/
-cp $CONFIG/config ~/.ssh/
-#cp $CONFIG/authorized_keys ~/.ssh/
-cp $CONFIG/id_rsa /tmp
-chmod 400 /tmp/id_rsa ~/.ssh/config
+# save a copy of certain pod env vars for pgbackrest ssh cmd wrapper
+env | grep "^KUBERNETES" | sed "s/^/export /" >> "/tmp/pod_env.sh"
+env | grep "^CLUSTER_NAME" | sed "s/^/export /" >> "/tmp/pod_env.sh"
+
+if [[ "${ENABLE_SSHD}" == "true" ]]
+then
+	trap 'trap_sigterm_sshd' SIGINT SIGTERM
 
-# start sshd which is used by pgbackrest for remote connections
-/usr/sbin/sshd -D -f $CONFIG/sshd_config   &
+	CONFIG=/sshd
+	echo "SSHD CONFIG is.."
+	ls $CONFIG
+
+	mkdir ~/.ssh/
+    cp $CONFIG/config ~/.ssh/
+    #cp $CONFIG/authorized_keys ~/.ssh/
+    cp $CONFIG/id_rsa /tmp
+    chmod 400 /tmp/id_rsa ~/.ssh/config
+
+    # start sshd which is used by pgbackrest for remote connections
+    /usr/sbin/sshd -D -f $CONFIG/sshd_config   &
+else
+	trap 'trap_sigterm_tail' SIGINT SIGTERM
+
+	# start a random process to keep the container running
+    tail -f /dev/null
+fi
 
 wait
 
diff --git a/centos7/Dockerfile.pgo-backrest-repo.centos7 b/centos7/Dockerfile.pgo-backrest-repo.centos7
@@ -11,6 +11,15 @@ ENV PGVERSION="11" PGDG_REPO="pgdg-redhat-repo-latest.noarch.rpm" PGDG_REPO_DISA
 # PGDG PostgreSQL Repository
 RUN rpm -Uvh https://download.postgresql.org/pub/repos/yum/${PGVERSION}/redhat/rhel-7-x86_64/${PGDG_REPO}
 
+# Kubernetes repository
+RUN echo $'[kubernetes] \n\
+name=Kubernetes \n\
+baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 \n\
+enabled=1 \n\
+gpgcheck=1 \n\
+repo_gpgcheck=1 \n\
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg' > /etc/yum.repos.d/kubernetes.repo
+
 RUN yum -y install \
 	--disablerepo="${PGDG_REPO_DISABLE}" \
 	--enablerepo="${PGDG_REPO_ENABLE}" \
@@ -20,6 +29,7 @@ RUN yum -y install \
     pgbackrest-"${BACKREST_VERSION}" \
     procps-ng \
     psmisc \
+    kubectl \
     && yum -y clean all
 
 RUN groupadd pgbackrest -g 2000 && useradd pgbackrest -u 2000 -g 2000
@@ -29,6 +39,7 @@ RUN chmod +x /usr/local/bin/pgo-backrest-repo.sh /usr/local/bin/archive-push-s3.
     && chown -R pgbackrest:pgbackrest /opt/cpm
 
 ADD bin/uid_pgbackrest.sh /opt/cpm/bin
+ADD bin/kube-ssh-wrapper.sh /opt/cpm/bin
 
 RUN chmod g=u /etc/passwd && \
         chmod g=u /etc/group
diff --git a/centos7/Dockerfile.pgo-backrest-restore.centos7 b/centos7/Dockerfile.pgo-backrest-restore.centos7
@@ -10,6 +10,15 @@ ENV PGVERSION="11" PGDG_REPO="pgdg-redhat-repo-latest.noarch.rpm" PGDG_REPO_DISA
 
 RUN rpm -Uvh https://download.postgresql.org/pub/repos/yum/${PGVERSION}/redhat/rhel-7-x86_64/${PGDG_REPO}
 
+# Kubernetes repository
+RUN echo $'[kubernetes] \n\
+name=Kubernetes \n\
+baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 \n\
+enabled=1 \n\
+gpgcheck=1 \n\
+repo_gpgcheck=1 \n\
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg' > /etc/yum.repos.d/kubernetes.repo
+
 RUN yum -y install \
 	--disablerepo="${PGDG_REPO_DISABLE}" \
 	--enablerepo="${PGDG_REPO_ENABLE}" \
@@ -19,11 +28,13 @@ RUN yum -y install \
     postgresql11-server \
     procps-ng \
     psmisc \
+    kubectl \
     && yum -y clean all
 
 RUN mkdir -p /opt/cpm/bin /pgdata && chown -R 26:26 /opt/cpm
 ADD bin/pgo-backrest-restore/ /opt/cpm/bin
 ADD bin/uid_postgres.sh /opt/cpm/bin
+ADD bin/kube-ssh-wrapper.sh /opt/cpm/bin
 
 RUN chmod g=u /etc/passwd && \
         chmod g=u /etc/group
diff --git a/conf/postgres-operator/backrest-restore-job.json b/conf/postgres-operator/backrest-restore-job.json
@@ -77,6 +77,19 @@
                     }, {
                         "name": "PGBACKREST_LOG_PATH",
                         "value": "/tmp"
+                    }, {
+                        "name": "PGBACKREST_LOG_LEVEL_CONSOLE",
+                        "value": "warn"
+                    }, {
+                        "name": "PGBACKREST_CMD_SSH",
+                        "value": "ssh"
+                    }, {
+                        "name": "CLUSTER_NAME",
+                        "valueFrom": {
+                            "fieldRef": {
+                                "fieldPath": "metadata.labels['pg-cluster']"
+                            }
+                        }
                     }, {
                         "name": "NAMESPACE",
                         "valueFrom": {
diff --git a/conf/postgres-operator/cluster-deployment.json b/conf/postgres-operator/cluster-deployment.json
@@ -84,6 +84,9 @@
             {{.PgbackrestS3EnvVars}}
                         "name": "PGHA_DATABASE",
                         "value": "{{.Database}}"
+                    }, {
+                        "name": "PGHA_CRUNCHYADM",
+                        "value": "true"
                     }, {
                         "name": "PATRONI_KUBERNETES_NAMESPACE",
                         "valueFrom": {
diff --git a/conf/postgres-operator/pgo-backrest-repo-template.json b/conf/postgres-operator/pgo-backrest-repo-template.json
@@ -34,7 +34,7 @@
             },
             "spec": {
 		{{.SecurityContext}}
-
+                "serviceAccountName": "pgo-backrest",
                 "containers": [{
                     "name": "database",
                     "image": "{{.PGOImagePrefix}}/pgo-backrest-repo:{{.PGOImageTag}}",
@@ -68,6 +68,25 @@
                     }, {
                         "name": "PGBACKREST_DB_HOST",
                         "value": "{{.PGbackrestDBHost}}"
+                    }, {
+                        "name": "PGBACKREST_CMD_SSH",
+                        "value": "ssh"
+                    }, {
+                        "name": "PGBACKREST_LOG_LEVEL_CONSOLE",
+                        "value": "warn"
+                    }, {
+                        "name": "CLUSTER_NAME",
+                        "valueFrom": {
+                            "fieldRef": {
+                                "fieldPath": "metadata.labels['pg-cluster']"
+                            }
+                        }
+                    }, {
+                        "name": "PGO_BACKREST_REPO",
+                        "value": "true"
+                    }, {
+                        "name": "ENABLE_SSHD",
+                        "value": "true"
                     }],
                     "volumeMounts": [{
                         "name": "sshd",
diff --git a/conf/postgres-operator/pgo-pg-role.json b/conf/postgres-operator/pgo-pg-role.json
@@ -27,6 +27,17 @@
             "deletecollection"
          ]
       },
+      {
+         "apiGroups":[
+            ""
+         ],
+         "resources":[
+            "pods/exec"
+         ],
+         "verbs":[
+            "create"
+         ]
+      },
       {
          "apiGroups":[
             ""
diff --git a/rhel7/Dockerfile.pgo-backrest-repo.rhel7 b/rhel7/Dockerfile.pgo-backrest-repo.rhel7
@@ -12,13 +12,23 @@ ENV PGVERSION="11" BACKREST_VERSION="2.18"
 COPY redhat/atomic/pgo_backrest_repo/help.1 /help.1
 COPY redhat/atomic/pgo_backrest_repo/help.md /help.md
 
+# Kubernetes repository
+RUN echo $'[kubernetes] \n\
+name=Kubernetes \n\
+baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 \n\
+enabled=1 \n\
+gpgcheck=1 \n\
+repo_gpgcheck=1 \n\
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg' > /etc/yum.repos.d/kubernetes.repo
+
 RUN yum -y install \
 	crunchy-backrest-"${BACKREST_VERSION}" \
 	hostname \
 	openssh-clients \
 	openssh-server \
 	procps-ng \
 	psmisc \
+	kubectl \
 	&& yum -y clean all
 
 RUN groupadd pgbackrest -g 2000 && useradd pgbackrest -u 2000 -g 2000
@@ -28,6 +38,7 @@ RUN chmod +x /usr/local/bin/pgo-backrest-repo.sh /usr/local/bin/archive-push-s3.
     && chown -R pgbackrest:pgbackrest /opt/cpm
 
 ADD bin/uid_pgbackrest.sh /opt/cpm/bin
+ADD bin/kube-ssh-wrapper.sh /opt/cpm/bin
 
 RUN chmod g=u /etc/passwd && \
         chmod g=u /etc/group
diff --git a/rhel7/Dockerfile.pgo-backrest-restore.rhel7 b/rhel7/Dockerfile.pgo-backrest-restore.rhel7
@@ -12,18 +12,28 @@ ENV PGVERSION="11" BACKREST_VERSION="2.18"
 COPY redhat/atomic/pgo_backrest_restore/help.1 /help.1
 COPY redhat/atomic/pgo_backrest_restore/help.md /help.md
 
+RUN echo $'[kubernetes] \n\
+name=Kubernetes \n\
+baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 \n\
+enabled=1 \n\
+gpgcheck=1 \n\
+repo_gpgcheck=1 \n\
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg' > /etc/yum.repos.d/kubernetes.repo
+
 RUN yum -y install \
 	crunchy-backrest-"${BACKREST_VERSION}" \
 	openssh-clients \
 	openssh-server \
 	postgresql11-server \
 	procps-ng \
 	psmisc \
+	kubectl \
 	&& yum -y clean all
 
 RUN mkdir -p /opt/cpm/bin /pgdata && chown -R 26:26 /opt/cpm
 ADD bin/pgo-backrest-restore/ /opt/cpm/bin
 ADD bin/uid_postgres.sh /opt/cpm/bin
+ADD bin/kube-ssh-wrapper.sh /opt/cpm/bin
 
 RUN chmod g=u /etc/passwd && \
         chmod g=u /etc/group
