Skip to content

Commit

Permalink
Run installmanager binary in install container for fips compatibility
Browse files Browse the repository at this point in the history 
As a result of the openshift installer transitioning from rhel8 to rhel9
(openshift/installer#8196), running
openshift-install in the rhel8 backed hive container in order to install
a cluster in fips mode results in a fips incompatibility.

Create a seperate installmanager binary that runs the install-manager
command previously invoked by hiveutil. Build a rhel8 and rhel9 version
of hive, and copy both versions of installmanager to the installer
container. The directory struture of the provisioning pod is also
adjusted to support this change. Lastly, the installmanager binary
corresponding to the rhel version of the installer container.

Signed-off-by: Leah Leshchinsky <[email protected]>
  • Loading branch information
@lleshchi
lleshchi committed Jun 3, 2024
1 parent d7ead60 commit 2e34680
Show file tree
Hide file tree
Showing 9 changed files with 80 additions and 103 deletions.
32 changes: 16 additions & 16 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
FROM registry.ci.openshift.org/openshift/release:rhel-8-release-golang-1.21-openshift-4.16 as builder
FROM registry.ci.openshift.org/openshift/release:rhel-8-release-golang-1.21-openshift-4.16 as builder_rhel8
RUN mkdir -p /go/src/github.com/openshift/hive
WORKDIR /go/src/github.com/openshift/hive
COPY . .
RUN make build

FROM registry.ci.openshift.org/openshift/release:rhel-9-release-golang-1.21-openshift-4.16 as builder_rhel9
RUN mkdir -p /go/src/github.com/openshift/hive
WORKDIR /go/src/github.com/openshift/hive
COPY . .
Expand All @@ -16,10 +22,12 @@ RUN if ! rpm -q openssh-clients; then $DNF install -y openssh-clients && $DNF cl
# libvirt libraries required for running bare metal installer.
RUN if ! rpm -q libvirt-libs; then $DNF install -y libvirt-libs && $DNF clean all && rm -rf /var/cache/dnf/*; fi

COPY --from=builder /go/src/github.com/openshift/hive/bin/manager /opt/services/
COPY --from=builder /go/src/github.com/openshift/hive/bin/hiveadmission /opt/services/
COPY --from=builder /go/src/github.com/openshift/hive/bin/hiveutil /usr/bin
COPY --from=builder /go/src/github.com/openshift/hive/bin/operator /opt/services/hive-operator
COPY --from=builder_rhel8 /go/src/github.com/openshift/hive/bin/manager /opt/services/
COPY --from=builder_rhel8 /go/src/github.com/openshift/hive/bin/hiveadmission /opt/services/
COPY --from=builder_rhel8 /go/src/github.com/openshift/hive/bin/operator /opt/services/hive-operator

COPY --from=builder_rhel8 /go/src/github.com/openshift/hive/bin/hiveutil /usr/bin
COPY --from=builder_rhel9 /go/src/github.com/openshift/hive/bin/hiveutil /usr/bin/hiveutil.rhel9

# Hacks to allow writing known_hosts, homedir is / by default in OpenShift.
# Bare metal installs need to write to $HOME/.cache, and $HOME/.ssh for as long as
Expand All @@ -30,17 +38,9 @@ RUN mkdir -p /home/hive && \
chgrp -R 0 /home/hive && \
chmod -R g=u /home/hive

# This is so that we can write source certificate anchors during container start up.
RUN mkdir -p /etc/pki/ca-trust/source/anchors && \
chgrp -R 0 /etc/pki/ca-trust/source/anchors && \
chmod -R g=u /etc/pki/ca-trust/source/anchors

# This is so that we can run update-ca-trust during container start up.
RUN mkdir -p /etc/pki/ca-trust/extracted/openssl && \
mkdir -p /etc/pki/ca-trust/extracted/pem && \
mkdir -p /etc/pki/ca-trust/extracted/java && \
chgrp -R 0 /etc/pki/ca-trust/extracted && \
chmod -R g=u /etc/pki/ca-trust/extracted
RUN mkdir -p /output/hive-trusted-cabundle && \
chgrp -R 0 /output/hive-trusted-cabundle && \
chmod -R g=u /output/hive-trusted-cabundle

# TODO: should this be the operator?
ENTRYPOINT ["/opt/services/manager"]
32 changes: 16 additions & 16 deletions Dockerfile.ubi
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
FROM registry.ci.openshift.org/openshift/release:rhel-8-release-golang-1.21-openshift-4.16 as builder
FROM registry.ci.openshift.org/openshift/release:rhel-8-release-golang-1.21-openshift-4.16 as builder_rhel8
RUN mkdir -p /go/src/github.com/openshift/hive
WORKDIR /go/src/github.com/openshift/hive
COPY . .
RUN make build

FROM registry.ci.openshift.org/openshift/release:rhel-9-release-golang-1.21-openshift-4.16 as builder_rhel9
RUN mkdir -p /go/src/github.com/openshift/hive
WORKDIR /go/src/github.com/openshift/hive
COPY . .
Expand All @@ -19,10 +25,12 @@ RUN if ! rpm -q libvirt-libs; then $DNF install -y libvirt-libs && $DNF clean al
# tar is needed to package must-gathers on install failure
RUN if ! which tar; then $DNF install -y tar && $DNF clean all && rm -rf /var/cache/dnf/*; fi

COPY --from=builder /go/src/github.com/openshift/hive/bin/manager /opt/services/
COPY --from=builder /go/src/github.com/openshift/hive/bin/hiveadmission /opt/services/
COPY --from=builder /go/src/github.com/openshift/hive/bin/hiveutil /usr/bin
COPY --from=builder /go/src/github.com/openshift/hive/bin/operator /opt/services/hive-operator
COPY --from=builder_rhel8 /go/src/github.com/openshift/hive/bin/manager /opt/services/
COPY --from=builder_rhel8 /go/src/github.com/openshift/hive/bin/hiveadmission /opt/services/
COPY --from=builder_rhel8 /go/src/github.com/openshift/hive/bin/operator /opt/services/hive-operator

COPY --from=builder_rhel8 /go/src/github.com/openshift/hive/bin/hiveutil /usr/bin
COPY --from=builder_rhel9 /go/src/github.com/openshift/hive/bin/hiveutil /usr/bin/hiveutil.rhel9

# Hacks to allow writing known_hosts, homedir is / by default in OpenShift.
# Bare metal installs need to write to $HOME/.cache, and $HOME/.ssh for as long as
Expand All @@ -33,17 +41,9 @@ RUN mkdir -p /home/hive && \
chgrp -R 0 /home/hive && \
chmod -R g=u /home/hive

# This is so that we can write source certificate anchors during container start up.
RUN mkdir -p /etc/pki/ca-trust/source/anchors && \
chgrp -R 0 /etc/pki/ca-trust/source/anchors && \
chmod -R g=u /etc/pki/ca-trust/source/anchors

# This is so that we can run update-ca-trust during container start up.
RUN mkdir -p /etc/pki/ca-trust/extracted/openssl && \
mkdir -p /etc/pki/ca-trust/extracted/pem && \
mkdir -p /etc/pki/ca-trust/extracted/java && \
chgrp -R 0 /etc/pki/ca-trust/extracted && \
chmod -R g=u /etc/pki/ca-trust/extracted
RUN mkdir -p /output/hive-trusted-cabundle && \
chgrp -R 0 /output/hive-trusted-cabundle && \
chmod -R g=u /output/hive-trusted-cabundle

# TODO: should this be the operator?
ENTRYPOINT ["/opt/services/manager"]
8 changes: 1 addition & 7 deletions contrib/pkg/utils/generic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
)

const (
caTrustDir = "/etc/pki/ca-trust/source/anchors/"
caTrustDir = "/output/hive-trusted-cabundle/"
)

type releasePayload struct {
Expand Down Expand Up @@ -205,10 +205,4 @@ func InstallCerts(sourceDir string) {
logger.WithError(err).WithField("output", string(b)).Fatal("failed to copy certs")
}
logger.WithField("output", string(b)).Info("copied certs")

b, err = exec.Command("update-ca-trust").CombinedOutput()
if err != nil {
logger.WithError(err).WithField("output", string(b)).Fatal("failed to update CA trust")
}
logger.WithField("output", string(b)).Info("updated CA trust")
}
8 changes: 5 additions & 3 deletions hack/e2e-common.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,9 @@ function save_hive_logs() {
done
# Let's try to save any prov/deprov pod logs
oc get po -A -l hive.openshift.io/install=true -o custom-columns=:.metadata.namespace,:.metadata.name --no-headers | while read ns po; do
oc logs -n $ns $po -c hive > ${ARTIFACT_DIR}/${ns}-${po}.log
oc logs -n $ns $po -c hive > ${ARTIFACT_DIR}/${ns}-${po}-hive.log
oc logs -n $ns $po -c installer > ${ARTIFACT_DIR}/${ns}-${po}-installer.log

done
oc get po -A -l hive.openshift.io/uninstall=true -o custom-columns=:.metadata.namespace,:.metadata.name --no-headers | while read ns po; do
oc logs -n $ns $po > ${ARTIFACT_DIR}/${ns}-${po}.log
Expand Down Expand Up @@ -269,12 +271,12 @@ function capture_cluster_logs() {
# Capture install logs
if IMAGESET_JOB_NAME=$(oc get job -l "hive.openshift.io/cluster-deployment-name=${CLUSTER_NAME},hive.openshift.io/imageset=true" -o name -n ${CLUSTER_NAMESPACE}) && [ "${IMAGESET_JOB_NAME}" ]
then
oc logs -c hive -n ${CLUSTER_NAMESPACE} ${IMAGESET_JOB_NAME} &> "${ARTIFACT_DIR}/hive_imageset_job.log" || true
oc logs -c installer -n ${CLUSTER_NAMESPACE} ${IMAGESET_JOB_NAME} &> "${ARTIFACT_DIR}/hive_imageset_job.log" || true
oc get ${IMAGESET_JOB_NAME} -n ${CLUSTER_NAMESPACE} -o yaml &> "${ARTIFACT_DIR}/hive_imageset_job.yaml" || true
fi
if INSTALL_JOB_NAME=$(oc get job -l "hive.openshift.io/cluster-deployment-name=${CLUSTER_NAME},hive.openshift.io/install=true" -o name -n ${CLUSTER_NAMESPACE}) && [ "${INSTALL_JOB_NAME}" ]
then
oc logs -c hive -n ${CLUSTER_NAMESPACE} ${INSTALL_JOB_NAME} &> "${ARTIFACT_DIR}/hive_install_job.log" || true
oc logs -c installer -n ${CLUSTER_NAMESPACE} ${INSTALL_JOB_NAME} &> "${ARTIFACT_DIR}/hive_install_job.log" || true
oc get ${INSTALL_JOB_NAME} -n ${CLUSTER_NAMESPACE} -o yaml &> "${ARTIFACT_DIR}/hive_install_job.yaml" || true
fi
echo "************* INSTALL JOB LOG *************"
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/clusterdeployment/clusterdeployment_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -688,7 +688,7 @@ func TestClusterDeploymentReconcile(t *testing.T) {
if assert.Len(t, provisions, 1, "expected exactly one ClusterProvision") {
podSpec := provisions[0].Spec.PodSpec
if assert.Len(t, podSpec.InitContainers, 1, "expected exactly one initContainer") {
assert.Equal(t, "installer", podSpec.InitContainers[0].Name, "expected the initContainer to be 'installer'")
assert.Equal(t, "hive", podSpec.InitContainers[0].Name, "expected the initContainer to be 'hive'")
}
}
},
Expand Down
19 changes: 5 additions & 14 deletions pkg/controller/utils/credentials.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,7 @@ package utils

import (
"context"
"net/url"
"os"
"strings"
"time"

log "github.com/sirupsen/logrus"

Expand All @@ -14,10 +11,6 @@ import (

"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/vmware/govmomi/vapi/rest"
"github.com/vmware/govmomi/vim25"
"github.com/vmware/govmomi/vim25/soap"

hivev1 "github.com/openshift/hive/apis/hive/v1"
"github.com/openshift/hive/pkg/constants"
)
Expand Down Expand Up @@ -60,11 +53,9 @@ func ValidateCredentialsForClusterDeployment(kubeClient client.Client, cd *hivev

}

return validateVSphereCredentials(cd.Spec.Platform.VSphere.VCenter,
string(secret.Data[constants.UsernameSecretKey]),
string(secret.Data[constants.PasswordSecretKey]),
rootCAFiles,
logger)
// TODO determine whether any vsphere credential checks are necessary
// or if we can remove this function entirely
return true, nil
default:
// If we have no platform-specific credentials verification
// assume the creds are valid.
Expand Down Expand Up @@ -94,7 +85,7 @@ func createRootCAFiles(certificateSecret *corev1.Secret) ([]string, error) {
return fileList, nil
}

func validateVSphereCredentials(vcenter, username, password string, rootCAFiles []string, logger log.FieldLogger) (bool, error) {
/*func validateVSphereCredentials(vcenter, username, password string, rootCAFiles []string, logger log.FieldLogger) (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
Expand Down Expand Up @@ -132,7 +123,7 @@ func validateVSphereCredentials(vcenter, username, password string, rootCAFiles
}
return err == nil, nil
}
}*/

// getClusterPlatform returns the platform of a given ClusterDeployment
func getClusterPlatform(cd *hivev1.ClusterDeployment) string {
Expand Down
10 changes: 3 additions & 7 deletions pkg/imageset/updateinstaller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,13 +167,9 @@ func (o *UpdateInstallerImageOptions) Run() (returnErr error) {
o.log.WithField("installerImage", installerImage).Info("installer image overridden")
} else {
// Glean the installer image from the release metadata
installerTagName := "installer"
// If this is a bare metal install, we need to get the openshift-install binary from a different image with
// bare metal functionality compiled in. The binary is named the same and in the same location, so after swapping
// out what image to get it from, we can proceed with the code as we normally would.
if cd.Spec.Platform.BareMetal != nil {
installerTagName = "baremetal-installer"
}
// "baremetal-installer" is the legacy name for the fips compliant installer image
installerTagName := "baremetal-installer"

// Override annotation is allowed to override baremetal-installer too
if cd.Annotations != nil {
if override := cd.Annotations[constants.OverrideInstallerImageNameAnnotation]; override != "" {
Expand Down
37 changes: 12 additions & 25 deletions pkg/imageset/updateinstaller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,41 +47,28 @@ func TestUpdateInstallerImageCommand(t *testing.T) {
name: "successful execution",
existingClusterDeployment: testClusterDeployment(),
images: map[string]string{
"installer": testInstallerImage,
"cli": testCLIImage,
"baremetal-installer": testInstallerImage,
"cli": testCLIImage,
},
validateClusterDeployment: validateSuccessfulExecution(testInstallerImage, testCLIImage, ""),
},
{
name: "failure execution missing cli",
existingClusterDeployment: testClusterDeployment(),
images: map[string]string{
"installer": testInstallerImage,
"baremetal-installer": testInstallerImage,
},
validateClusterDeployment: validateFailureExecution("could not get cli image"),
expectError: true,
},
{
name: "successful execution after failure",
existingClusterDeployment: testClusterDeploymentWithErrorCondition(),
images: map[string]string{
"installer": testInstallerImage,
"cli": testCLIImage,
},
validateClusterDeployment: validateSuccessfulExecution(testInstallerImage, testCLIImage, installerImageResolvedReason),
},
{
name: "successful execution baremetal platform",
existingClusterDeployment: func() *hivev1.ClusterDeployment {
cd := testClusterDeployment()
cd.Spec.Platform.BareMetal = &baremetal.Platform{}
return cd
}(),
images: map[string]string{
"baremetal-installer": testInstallerImage,
"cli": testCLIImage,
},
validateClusterDeployment: validateSuccessfulExecution(testInstallerImage, testCLIImage, ""),
validateClusterDeployment: validateSuccessfulExecution(testInstallerImage, testCLIImage, installerImageResolvedReason),
},
{
name: "installer image name override",
Expand Down Expand Up @@ -118,8 +105,8 @@ func TestUpdateInstallerImageCommand(t *testing.T) {
name: "successful execution with version in release metadata",
existingClusterDeployment: testClusterDeployment(),
images: map[string]string{
"installer": testInstallerImage,
"cli": testCLIImage,
"baremetal-installer": testInstallerImage,
"cli": testCLIImage,
},
version: testReleaseVersion,
validateClusterDeployment: validateSuccessfulExecution(testInstallerImage, testCLIImage, ""),
Expand All @@ -137,17 +124,17 @@ func TestUpdateInstallerImageCommand(t *testing.T) {
name: "CLI image domain copied from installer image",
existingClusterDeployment: testClusterDeploymentWithCLIDomainCopy(),
images: map[string]string{
"installer": testInstallerImage,
"cli": cliImageWithDifferentDomain,
"baremetal-installer": testInstallerImage,
"cli": cliImageWithDifferentDomain,
},
validateClusterDeployment: validateSuccessfulExecution(testInstallerImage, "registry.io/foo/cli:blah", ""),
},
{
name: "copy requested, invalid installer image",
existingClusterDeployment: testClusterDeploymentWithCLIDomainCopy(),
images: map[string]string{
"installer": "invalid image",
"cli": cliImageWithDifferentDomain,
"baremetal-installer": "invalid image",
"cli": cliImageWithDifferentDomain,
},
validateClusterDeployment: validateFailureExecution("invalid installer image"),
expectError: true,
Expand All @@ -156,8 +143,8 @@ func TestUpdateInstallerImageCommand(t *testing.T) {
name: "copy requested, invalid cli image",
existingClusterDeployment: testClusterDeploymentWithCLIDomainCopy(),
images: map[string]string{
"installer": testInstallerImage,
"cli": "invalid image",
"baremetal-installer": testInstallerImage,
"cli": "invalid image",
},
validateClusterDeployment: validateFailureExecution("invalid cli image"),
expectError: true,
Expand Down
35 changes: 21 additions & 14 deletions pkg/install/generate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,14 @@ func InstallerPodSpec(
Name: "PULLSECRET_SECRET_NAME",
Value: constants.GetMergedPullSecretName(cd),
},
{
Name: "SSL_CERT_DIR",
Value: "/etc/ssl/certs/:/output/hive-trusted-cabundle/",
},
{
Name: "HOME",
Value: "/home/hive",
},
}

env = append(env, extraEnvVars...)
Expand All @@ -164,6 +172,7 @@ func InstallerPodSpec(
"logs": "/logs",
"installconfig": "/installconfig",
"pullsecret": "/pullsecret",
"hive": "/home/hive",
}

var credentialRef, certificateRef string
Expand Down Expand Up @@ -308,14 +317,14 @@ func InstallerPodSpec(
// where our container will run them. This is effectively downloading the all-in-one installer.
initContainers := []corev1.Container{
{
Name: "installer",
Image: installerImage,
ImagePullPolicy: corev1.PullIfNotPresent,
Name: "hive",
Image: images.GetHiveImage(),
ImagePullPolicy: images.GetHiveClusterProvisionImagePullPolicy(),
Env: env,
Command: []string{"/bin/sh", "-c"},
// Large file copy here has shown to cause problems in clusters under load, safer to copy then rename to the file the install manager is waiting for
// so it doesn't try to run a partially copied binary.
Args: []string{"cp -v /bin/openshift-install /output/openshift-install.tmp && mv -v /output/openshift-install.tmp /output/openshift-install && ls -la /output"},
Args: []string{"cp -v /usr/bin/hiveutil /output/hiveutil8.tmp && mv -v /output/hiveutil8.tmp /output/hiveutil.rhel8 && cp -v /usr/bin/hiveutil.rhel9 /output/hiveutil9.tmp && mv -v /output/hiveutil9.tmp /output/hiveutil.rhel9"},
VolumeMounts: volumeMounts,
},
}
Expand All @@ -333,19 +342,17 @@ func InstallerPodSpec(
VolumeMounts: volumeMounts,
})
}

containers := []corev1.Container{
{
Name: "hive",
Image: images.GetHiveImage(),
ImagePullPolicy: images.GetHiveClusterProvisionImagePullPolicy(),
Name: "installer",
Image: installerImage,
ImagePullPolicy: corev1.PullIfNotPresent,
Env: append(env, cd.Spec.Provisioning.InstallerEnv...),
Command: []string{"/usr/bin/hiveutil"},
Args: []string{
"install-manager",
"--work-dir", "/output",
"--log-level", "debug",
cd.Namespace, provisionName,
},
Command: []string{"/bin/sh", "-c"},
// Large file copy here has shown to cause problems in clusters under load, safer to copy then rename to the file the install manager is waiting for
// so it doesn't try to run a partially copied binary.
Args: []string{fmt.Sprintf("cp -v /bin/openshift-install /output/openshift-install && major_version=$(sed -n 's/.*release \\([0-9]*\\).*/\\1/p' /etc/redhat-release) && /output/hiveutil.rhel${major_version} install-manager --work-dir /output --log-level debug %s %s", cd.Namespace, provisionName)},
VolumeMounts: volumeMounts,
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
Expand Down

0 comments on commit 2e34680

Please sign in to comment.