[WIP] CNTRLPLANE-252: Build CI to cover cilium CNI test for public HCP on A…

ci-operator/config/openshift/hypershift/openshift-hypershift-release-4.18__periodics.yaml

@@ -103,6 +103,11 @@ tests:
   steps:
     cluster_profile: hypershift
     workflow: hypershift-aws-conformance-proxy
+- as: e2e-aws-conformance-cilium
+  minimum_interval: 168h
+  steps:
+    cluster_profile: hypershift
+    workflow: hypershift-aws-conformance-cilium
 - as: e2e-powervs-ovn
   cron: 0 8 * * *
   steps:

ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-4.18-periodics.yaml

@@ -74,6 +74,81 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build11
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.18
+    org: openshift
+    repo: hypershift
+  labels:
+    ci-operator.openshift.io/cloud: hypershift
+    ci-operator.openshift.io/cloud-cluster-profile: hypershift
+    ci-operator.openshift.io/variant: periodics
+    ci.openshift.io/generator: prowgen
+    job-release: "4.18"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  minimum_interval: 168h
+  name: periodic-ci-openshift-hypershift-release-4.18-periodics-e2e-aws-conformance-cilium
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=e2e-aws-conformance-cilium
+      - --variant=periodics
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build11
   cron: 0 2 * * *

ci-operator/step-registry/hypershift/aws/conformance-cilium/OWNERS

@@ -0,0 +1 @@
+../../OWNERS

ci-operator/step-registry/hypershift/aws/conformance-cilium/hypershift-aws-conformance-cilium-workflow.metadata.json

@@ -0,0 +1,23 @@
+{
+	"path": "hypershift/aws/conformance-cilium/hypershift-aws-conformance-cilium-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"csrwng",
+			"enxebre",
+			"sjenning",
+			"imain",
+			"davidvossel",
+			"LiangquanLi930",
+			"Patryk-Stefanski",
+			"bryan-cox"
+		],
+		"reviewers": [
+			"csrwng",
+			"enxebre",
+			"sjenning",
+			"imain",
+			"Patryk-Stefanski",
+			"bryan-cox"
+		]
+	}
+}

ci-operator/step-registry/hypershift/aws/conformance-cilium/hypershift-aws-conformance-cilium-workflow.yaml

@@ -0,0 +1,40 @@
+workflow:
+  as: hypershift-aws-conformance-cilium
+  documentation: |-
+    The HyperShift aws conformance workflow executes tests against a new ephemeral
+    HyperShift cluster. Administrative access to the control plane is provided
+    via the `KUBECONFIG` environment variable.
+
+    Note that tests will execute once the control plane is ready, regardless of
+    whether nodes have been requested, have joined, or are available.
+
+    The control plane is hosted on a dedicated standalone OCP cluster running
+    HyperShift in the `osd-hypershift` AWS account and is destroyed after tests
+    complete.
+
+    The HyperShift launch capability is currently supported by the HyperShift
+    team. For now, please direct all questions and comments to:
+
+    - Alberto Lamela ([email protected])
+    - Seth Jennings ([email protected])
+    - Dan Mace ([email protected])
+    - Cesar Wong ([email protected])
+
+    Learn more about HyperShift here: https://github.com/openshift/hypershift
+
+    Track HyperShift's development here: https://issues.redhat.com/projects/HOSTEDCP
+  steps:
+    env:
+      HYPERSHIFT_NETWORK_TYPE: "Other" # Required for Cilium.
+    post:
+    - chain: hypershift-dump
+    - chain: hypershift-aws-destroy
+    - chain: hypershift-destroy-nested-management-cluster
+    test:
+    - chain: hypershift-conformance
+    pre:
+    - ref: ipi-install-rbac
+    - chain: hypershift-setup-nested-management-cluster
+    - ref: hypershift-install
+    - chain: hypershift-aws-create
+    - ref: cucushift-hypershift-extended-cilium

ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml

@@ -11,6 +11,9 @@ chain:
     - name: HYPERSHIFT_NODE_COUNT
       default: "3"
       documentation: "The number nodes to automatically create and join to the cluster."
+    - name: HYPERSHIFT_NETWORK_TYPE
+      default: ""
+      documentation: "Specifies the cluster SDN provider."
     - name: HYPERSHIFT_BASE_DOMAIN
       default: ""
       documentation: "The cluster's FQDN will be a subdomain of the base domain."
@@ -115,6 +118,10 @@ chain:
         COMMAND+=(--node-upgrade-type=InPlace)
       fi
 
+      if [[ -n $HYPERSHIFT_NETWORK_TYPE ]]; then
+        COMMAND+=(--network-type="${HYPERSHIFT_NETWORK_TYPE}")
+      fi
+
       if [[ -n $EXTRA_ARGS ]]; then
         COMMAND+=(${EXTRA_ARGS})
       fi
@@ -137,13 +144,39 @@ chain:
 
       "${COMMAND[@]}"
 
-      set +e
-      export CLUSTER_NAME
-      timeout 25m bash -c '
-        until [[ "$(oc get -n clusters hostedcluster/${CLUSTER_NAME} -o jsonpath='"'"'{.status.version.history[?(@.state!="")].state}'"'"')" = "Completed" ]]; do
-            sleep 15
-        done
-      '
+      echo "Waiting for cluster to become available"
+      oc wait --timeout=30m --for=condition=Available --namespace=clusters hostedcluster/${CLUSTER_NAME}
+      echo "Cluster became available, creating kubeconfig"
+      bin/hypershift create kubeconfig --namespace=clusters --name=${CLUSTER_NAME} >${SHARED_DIR}/nested_kubeconfig
+      echo "${CLUSTER_NAME}" > "${SHARED_DIR}/cluster-name"
+
+      # In case of BYO CNI, the HC will never be ready before CNI components are up and running
+      if [[ "$HYPERSHIFT_NETWORK_TYPE" == "Other" || "$HYPERSHIFT_NETWORK_TYPE" == "Calico" ]]; then
+        echo "Waiting for nodes to join the hosted cluster"
+        num_nodes_expected=$(oc get np -A -o jsonpath='{.items[0].spec.replicas}')
+        export num_nodes_expected
+        export KUBECONFIG=${SHARED_DIR}/nested_kubeconfig
+        set +e
+        timeout 25m bash -c '
+          while true; do
+            num_nodes=$(oc get nodes --no-headers | wc -l | xargs)
+            if (( num_nodes == num_nodes_expected )); then
+              break
+            fi
+            echo "$num_nodes/$num_nodes_expected have joined the HC"
+            sleep 15s
+          done
+        '
+      else
+        set +e
+        export CLUSTER_NAME
+        timeout 25m bash -c '
+          until [[ "$(oc get -n clusters hostedcluster/${CLUSTER_NAME} -o jsonpath='"'"'{.status.version.history[?(@.state!="")].state}'"'"')" = "Completed" ]]; do
+              sleep 15
+          done
+        '
+      fi
+
       if [[ $? -ne 0 ]]; then
         cat << EOF > ${ARTIFACT_DIR}/junit_hosted_cluster.xml
       <?xml version="1.0" encoding="UTF-8"?>
@@ -175,10 +208,6 @@ chain:
       EOF
       fi
       set -e
-
-      echo "Cluster became available, creating kubeconfig"
-      bin/hypershift create kubeconfig --namespace=clusters --name=${CLUSTER_NAME} >${SHARED_DIR}/nested_kubeconfig
-      echo "${CLUSTER_NAME}" > "${SHARED_DIR}/cluster-name"
     from: hypershift-operator
     grace_period: 5m0s
     resources: