Support installing specific version of docker

Change-Id: I12015c28f6f8ffc125097a14514a6a90a20cf35b (cherry picked from commit f8e786f0d5)
Support config image repository for kubeadm
2023-06-12 02:29:59 +00:00 · 2022-03-22 04:41:34 +00:00 · 2022-03-11 14:35:04 +00:00
6 changed files with 40 additions and 131 deletions
--- a/.gitreview
+++ b/.gitreview
@@ -2,4 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/devstack-plugin-container.git
-defaultbranch=stable/2025.2
+defaultbranch=stable/yoga
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -7,21 +7,25 @@
    timeout: 4200
    required-projects:
      - openstack/devstack
+      - openstack/devstack-gate
      - openstack/devstack-plugin-container
    vars:
+      devstack_localrc:
+        USE_PYTHON3: true
      devstack_plugins:
        devstack-plugin-container: https://opendev.org/openstack/devstack-plugin-container

 - job:
    name: devstack-plugin-container-k8s
    parent: devstack-minimal
-    nodeset: openstack-two-node-noble
+    nodeset: openstack-two-node-focal
    pre-run: playbooks/devstack-plugin-container-k8s/pre.yaml
    run: playbooks/devstack-plugin-container-k8s/run.yaml
    post-run: playbooks/devstack-plugin-container-k8s/post.yaml
    timeout: 7200
    required-projects:
      - openstack/devstack
+      - openstack/devstack-gate
      - openstack/devstack-plugin-container
    vars:
      devstack_services:
@@ -32,6 +36,7 @@
        k8s-master: true
      devstack_localrc:
        K8S_TOKEN: "9agf12.zsu5uh2m4pzt3qba"
+        USE_PYTHON3: true
      devstack_plugins:
        devstack-plugin-container: https://opendev.org/openstack/devstack-plugin-container
    group-vars:
@@ -43,6 +48,7 @@
          k8s-node: true
        devstack_localrc:
          K8S_TOKEN: "9agf12.zsu5uh2m4pzt3qba"
+          USE_PYTHON3: true

 - project:
    check:
--- a/devstack/lib/crio
+++ b/devstack/lib/crio
@@ -27,7 +27,7 @@ CRIO_ALLOW_ICMP=$(trueorfalse True CRIO_ALLOW_ICMP)

 function check_crio {
    if is_ubuntu; then
-        dpkg -l | grep cri-o > /dev/null 2>&1
+        dpkg -l | grep crio-o > /dev/null 2>&1
    else
        false
        # TODO: CentOS/Fedora support.
@@ -40,22 +40,23 @@ function install_crio {
    fi

    local lsb_dist=${os_VENDOR,,}
+    local dist_version=${os_CODENAME}
+    local kubic_obs_project_key="2472d6d0d2f66af87aba8da34d64390375060aa4"
+    local os="x${os_VENDOR}_${os_RELEASE}"
    if is_ubuntu; then
-        local stream="https://pkgs.k8s.io/addons:/cri-o:/stable:/v${CRIO_VERSION%.*}"
-        local key_path="/etc/apt/keyrings/cri-o-apt-keyring.gpg"
-
        apt_get install apt-transport-https ca-certificates \
-            software-properties-common curl
-        curl -fsSL "${stream}/deb/Release.key" | sudo gpg --dearmor -o "${key_path}"
-        echo "deb [signed-by=${key_path}] ${stream}/deb/ /" | \
-            sudo tee /etc/apt/sources.list.d/cri-o.list
+            software-properties-common
+        sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 \
+            --recv ${kubic_obs_project_key}
+        sudo apt-add-repository "deb https://download.opensuse.org/"`
+            `"repositories/devel:/kubic:/libcontainers:/stable/${os}/ /"
+        sudo apt-add-repository "deb http://download.opensuse.org/"`
+            `"repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/"`
+            `"${CRIO_VERSION}/${os}/ /"

        # Installing podman and containerd will get us compatible versions of
-        # cri-o. And we need podman to manage container images anyway.
-        REPOS_UPDATED=False apt_get_update
-        crio_pkg_version=$(sudo apt-cache show cri-o | grep "Version: $CRIO_VERSION-" | awk '{ print $2 }' | head -n 1)
-        apt_get install podman buildah cri-o="${crio_pkg_version}"
-        sudo systemctl enable crio
+        # cri-o and runc. And we need podman to manage container images anyway.
+        apt_get install podman buildah cri-o-runc cri-o
    elif is_fedora; then
        if [[ "$lsb_dist" = "centos" ]]; then
            sudo yum-config-manager \
@@ -65,18 +66,6 @@ function install_crio {
                --add-repo \
                https://cbs.centos.org/repos/paas7-crio-311-candidate/x86_64/os/
        fi
-        if [[ "${os_VENDOR}" == *'Stream' ]]; then
-                    local stream="_Stream"
-        fi
-        # NOTE: All crio versions are not supported for Centos 8 stream
-        # because crio rpm is not present for some minor versions
-        sudo yum-config-manager \
-            --add-repo \
-            "https://download.opensuse.org/repositories/"`
-            `"devel:/kubic:/libcontainers:/stable:/cri-o:/${CRIO_VERSION}/"`
-            `"CentOS_${os_RELEASE}${stream}/"`
-            `"devel:kubic:libcontainers:stable:cri-o:${CRIO_VERSION}.repo"
-
        yum_install cri-o podman buildah
    fi
 }
@@ -89,45 +78,21 @@ function configure_crio {

    # We're wrapping values in \"<val>\" because that's the format cri-o wants.
    iniset -sudo ${CRIO_CONF} crio.api listen \"${CRIO_ENGINE_SOCKET_FILE}\"
-    iniset -sudo ${CRIO_CONF} crio.image pause_image \"${CRIO_PAUSE_IMAGE}\"
-    iniset -sudo ${CRIO_CONF} crio.image pause_command \"${CRIO_PAUSE_COMMAND}\"
    if [[ "$ENABLE_DEBUG_LOG_LEVEL" == "True" ]]; then
        # debug is way too verbose, info will be enough
        iniset -sudo ${CRIO_CONF} crio.runtime log_level \"info\"
    fi
    if is_ubuntu; then
-        local crio_minor=${CRIO_VERSION#*.}
        # At least for 18.04 we need to set up /etc/containers/registries.conf
        # with some initial content. That's another bug with that PPA.
        local registries_conf
        registries_conf="/etc/containers/registries.conf"
-
-        if [[ ! -f ${registries_conf} && $crio_minor -lt 24 ]]; then
+        if [[ ! -f ${registries_conf} ]]; then
            sudo mkdir -p `dirname ${registries_conf}`
            cat << EOF | sudo tee ${registries_conf}
 [registries.search]
 registries = ['docker.io']
 EOF
-        else
-            # If there is a config file, that means, we are probably on the
-            # newer version of crio/container/podman, which basically means
-            # we cannot mix [registries.search] registries filled with
-            # something and unqualified-search-registries setting which appear
-            # on sysregistry v2 config syntax. And because it's a TOML now, we
-            # cannot rely on iniset, but directly change the file.
-
-            local rname='unqualified-search-registries'
-            local rval='["docker.io", "quay.io"]'
-            if [[ ! -f ${registries_conf} ]]; then
-                cat << EOF | sudo tee ${registries_conf}
-unqualified-search-registries = ["docker.io", "quay.io"]
-EOF
-            elif grep -wq "^${rname}" "${registries_conf}"; then
-                sudo sed -i -e \
-                    "s/^${rname}.*$/${rname} = ${rval}/" "${registries_conf}"
-            else
-                sudo sed -i "1s/^/${rname} = ${rval}\n/" "${registries_conf}"
-            fi
        fi
        # CRI-O from kubic repo have placed runc in different place, not even
        # in path, just to not conflict with runc package from official repo.
@@ -146,7 +111,7 @@ EOF
        # By default CRI-O doesn't allow ICMP between containers, although it
        # is ususally expected for testing purposes.
        if [ "${CRIO_ALLOW_ICMP}" == "True" ]; then
-            if grep -wq '^default_sysctls' ${CRIO_CONF}; then
+            if grep -q 'default_sysctls =' ${CRIO_CONF}; then
                export CRIO_KEY="default_sysctls"
                export CRIO_VAL='[ "net.ipv4.ping_group_range=0 2147483647", ]'
                _update_config
--- a/devstack/lib/docker
+++ b/devstack/lib/docker
@@ -24,8 +24,7 @@ set +o xtrace
 DOCKER_ENGINE_SOCKET_FILE=${DOCKER_ENGINE_SOCKET_FILE:-/var/run/docker.sock}
 DOCKER_ENGINE_PORT=${DOCKER_ENGINE_PORT:-2375}
 DOCKER_CLUSTER_STORE=${DOCKER_CLUSTER_STORE:-}
-STACK_GROUP="$( id --group --name "$STACK_USER" )"
-DOCKER_GROUP=${DOCKER_GROUP:-$STACK_GROUP}
+DOCKER_GROUP=${DOCKER_GROUP:-$STACK_USER}
 DOCKER_CGROUP_DRIVER=${DOCKER_CGROUP_DRIVER:-}
 # TODO(hongbin): deprecate and remove clear container
 ENABLE_CLEAR_CONTAINER=$(trueorfalse False ENABLE_CLEAR_CONTAINER)
@@ -57,12 +56,9 @@ function install_docker {

    local lsb_dist=${os_VENDOR,,}
    local dist_version=${os_CODENAME}
-    if [[ "$lsb_dist" != "centosstream" ]]; then
-        local arch
-        arch=$(dpkg --print-architecture)
-    fi
+    local arch
+    arch=$(dpkg --print-architecture)
    if is_ubuntu; then
-        apt_get install apparmor
        if [[ ${dist_version} == 'trusty' ]]; then
            if uname -r | grep -q -- '-generic' && dpkg -l 'linux-image-*-generic' | grep -qE '^ii|^hi' 2>/dev/null; then
                apt_get install linux-image-extra-$(uname -r) linux-image-extra-virtual
@@ -87,17 +83,6 @@ function install_docker {
            sudo yum-config-manager \
                --add-repo \
                https://download.docker.com/linux/centos/docker-ce.repo
-        elif  [[ "$lsb_dist" = "centosstream" ]]; then
-            sudo yum-config-manager \
-                --add-repo \
-                https://download.docker.com/linux/centos/docker-ce.repo
-            sudo yum-config-manager \
-                --add-repo \
-                https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 #noqa
-            sudo yum-config-manager \
-                --enable \
-                packages.cloud.google.com_yum_repos_kubernetes-el7-x86_64
-            sudo dnf -y install kubeadm --nogpgcheck
        elif [[ "$lsb_dist" = "fedora" ]]; then
            sudo dnf config-manager \
                --add-repo \
@@ -236,7 +221,7 @@ ExecStart=/usr/bin/dockerd --config-file=$docker_config_file
 Environment="HTTP_PROXY=$http_proxy" "HTTPS_PROXY=$https_proxy" "NO_PROXY=$no_proxy"
 EOF
    sudo systemctl daemon-reload
-    sudo systemctl restart docker.service
+    sudo systemctl --no-block restart docker.service
 }

 function configure_containerd {
--- a/devstack/lib/k8s
+++ b/devstack/lib/k8s
@@ -27,7 +27,7 @@ K8S_NODE_IP=${K8S_NODE_IP:-$HOST_IP}
 K8S_API_SERVER_PORT=${K8S_API_SERVER_PORT:-6443}
 K8S_POD_NETWORK_CIDR=${K8S_POD_NETWORK_CIDR:-10.244.0.0/16}
 K8S_SERVICE_NETWORK_CIDR=${K8S_SERVICE_NETWORK_CIDR:-10.96.0.0/12}
-K8S_VERSION=${K8S_VERSION:-"1.30.5"}
+K8S_VERSION=${K8S_VERSION:-1.19.0-00}
 K8S_NETWORK_ADDON=${K8S_NETWORK_ADDON:-flannel}

 # Functions
@@ -40,17 +40,12 @@ function is_k8s_enabled {

 function install_kubeadm {
    if is_ubuntu; then
-        local stream="https://pkgs.k8s.io/core:/stable:/v${K8S_VERSION%.*}"
-        local key_path="/etc/apt/keyrings/kubernetes-apt-keyring.gpg"
-
-        apt_get install apt-transport-https ca-certificates curl gpg
-        curl -fsSL "${stream}/deb/Release.key" | sudo gpg --dearmor -o "${key_path}"
-        echo "deb [signed-by=${key_path}] ${stream}/deb/ /" | \
-            sudo tee /etc/apt/sources.list.d/kubernetes.list
-
+        apt_get install apt-transport-https curl
+        curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+        sudo add-apt-repository -y \
+            "deb https://apt.kubernetes.io/ kubernetes-xenial main"
        REPOS_UPDATED=False apt_get_update
-        kube_pkg_version=$(sudo apt-cache show kubeadm | grep "Version: $K8S_VERSION-" | awk '{ print $2 }' | head -n 1)
-        apt_get install kubelet="${kube_pkg_version}" kubeadm="${kube_pkg_version}" kubectl="${kube_pkg_version}"
+        apt_get install kubelet=$K8S_VERSION kubeadm=$K8S_VERSION kubectl=$K8S_VERSION
        sudo apt-mark hold kubelet kubeadm kubectl
        # NOTE(hongbin): This work-around an issue that kubelet pick a wrong
        # IP address if the node has multiple network interfaces.
@@ -65,18 +60,8 @@ function install_kubeadm {
 function kubeadm_init {
    local kubeadm_config_file
    kubeadm_config_file=$(mktemp)
-
-    if [[ ${CONTAINER_ENGINE} == 'crio' ]]; then
-        CGROUP_DRIVER=$(iniget "/etc/crio/crio.conf" crio.runtime cgroup_manager)
-        CRI_SOCKET="unix:///var/run/crio/crio.sock"
-    else
-        # docker is used
-        CGROUP_DRIVER=$(docker info -f '{{.CgroupDriver}}')
-        CRI_SOCKET="/var/run/dockershim.sock"
-    fi
-
    cat <<EOF | tee $kubeadm_config_file >/dev/null
-apiVersion: kubeadm.k8s.io/v1beta3
+apiVersion: kubeadm.k8s.io/v1beta1
 kind: ClusterConfiguration
 imageRepository: "${KUBEADMIN_IMAGE_REPOSITORY}"
 etcd:
@@ -87,7 +72,7 @@ networking:
  podSubnet: "${K8S_POD_NETWORK_CIDR}"
  serviceSubnet: "${K8S_SERVICE_NETWORK_CIDR}"
 ---
-apiVersion: kubeadm.k8s.io/v1beta3
+apiVersion: kubeadm.k8s.io/v1beta1
 bootstrapTokens:
 - token: "${K8S_TOKEN}"
  ttl: 0s
@@ -95,19 +80,10 @@ kind: InitConfiguration
 localAPIEndpoint:
  advertiseAddress: "${K8S_API_SERVER_IP}"
  bindPort: ${K8S_API_SERVER_PORT}
-nodeRegistration:
-  criSocket: "$CRI_SOCKET"
-  kubeletExtraArgs:
-    enable-server: "true"
-  taints:
-    []
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
 failSwapOn: false
-address: "0.0.0.0"
-enableServer: true
-cgroupDriver: $CGROUP_DRIVER
 EOF
    sudo kubeadm config images pull --image-repository=${KUBEADMIN_IMAGE_REPOSITORY}
    sudo kubeadm init --config $kubeadm_config_file --ignore-preflight-errors Swap
@@ -118,25 +94,15 @@ EOF
    safe_chown $STACK_USER:$STACK_USER $kube_config_file

    if [[ "$K8S_NETWORK_ADDON" == "flannel" ]]; then
-        kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml
+        kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/4ff77dc7c35851913587f7daccf25d754e77aa65/Documentation/kube-flannel.yml
    fi
 }

 function kubeadm_join {
    local kubeadm_config_file
    kubeadm_config_file=$(mktemp)
-
-    if [[ ${CONTAINER_ENGINE} == 'crio' ]]; then
-        CGROUP_DRIVER=$(iniget "/etc/crio/crio.conf" crio.runtime cgroup_manager)
-        CRI_SOCKET="unix:///var/run/crio/crio.sock"
-    else
-        # docker is used
-        CGROUP_DRIVER=$(docker info -f '{{.CgroupDriver}}')
-        CRI_SOCKET="/var/run/dockershim.sock"
-    fi
-
    cat <<EOF | tee $kubeadm_config_file >/dev/null
-apiVersion: kubeadm.k8s.io/v1beta3
+apiVersion: kubeadm.k8s.io/v1beta1
 kind: JoinConfiguration
 discovery:
  bootstrapToken:
@@ -144,19 +110,10 @@ discovery:
    token: "${K8S_TOKEN}"
    unsafeSkipCAVerification: true
  tlsBootstrapToken: "${K8S_TOKEN}"
-nodeRegistration:
-  criSocket: "$CRI_SOCKET"
-  kubeletExtraArgs:
-    enable-server: "true"
-  taints:
-    []
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
 failSwapOn: false
-address: "0.0.0.0"
-enableServer: true
-cgroupDriver: $CGROUP_DRIVER
 EOF
    sudo kubeadm join --config $kubeadm_config_file --ignore-preflight-errors Swap
 }
--- a/devstack/settings
+++ b/devstack/settings
@@ -9,7 +9,7 @@ ENABLE_LIVE_RESTORE=${ENABLE_LIVE_RESTORE:-false}
 ENABLE_IPV6=${ENABLE_IPV6:-false}
 K8S_NETWORK_ADDON=${K8S_NETWORK_ADDON:-flannel}
 ENABLE_CONTAINERD_CRI=${ENABLE_CONTAINERD_CRI:-false}
-CRIO_VERSION=${CRIO_VERSION:-"1.30.5"}
+CRIO_VERSION=${CRIO_VERSION:-"1.18:/1.18.0"}
 CRIO_ALLOW_ICMP=${CRIO_ALLOW_ICMP:-true}
 CNI_CONF_DIR=${CNI_CONF_DIR:-}
 CNI_PLUGIN_DIR=${CNI_PLUGIN_DIR:-}
@@ -28,8 +28,4 @@ if [[ ,${ENABLED_SERVICES} =~ ,"k8s-master" ]]; then
 fi

 # Customize kubeadm container images repository
-KUBEADMIN_IMAGE_REPOSITORY=${KUBEADMIN_IMAGE_REPOSITORY:-"registry.k8s.io"}
-
-# Configure crio pause image
-CRIO_PAUSE_IMAGE=${CRIO_PAUSE_IMAGE:-"registry.k8s.io/pause:3.6"}
-CRIO_PAUSE_COMMAND=${CRIO_PAUSE_COMMAND:-"/pause"}
+KUBEADMIN_IMAGE_REPOSITORY=${KUBEADMIN_IMAGE_REPOSITORY:-"k8s.gcr.io"}