Merge pull request #664 from red-hat-storage/sync_us--master

Syncing latest changes from upstream master for rook

Documentation/CRDs/Cluster/network-providers.md

@@ -73,6 +73,14 @@ Ceph daemons will use any network available on the host for communication. To re
 only a specific specific host interfaces or networks, use `addressRanges` to select the network
 CIDRs Ceph will bind to on the host.
 
+If the Ceph mons are expected to bind to a public network that is different from the IP address
+assign to the K8s node where the mon is running, the IP address for the mon can be set by
+adding an annotation to the node:
+
+```yaml
+network.rook.io/mon-ip: <IPAddress>
+```
+
 If the host networking setting is changed in a cluster where mons are already running, the existing mons will
 remain running with the same network settings with which they were created. To complete the conversion
 to or from host networking after you update this setting, you will need to

ROADMAP.md

@@ -8,26 +8,20 @@ We hope that the items listed below will inspire further engagement from the com
 Any dates listed below and the specific issues that will ship in a given milestone are subject to change but should give a general idea of what we are planning.
 See the [GitHub project boards](https://github.com/rook/rook/projects) for the most up-to-date issues and their status.
 
-## Rook Ceph 1.14
+## Rook Ceph 1.15
 
-The following high level features are targeted for Rook v1.14 (April 2024). For more detailed project tracking see the [v1.14 board](https://github.com/rook/rook/projects/31).
+The following high level features are targeted for Rook v1.15 (July 2024). For more detailed project tracking see the [v1.15 board](https://github.com/rook/rook/projects/32).
 
-* Support for Ceph Squid (v19)
-* Allow setting the application name on a CephBlockPool [#13744](https://github.com/rook/rook/pull/13744)
-* Pool sharing for multiple object stores [#11411](https://github.com/rook/rook/issues/11411)
-* DNS subdomain style access to RGW buckets [#4780](https://github.com/rook/rook/issues/4780)
 * Replace a single OSD when a metadataDevice is configured with multiple OSDs [#13240](https://github.com/rook/rook/issues/13240)
-* Create a default service account for all Ceph daemons [#13362](https://github.com/rook/rook/pull/13362)
-* Enable the rook orchestrator mgr module by default for improved dashboard integration [#13760](https://github.com/rook/rook/issues/13760)
-* Option to run all components on the host network [#13571](https://github.com/rook/rook/issues/13571)
-* Multus-enabled clusters to begin "holder" pod deprecation [#13055](https://github.com/rook/rook/issues/13055)
-* Separate CSI image repository and tag for all images in the helm chart [#13585](https://github.com/rook/rook/issues/13585)
-* Ceph-CSI [v3.11](https://github.com/ceph/ceph-csi/issues?q=is%3Aopen+is%3Aissue+milestone%3Arelease-v3.11.0)
-* Add build support for Go 1.22 [#13738](https://github.com/rook/rook/pull/13738)
-* Add topology based provisioning for external clusters [#13821](https://github.com/rook/rook/pull/13821)
+* Multus-enabled clusters will potentially remove "holder" pods [#14289](https://github.com/rook/rook/issues/14289)
+* Key rotation for Ceph object store users [#11563](https://github.com/rook/rook/issues/11563)
+* CSI Driver
+  * Integrate the new Ceph-CSI operator [#14260](https://github.com/rook/rook/issues/14260)
+  * Ceph-CSI [v3.12](https://github.com/ceph/ceph-csi/issues?q=is%3Aopen+is%3Aissue+milestone%3Arelease-v3.12.0)
+  * Support log rotation for the Ceph-CSI pods [#12809](https://github.com/rook/rook/issues/12809)
 
 ## Kubectl Plugin
 
-Features are planned in the 1.14 time frame for the [Kubectl Plugin](https://github.com/rook/kubectl-rook-ceph).
+Features are planned for the [Kubectl Plugin](https://github.com/rook/kubectl-rook-ceph), though without a committed timeline.
 * Collect details to help troubleshoot the csi driver [#69](https://github.com/rook/kubectl-rook-ceph/issues/69)
-* Command to flatten an RBD image [#222](https://github.com/rook/kubectl-rook-ceph/issues/222)
+* Support `radosgw-admin` commands from the plugin [#253](https://github.com/rook/kubectl-rook-ceph/issues/253)

build/makelib/golang.mk

@@ -201,7 +201,7 @@ $(GOFMT):
 $(GOJUNIT):
 	@echo === installing go-junit-report
 	@mkdir -p $(TOOLS_DIR)/tmp
-	@curl -sL https://github.com/jstemmer/go-junit-report/releases/download/v2.0.0/go-junit-report-v2.0.0-$(GOOS)-$(GOHOSTARCH).tar.gz | tar -xz -C $(TOOLS_DIR)/tmp
+	@curl -sL https://github.com/jstemmer/go-junit-report/releases/download/v2.1.0/go-junit-report-v2.1.0-$(GOOS)-$(GOHOSTARCH).tar.gz | tar -xz -C $(TOOLS_DIR)/tmp
 	@mv $(TOOLS_DIR)/tmp/go-junit-report $(TOOLS_DIR)
 	@rm -fr $(TOOLS_DIR)/tmp
 

deploy/examples/create-external-cluster-resources.py

@@ -14,8 +14,6 @@
 limitations under the License.
 """
 
-# pylint: disable=E0606
-
 import errno
 import sys
 import json
@@ -29,12 +27,12 @@
 from email.utils import formatdate
 import requests
 from requests.auth import AuthBase
-
-py3k = False
-if sys.version_info.major >= 3:
-    py3k = True
-    import urllib.parse
-    from ipaddress import ip_address, IPv4Address
+from io import StringIO
+from urllib.parse import urlparse
+from urllib.parse import urlencode as urlencode
+from ipaddress import ip_address
+from ipaddress import IPv4Address
+from base64 import encodebytes as encodestring
 
 ModuleNotFoundError = ImportError
 
@@ -50,27 +48,6 @@
     print(f"Error: {noModErr}\nExiting the script...")
     sys.exit(1)
 
-try:
-    # for 2.7.x
-    from StringIO import StringIO
-except ModuleNotFoundError:
-    # for 3.x
-    from io import StringIO
-
-try:
-    # for 2.7.x
-    from urlparse import urlparse
-    from urllib import urlencode as urlencode
-except ModuleNotFoundError:
-    # for 3.x
-    from urllib.parse import urlparse
-    from urllib.parse import urlencode as urlencode
-
-try:
-    from base64 import encodestring
-except:
-    from base64 import encodebytes as encodestring
-
 
 class ExecutionFailureException(Exception):
     pass
@@ -235,19 +212,14 @@ def __call__(self, r):
         if "date" not in r.headers and "x-amz-date" not in r.headers:
             r.headers["date"] = formatdate(timeval=None, localtime=False, usegmt=True)
         signature = self.get_signature(r)
-        if py3k:
-            signature = signature.decode("utf-8")
+        signature = signature.decode("utf-8")
         r.headers["Authorization"] = f"AWS {self.access_key}:{signature}"
         return r
 
     def get_signature(self, r):
         canonical_string = self.get_canonical_string(r.url, r.headers, r.method)
-        if py3k:
-            key = self.secret_key.encode("utf-8")
-            msg = canonical_string.encode("utf-8")
-        else:
-            key = self.secret_key
-            msg = canonical_string
+        key = self.secret_key.encode("utf-8")
+        msg = canonical_string.encode("utf-8")
         h = hmac.new(key, msg, digestmod=sha)
         return encodestring(h.digest()).strip()
 
@@ -273,12 +245,8 @@ def get_canonical_string(self, url, headers, method):
                 interesting_headers[lk] = headers[key].strip()
 
         # If x-amz-date is used it supersedes the date header.
-        if not py3k:
-            if "x-amz-date" in interesting_headers:
-                interesting_headers["date"] = ""
-        else:
-            if "x-amz-date" in interesting_headers:
-                interesting_headers["date"] = ""
+        if "x-amz-date" in interesting_headers:
+            interesting_headers["date"] = ""
 
         buf = f"{method}\n"
         for key in sorted(interesting_headers.keys()):
@@ -757,6 +725,7 @@ def _convert_hostname_to_ip(self, host_name, port, ip_type):
             host_ip_type = self._invalid_endpoint(host_addr + ":80")
             import socket
 
+            ip = []
             # example output [(<AddressFamily.AF_INET: 2>, <SocketKind.SOCK_STREAM: 1>, 6, '', ('93.184.216.34', 80)), ...]
             # we need to get 93.184.216.34 so it would be ip[0][4][0]
             if host_ip_type == "IPv6":
@@ -1287,16 +1256,6 @@ def create_rgw_admin_ops_user(self):
                 sys.stderr.write(err_msg)
                 return None, None, False, "-1"
 
-        # if it is python2, don't check for ceph version for adding `info=read` cap(rgw_validation)
-        if sys.version_info.major < 3:
-            jsonoutput = json.loads(output)
-            return (
-                jsonoutput["keys"][0]["access_key"],
-                jsonoutput["keys"][0]["secret_key"],
-                False,
-                "",
-            )
-
         # separately add info=read caps for rgw-endpoint ip validation
         info_cap_supported = True
         cmd = [

go.mod

@@ -14,7 +14,7 @@ replace (
 
 require (
 	github.com/IBM/keyprotect-go-client v0.14.0
-	github.com/aws/aws-sdk-go v1.53.10
+	github.com/aws/aws-sdk-go v1.53.14
 	github.com/banzaicloud/k8s-objectmatcher v1.8.0
 	github.com/ceph/go-ceph v0.27.0
 	github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8

go.sum

@@ -140,8 +140,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/aws-sdk-go v1.53.10 h1:3enP5l5WtezT9Ql+XZqs56JBf5YUd/FEzTCg///OIGY=
-github.com/aws/aws-sdk-go v1.53.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/aws/aws-sdk-go v1.53.14 h1:SzhkC2Pzag0iRW8WBb80RzKdGXDydJR9LAMs2GyKJ2M=
+github.com/aws/aws-sdk-go v1.53.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/banzaicloud/k8s-objectmatcher v1.8.0 h1:Nugn25elKtPMTA2br+JgHNeSQ04sc05MDPmpJnd1N2A=
 github.com/banzaicloud/k8s-objectmatcher v1.8.0/go.mod h1:p2LSNAjlECf07fbhDyebTkPUIYnU05G+WfGgkTmgeMg=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=

pkg/daemon/multus/nginx-config.yaml

@@ -7,7 +7,10 @@ metadata:
 data:
   server.conf: |
     server {
+        # listen on all IPv4 addrs
         listen       8080;
+        # listen on all IPv6 addrs
+        listen       [::]:8080;
         server_name  localhost;
 
         # return the client ip upon connect

pkg/daemon/multus/templates.go

@@ -113,6 +113,12 @@ func (vt *ValidationTest) generateClientTemplateConfig(
 	if attachCluster && serverClusterAddr != "" {
 		netNamesAndAddresses["cluster"] = serverClusterAddr
 	}
+	for name, addr := range netNamesAndAddresses {
+		if strings.Contains(addr, ":") {
+			// it's an IPv6 address and needs square brackets around it to support :<port> addition
+			netNamesAndAddresses[name] = "[" + addr + "]"
+		}
+	}
 	return clientTemplateConfig{
 		NodeType:                 nodeType,
 		ClientType:               clientType,

pkg/operator/ceph/cluster/mon/node.go

@@ -22,12 +22,24 @@ import (
 	v1 "k8s.io/api/core/v1"
 )
 
+const (
+	monIPAnnotation = "network.rook.io/mon-ip"
+)
+
 func getNodeInfoFromNode(n v1.Node) (*opcontroller.MonScheduleInfo, error) {
 	nr := &opcontroller.MonScheduleInfo{
 		Name:     n.Name,
 		Hostname: n.Labels[v1.LabelHostname],
 	}
 
+	// If the host networking is setup such that a different IP should be used
+	// than the one that is to the K8s node.
+	if customIP, ok := n.Annotations[monIPAnnotation]; ok {
+		logger.Infof("found %s annotation on node %q --> %q", monIPAnnotation, n.Name, customIP)
+		nr.Address = customIP
+		return nr, nil
+	}
+
 	for _, ip := range n.Status.Addresses {
 		if ip.Type == v1.NodeInternalIP {
 			logger.Debugf("using internal IP %s for node %s", ip.Address, n.Name)

pkg/operator/ceph/cluster/mon/node_test.go

@@ -246,4 +246,11 @@ func TestGetNodeInfoFromNode(t *testing.T) {
 	info, err = getNodeInfoFromNode(*node)
 	assert.NoError(t, err)
 	assert.Equal(t, "1.2.3.4", info.Address)
+
+	node.Annotations = map[string]string{
+		monIPAnnotation: "9.8.7.6",
+	}
+	info, err = getNodeInfoFromNode(*node)
+	assert.NoError(t, err)
+	assert.Equal(t, "9.8.7.6", info.Address)
 }

tests/scripts/multus/default-public-cluster-nads.yaml

@@ -17,4 +17,4 @@ metadata:
   labels:
   annotations:
 spec:
-  config: '{ "cniVersion": "0.3.1", "type": "macvlan", "master": "eth0", "mode": "bridge", "ipam": { "type": "whereabouts", "range": "192.168.30.0/24" } }'
+  config: '{ "cniVersion": "0.3.1", "type": "macvlan", "master": "eth0", "mode": "bridge", "ipam": { "type": "whereabouts", "range": "fc00::/96" } }'