Skip to content

Commit 64ae746

Browse files
authored
add support for gpu-operator-ready flag on k8s clusters (#2051)
1 parent ce2e254 commit 64ae746

15 files changed

Lines changed: 733 additions & 18 deletions

‎args.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,8 @@ const (
155155
ArgHA = "ha"
156156
// ArgIsolatedWorkers is a cluster's isolated workers argument.
157157
ArgIsolatedWorkers = "isolated-workers"
158+
// ArgGPUOperatorReady is a cluster's GPU operator ready argument.
159+
ArgGPUOperatorReady = "gpu-operator-ready"
158160
// ArgEnableControlPlaneFirewall enable control plane firewall.
159161
ArgEnableControlPlaneFirewall = "enable-control-plane-firewall"
160162
// ArgControlPlaneFirewallAllowedAddresses list of allowed addresses that can access the control plane.

‎commands/displayers/kubernetes.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ func (clusters *KubernetesClusters) Cols() []string {
6262
"NvidiaGpuDraDriver",
6363
"AmdGpuDraDriver",
6464
"RDMASharedDevicePlugin",
65+
"GPUOperatorReady",
6566
}
6667
}
6768

@@ -106,6 +107,7 @@ func (clusters *KubernetesClusters) ColMap() map[string]string {
106107
"NvidiaGpuDraDriver": "NVIDIA GPU DRA Driver",
107108
"AmdGpuDraDriver": "AMD GPU DRA Driver",
108109
"RDMASharedDevicePlugin": "RDMA Shared Device Plugin",
110+
"GPUOperatorReady": "GPU Operator Ready",
109111
}
110112
}
111113

@@ -151,6 +153,7 @@ func (clusters *KubernetesClusters) KV() []map[string]any {
151153
"NvidiaGpuDraDriver": cluster.NvidiaGpuDraDriver != nil && *cluster.NvidiaGpuDraDriver.Enabled,
152154
"AmdGpuDraDriver": cluster.AmdGpuDraDriver != nil && *cluster.AmdGpuDraDriver.Enabled,
153155
"RDMASharedDevicePlugin": cluster.RdmaSharedDevicePlugin != nil && *cluster.RdmaSharedDevicePlugin.Enabled,
156+
"GPUOperatorReady": cluster.GPUOperatorReady,
154157
}
155158

156159
if cfg := cluster.ClusterAutoscalerConfiguration; cfg != nil {

‎commands/kubernetes.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,8 @@ After creating a cluster, a configuration context is added to kubectl and made a
314314
AddBoolFlag(cmdKubeClusterCreate, doctl.ArgSurgeUpgrade, "", true,
315315
"Enables surge-upgrade for the cluster")
316316
AddBoolFlag(cmdKubeClusterCreate, doctl.ArgIsolatedWorkers, "", false, "Enables isolated workers for the cluster. Defaults to `false`, To enable isolated workers for the cluster, supply `--isolated-workers=true`. NAT GW must be available in the vpc when this option is enabled.")
317+
AddBoolFlag(cmdKubeClusterCreate, doctl.ArgGPUOperatorReady, "", false,
318+
"Launches GPU nodes in the cluster with images without GPU drivers, allowing GPU operators to manage drivers. Defaults to `false`. To enable it, supply `--gpu-operator-ready=true`. Mutually exclusive with other GPU-related cluster options.")
317319
AddBoolFlag(cmdKubeClusterCreate, doctl.ArgHA, "", false,
318320
"Creates the cluster with a highly-available control plane. When omitted, API applies version-specific default (true for 1.36.0+; false for older). Use --ha to enable, --ha=false to disable.")
319321
AddBoolFlag(cmdKubeClusterCreate, doctl.ArgEnableControlPlaneFirewall, "", false,
@@ -398,6 +400,8 @@ Updates the configuration values for a Kubernetes cluster. The cluster must be r
398400
"Indicates whether the cluster automatically upgrades to new patch releases during its maintenance window. To enable automatic upgrade, use `--auto-upgrade=true`.")
399401
AddBoolFlag(cmdKubeClusterUpdate, doctl.ArgSurgeUpgrade, "", false,
400402
"Enables surge-upgrade for the cluster")
403+
AddBoolFlag(cmdKubeClusterUpdate, doctl.ArgGPUOperatorReady, "", false,
404+
"Launches GPU nodes in the cluster with images without GPU drivers, allowing GPU operators to manage drivers. To enable it, supply `--gpu-operator-ready=true`; to disable it, supply `--gpu-operator-ready=false`. Mutually exclusive with other GPU-related cluster options.")
401405
AddBoolFlag(cmdKubeClusterUpdate, doctl.ArgHA, "", false,
402406
"Enables the highly-available control plane for the cluster")
403407
AddBoolFlag(cmdKubeClusterUpdate, doctl.ArgEnableControlPlaneFirewall, "", false,
@@ -1851,6 +1855,12 @@ func buildClusterCreateRequestFromArgs(c *CmdConfig, r *godo.KubernetesClusterCr
18511855
}
18521856
r.IsolatedWorkers = isolatedWorkers
18531857

1858+
gpuOperatorReady, err := c.Doit.GetBool(c.NS, doctl.ArgGPUOperatorReady)
1859+
if err != nil {
1860+
return err
1861+
}
1862+
r.GPUOperatorReady = gpuOperatorReady
1863+
18541864
ha, err := c.Doit.GetBoolPtr(c.NS, doctl.ArgHA)
18551865
if err != nil {
18561866
return err
@@ -2128,6 +2138,12 @@ func buildClusterUpdateRequestFromArgs(c *CmdConfig, r *godo.KubernetesClusterUp
21282138
}
21292139
r.HA = ha
21302140

2141+
gpuOperatorReady, err := c.Doit.GetBoolPtr(c.NS, doctl.ArgGPUOperatorReady)
2142+
if err != nil {
2143+
return err
2144+
}
2145+
r.GPUOperatorReady = gpuOperatorReady
2146+
21312147
enableControlPlaneFirewall, err := c.Doit.GetBoolPtr(c.NS, doctl.ArgEnableControlPlaneFirewall)
21322148
if err != nil {
21332149
return err

‎commands/kubernetes_test.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -774,7 +774,8 @@ func TestKubernetesCreate(t *testing.T) {
774774
Enabled: true,
775775
Required: false,
776776
},
777-
IsolatedWorkers: true,
777+
IsolatedWorkers: true,
778+
GPUOperatorReady: true,
778779
}
779780
tm.kubernetes.EXPECT().Create(&r).Return(&testCluster, nil)
780781

@@ -812,6 +813,7 @@ func TestKubernetesCreate(t *testing.T) {
812813
config.Doit.Set(config.NS, doctl.ArgEnableRDMASharedDevicePlugin, testCluster.RdmaSharedDevicePlugin.Enabled)
813814
config.Doit.Set(config.NS, doctl.ArgEnableCorednsAutoscaler, testCluster.CorednsAutoscaler.Enabled)
814815
config.Doit.Set(config.NS, doctl.ArgIsolatedWorkers, true)
816+
config.Doit.Set(config.NS, doctl.ArgGPUOperatorReady, true)
815817
config.Doit.Set(config.NS, doctl.ArgKubernetesEnableSSO, true)
816818
config.Doit.Set(config.NS, doctl.ArgKubernetesRequireSSO, false)
817819
config.Doit.Set(config.NS, doctl.ArgKubernetesSSOIssuerURL, "https://issuer.example")
@@ -985,8 +987,9 @@ func TestKubernetesUpdate(t *testing.T) {
985987
StartTime: "00:00",
986988
Day: godo.KubernetesMaintenanceDayAny,
987989
},
988-
AutoUpgrade: boolPtr(false),
989-
HA: boolPtr(true),
990+
AutoUpgrade: boolPtr(false),
991+
HA: boolPtr(true),
992+
GPUOperatorReady: boolPtr(true),
990993
ControlPlaneFirewall: &godo.KubernetesControlPlaneFirewall{
991994
Enabled: boolPtr(true),
992995
AllowedAddresses: []string{
@@ -1042,6 +1045,7 @@ func TestKubernetesUpdate(t *testing.T) {
10421045
config.Doit.Set(config.NS, doctl.ArgMaintenanceWindow, "any=00:00")
10431046
config.Doit.Set(config.NS, doctl.ArgAutoUpgrade, false)
10441047
config.Doit.Set(config.NS, doctl.ArgHA, true)
1048+
config.Doit.Set(config.NS, doctl.ArgGPUOperatorReady, true)
10451049
config.Doit.Set(config.NS, doctl.ArgEnableControlPlaneFirewall, testCluster.ControlPlaneFirewall.Enabled)
10461050
config.Doit.Set(config.NS, doctl.ArgControlPlaneFirewallAllowedAddresses, testCluster.ControlPlaneFirewall.AllowedAddresses)
10471051
config.Doit.Set(config.NS, doctl.ArgClusterAutoscalerScaleDownUtilizationThreshold, testCluster.ClusterAutoscalerConfiguration.ScaleDownUtilizationThreshold)

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.26.0
55
require (
66
github.com/blang/semver v3.5.1+incompatible
77
github.com/creack/pty v1.1.21
8-
github.com/digitalocean/godo v1.224.0
8+
github.com/digitalocean/godo v1.226.0
99
github.com/docker/cli v24.0.5+incompatible
1010
github.com/docker/docker v25.0.6+incompatible
1111
github.com/docker/docker-credential-helpers v0.7.0 // indirect

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
120120
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
121121
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
122122
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
123-
github.com/digitalocean/godo v1.224.0 h1:1IdHFvm/QLAvWPimwcZuVJoo5yLwFOIDvF3Ui+Ktnww=
124-
github.com/digitalocean/godo v1.224.0/go.mod h1:xQsWpVCCbkDrWisHA72hPzPlnC+4W5w/McZY5ij9uvU=
123+
github.com/digitalocean/godo v1.226.0 h1:SHunjAvAyQO9QvtDsPEAYg2hwWW1s1Z160QPAJ98jJo=
124+
github.com/digitalocean/godo v1.226.0/go.mod h1:xQsWpVCCbkDrWisHA72hPzPlnC+4W5w/McZY5ij9uvU=
125125
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
126126
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
127127
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=

‎integration/kubernetes_clusters_create_test.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,9 @@ var _ = suite("kubernetes/clusters/create", func(t *testing.T, when spec.G, it s
5454
if strings.Contains(string(reqBody), "some-non-ha-cluster") {
5555
matchedRequest = kubeClustersCreateNonHAJSONReq
5656
}
57+
if strings.Contains(string(reqBody), "some-gpu-operator-cluster") {
58+
matchedRequest = kubeClustersCreateGPUOperatorReadyJSONReq
59+
}
5760
// When --ha is omitted, request has no "ha" field; API applies version-specific default
5861
if strings.Contains(string(reqBody), "some-cluster-name") && !strings.Contains(string(reqBody), `"ha"`) {
5962
matchedRequest = kubeClustersCreateJSONReqOmitHA
@@ -178,6 +181,33 @@ var _ = suite("kubernetes/clusters/create", func(t *testing.T, when spec.G, it s
178181
output, err := cmd.CombinedOutput()
179182
expect.NoError(err, fmt.Sprintf("received error output: %s", output))
180183
})
184+
185+
it("creates a kube cluster with GPU operator ready when --gpu-operator-ready", func() {
186+
f, err := os.CreateTemp(t.TempDir(), "fake-kube-config")
187+
expect.NoError(err)
188+
189+
err = f.Close()
190+
expect.NoError(err)
191+
192+
cmd := exec.Command(builtBinaryPath,
193+
"-t", "some-magic-token",
194+
"-u", server.URL,
195+
"kubernetes",
196+
"clusters",
197+
"create",
198+
"some-gpu-operator-cluster",
199+
"--region", "mars",
200+
"--version", "some-kube-version",
201+
"--gpu-operator-ready",
202+
)
203+
204+
cmd.Env = append(os.Environ(),
205+
fmt.Sprintf("KUBECONFIG=%s", f.Name()),
206+
)
207+
208+
output, err := cmd.CombinedOutput()
209+
expect.NoError(err, fmt.Sprintf("received error output: %s", output))
210+
})
181211
})
182212

183213
when("using node-pool", func() {
@@ -345,6 +375,28 @@ some-cluster-id some-cluster-name mars some-kube-version false
345375
}
346376
]
347377
}
378+
`
379+
kubeClustersCreateGPUOperatorReadyJSONReq = `
380+
{
381+
"name": "some-gpu-operator-cluster",
382+
"region": "mars",
383+
"version": "some-kube-version",
384+
"auto_upgrade": false,
385+
"surge_upgrade": true,
386+
"gpu_operator_ready": true,
387+
"maintenance_policy": {
388+
"day": "any",
389+
"duration": "",
390+
"start_time": "00:00"
391+
},
392+
"node_pools": [
393+
{
394+
"size": "s-1vcpu-2gb-intel",
395+
"count": 3,
396+
"name": "some-gpu-operator-cluster-default-pool"
397+
}
398+
]
399+
}
348400
`
349401
kubeNodePoolCreateJSONReq = `
350402
{

‎integration/kubernetes_clusters_get_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ var (
109109
"scale_down_unneeded_time": "1m30s",
110110
"expanders": ["priority", "random"]
111111
},
112+
"gpu_operator_ready": true,
112113
"created_at": "2018-11-15T16:00:11Z",
113114
"updated_at": "2018-11-15T16:00:11Z"
114115
}
@@ -117,7 +118,7 @@ var (
117118
`
118119

119120
k8sGetOutput = `
120-
ID Name Region Version Auto Upgrade HA Control Plane Status Endpoint IPv4 Cluster Subnet Service Subnet Tags Created At Updated At Node Pools Autoscaler Scale Down Utilization Autoscaler Scale Down Unneeded Time Autoscaler Custom Expanders Routing Agent Peer-to-peer OCI registry Plugin NFS CSI Plugin CoreDNS Autoscaler AMD GPU Device Plugin AMD GPU Device Metrics Exporter Plugin NVIDIA GPU Device Plugin NVIDIA GPU DRA Driver AMD GPU DRA Driver RDMA Shared Device Plugin
121-
some-cluster-id some-cluster-id nyc3 some-kube-version true false running production 2018-11-15 16:00:11 +0000 UTC 2018-11-15 16:00:11 +0000 UTC frontend-pool 50% 1m30s priority, random false false false false false false false false false false
121+
ID Name Region Version Auto Upgrade HA Control Plane Status Endpoint IPv4 Cluster Subnet Service Subnet Tags Created At Updated At Node Pools Autoscaler Scale Down Utilization Autoscaler Scale Down Unneeded Time Autoscaler Custom Expanders Routing Agent Peer-to-peer OCI registry Plugin NFS CSI Plugin CoreDNS Autoscaler AMD GPU Device Plugin AMD GPU Device Metrics Exporter Plugin NVIDIA GPU Device Plugin NVIDIA GPU DRA Driver AMD GPU DRA Driver RDMA Shared Device Plugin GPU Operator Ready
122+
some-cluster-id some-cluster-id nyc3 some-kube-version true false running production 2018-11-15 16:00:11 +0000 UTC 2018-11-15 16:00:11 +0000 UTC frontend-pool 50% 1m30s priority, random false false false false false false false false false false true
122123
`
123124
)

‎integration/projects_resources_get_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -343,8 +343,8 @@ ID Name Size Region Filesyste
343343
}
344344
`
345345
projectsResourcesGetKubernetesOutput = `
346-
ID Name Region Version Auto Upgrade HA Control Plane Status Endpoint IPv4 Cluster Subnet Service Subnet Tags Created At Updated At Node Pools Autoscaler Scale Down Utilization Autoscaler Scale Down Unneeded Time Autoscaler Custom Expanders Routing Agent Peer-to-peer OCI registry Plugin NFS CSI Plugin CoreDNS Autoscaler AMD GPU Device Plugin AMD GPU Device Metrics Exporter Plugin NVIDIA GPU Device Plugin NVIDIA GPU DRA Driver AMD GPU DRA Driver RDMA Shared Device Plugin
347-
1111 false false provisioning k8s 2021-01-29 16:02:02 +0000 UTC 0001-01-01 00:00:00 +0000 UTC pool-test false false false false false false false false false false
346+
ID Name Region Version Auto Upgrade HA Control Plane Status Endpoint IPv4 Cluster Subnet Service Subnet Tags Created At Updated At Node Pools Autoscaler Scale Down Utilization Autoscaler Scale Down Unneeded Time Autoscaler Custom Expanders Routing Agent Peer-to-peer OCI registry Plugin NFS CSI Plugin CoreDNS Autoscaler AMD GPU Device Plugin AMD GPU Device Metrics Exporter Plugin NVIDIA GPU Device Plugin NVIDIA GPU DRA Driver AMD GPU DRA Driver RDMA Shared Device Plugin GPU Operator Ready
347+
1111 false false provisioning k8s 2021-01-29 16:02:02 +0000 UTC 0001-01-01 00:00:00 +0000 UTC pool-test false false false false false false false false false false false
348348
`
349349

350350
projectsResourcesListKubernetesOutput = `

‎vendor/github.com/digitalocean/godo/CHANGELOG.md‎

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)
⌂ 换网址