Mirrors/dokku
1
0
Fork 0
mirror of https://github.com/dokku/dokku.git synced 2025-12-29 00:25:08 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6608 from dokku/k3s-resources

Browse Source 
Fix issue with setting k3s resource values and lower the initial default values
This commit is contained in:
Jose Diaz-Gonzalez
2024-02-22 07:30:10 -05:00
committed by GitHub
parent 20a9e3a61f 93323bd0ff
commit 7d85e213be
5 changed files with 163 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
docs/deployment/schedulers/k3s.md
View File

@@ -361,20 +361,12 @@ App logs for the `logs` command are fetched by Dokku from running containers via
### Supported Resource Management Properties
The `k3s` scheduler supports a minimal list of resource _limits_ and _reservations_. The following properties are supported:
#### Resource Limits
> [!NOTE]
> Cron tasks retrieve resource limits based on the computed cron task ID. If unspecified, the default will be 1 CPU and 512m RAM.
The `k3s` scheduler supports a minimal list of resource _limits_ and _reservations_:
- cpu: is specified in number of CPUs a process can access.
- memory: should be specified with a suffix of `b` (bytes), `k` (kilobytes), `m` (megabytes), `g` (gigabytes). Default unit is `m` (megabytes).
- memory: should be specified with a suffix of `b` (bytes), `Ki` (kilobytes), `Mi` (megabytes), `Gi` (gigabytes). Default unit is `Mi` (megabytes).
#### Resource Reservations
If unspecified for any task, the default reservation will be `.1` CPU and `128Mi` RAM, with no limit set for either CPU or RAM. This is to avoid issues with overscheduling pods on a cluster. To avoid issues, set more specific values for at least resource reservations. If unbounded utilization is desired, set CPU and Memory to `0m` and `0Mi`, respectively.
> [!NOTE]
> Cron tasks retrieve resource reservations based on the computed cron task ID. If unspecified, the default will be 1 CPU and 512m RAM.
- cpu: is specified in number of CPUs a process can access.
- memory: should be specified with a suffix of `b` (bytes), `k` (kilobytes), `m` (megabytes), `g` (gigabytes). Default unit is `m` (megabytes).
> Cron tasks retrieve resource limits based on the computed cron task ID.

6
plugins/common/exec.go
View File

@@ -125,7 +125,11 @@ func CallExecCommandWithContext(ctx context.Context, input ExecCommandInput) (Ex
}
if os.Getenv("DOKKU_TRACE") == "1" {
cmd.PrintCommand = true
argsSt := ""
if len(cmd.Args) > 0 {
argsSt = strings.Join(cmd.Args, " ")
}
LogWarn(fmt.Sprintf("exec: %s %s", cmd.Command, argsSt))
}
if input.Stdin != nil {

16
plugins/common/plugn.go
View File

@@ -2,7 +2,10 @@ package common
import (
"context"
"fmt"
"io"
"os"
"strings"
)
// PlugnTriggerInput is the input for CallPlugnTrigger
@@ -41,7 +44,7 @@ func CallPlugnTrigger(input PlugnTriggerInput) (ExecCommandResponse, error) {
func CallPlugnTriggerWithContext(ctx context.Context, input PlugnTriggerInput) (ExecCommandResponse, error) {
args := []string{"trigger", input.Trigger}
args = append(args, input.Args...)
return CallExecCommandWithContext(ctx, ExecCommandInput{
result, err := CallExecCommandWithContext(ctx, ExecCommandInput{
Command: "plugn",
Args: args,
DisableStdioBuffer: input.DisableStdioBuffer,
@@ -51,4 +54,15 @@ func CallPlugnTriggerWithContext(ctx context.Context, input PlugnTriggerInput) (
StreamStdout: input.StreamStdout,
StreamStderr: input.StreamStderr,
})
if os.Getenv("DOKKU_TRACE") == "1" {
for _, line := range strings.Split(result.Stderr, "\n") {
LogDebug(fmt.Sprintf("plugn trigger %s stderr: %s", input.Trigger, line))
}
for _, line := range strings.Split(result.Stdout, "\n") {
LogDebug(fmt.Sprintf("plugn trigger %s stdout: %s", input.Trigger, line))
}
}
return result, err
}

90
plugins/scheduler-k3s/functions.go
View File

@@ -824,55 +824,86 @@ func getProcessHealtchecks(healthchecks []appjson.Healthcheck, primaryPort int32
func getProcessResources(appName string, processType string) (ProcessResourcesMap, error) {
processResources := ProcessResourcesMap{
Limits: ProcessResources{
CPU: "1000m",
Memory: "512Mi",
},
Limits: ProcessResources{},
Requests: ProcessResources{
CPU: "1000m",
Memory: "512Mi",
CPU: "100m",
Memory: "128Mi",
},
}
cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "cpu"}...)
if err != nil && cpuLimit != "" && cpuLimit != "0" {
_, err := resource.ParseQuantity(cpuLimit)
emptyValues := map[string]bool{
"": true,
"0": true,
}
result, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
Trigger: "resource-get-property",
Args: []string{appName, processType, "limit", "cpu"},
})
if err == nil && !emptyValues[result.StdoutContents()] {
quantity, err := resource.ParseQuantity(result.StdoutContents())
if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu limit: %w", err)
}
processResources.Limits.CPU = cpuLimit
if quantity.MilliValue() != 0 {
processResources.Limits.CPU = quantity.String()
} else {
processResources.Limits.CPU = ""
}
}
nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "nvidia-gpu"}...)
if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
if err == nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
_, err := resource.ParseQuantity(nvidiaGpuLimit)
if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
}
processResources.Limits.NvidiaGPU = nvidiaGpuLimit
}
memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "memory"}...)
if err != nil && memoryLimit != "" && memoryLimit != "0" {
_, err := resource.ParseQuantity(memoryLimit)
result, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
Trigger: "resource-get-property",
Args: []string{appName, processType, "limit", "memory"},
})
if err == nil && !emptyValues[result.StdoutContents()] {
quantity, err := parseMemoryQuantity(result.StdoutContents())
if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory limit: %w", err)
}
processResources.Limits.Memory = memoryLimit
if quantity != "0Mi" {
processResources.Limits.Memory = quantity
} else {
processResources.Limits.Memory = ""
}
}
cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "reserve", "cpu"}...)
if err != nil && cpuRequest != "" && cpuRequest != "0" {
_, err := resource.ParseQuantity(cpuRequest)
result, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
Trigger: "resource-get-property",
Args: []string{appName, processType, "reserve", "cpu"},
})
if err == nil && !emptyValues[result.StdoutContents()] {
quantity, err := resource.ParseQuantity(result.StdoutContents())
if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu request: %w", err)
}
processResources.Requests.CPU = cpuRequest
if quantity.MilliValue() != 0 {
processResources.Requests.CPU = quantity.String()
} else {
processResources.Requests.CPU = ""
}
}
memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "reserve", "memory"}...)
if err != nil && memoryRequest != "" && memoryRequest != "0" {
_, err := resource.ParseQuantity(memoryRequest)
result, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
Trigger: "resource-get-property",
Args: []string{appName, processType, "reserve", "memory"},
})
if err == nil && !emptyValues[result.StdoutContents()] {
quantity, err := parseMemoryQuantity(result.StdoutContents())
if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory request: %w", err)
}
processResources.Requests.Memory = memoryRequest
if quantity != "0Mi" {
processResources.Requests.Memory = quantity
} else {
processResources.Requests.Memory = ""
}
}
return processResources, nil
@@ -1182,6 +1213,19 @@ func kubernetesNodeToNode(node v1.Node) Node {
}
}
// parseMemoryQuantity parses a string into a valid memory quantity
func parseMemoryQuantity(input string) (string, error) {
if _, err := strconv.ParseInt(input, 10, 64); err == nil {
input = fmt.Sprintf("%sMi", input)
}
quantity, err := resource.ParseQuantity(input)
if err != nil {
return "", err
}
return quantity.String(), nil
}
func uninstallHelperCommands(ctx context.Context) error {
errs, _ := errgroup.WithContext(ctx)
errs.Go(func() error {

89
tests/unit/scheduler-k3s.bats
View File

@@ -70,22 +70,6 @@ uninstall_k3s() {
assert_success
}
@test "(scheduler-k3s) install traefik" {
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
fi
install_k3s
}
@test "(scheduler-k3s) install nginx" {
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
fi
INGRESS_CLASS=nginx install_k3s
}
@test "(scheduler-k3s) install traefik with taint" {
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
@@ -102,7 +86,7 @@ uninstall_k3s() {
INGRESS_CLASS=nginx TAINT_SCHEDULING=true install_k3s
}
@test "(scheduler-k3s) deploy traefik" {
@test "(scheduler-k3s) deploy traefik [resource]" {
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
fi
@@ -130,6 +114,77 @@ uninstall_k3s() {
assert_success
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
# include resource tests
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.cpu}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "100m"
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.memory}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "128Mi"
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.limits.cpu}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output ""
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.limits.memory}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output ""
run /bin/bash -c "dokku resource:reserve $TEST_APP --memory 300 --cpu 0m --process-type web"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku resource:limit $TEST_APP --memory 512 --process-type web"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "sleep 20"
echo "output: $output"
echo "status: $status"
assert_success
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.cpu}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output ""
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.memory}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "300Mi"
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.limits.cpu}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output ""
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.limits.memory}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "512Mi"
}
@test "(scheduler-k3s) deploy nginx" {