feat: manage vector container via compose

Using compose instead of manual docker calls allows folks to customize the vector container by using a custom compose.yml template file. This opens us up to more customizations while aligning container management with how we do other external containers (such as for the proxy plugins).

Refs #5784

docs/development/plugin-triggers.md

@@ -2950,3 +2950,18 @@ main() {
 
 main "$@"
 ```
+
+### `vector-template-source`
+
+- Description: Retrieves an alternative template for the vector compose config
+- Invoked by: caddy-vhosts
+- Arguments:
+- Example:
+
+```shell
+#!/usr/bin/env bash
+
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+
+# TODO
+```

plugins/common/docker.go

@@ -33,6 +33,18 @@ func ContainerStart(containerID string) bool {
 	return true
 }
 
+// ContainerRemove runs 'docker container remove' against an existing container
+func ContainerRemove(containerID string) bool {
+	cmd := sh.Command(DockerBin(), "container", "remove", "-f", containerID)
+	cmd.Stdout = nil
+	cmd.Stderr = nil
+	if err := cmd.Run(); err != nil {
+		return false
+	}
+
+	return true
+}
+
 // ContainerExists checks to see if a container exists
 func ContainerExists(containerID string) bool {
 	cmd := sh.Command(DockerBin(), "container", "inspect", containerID)
@@ -259,6 +271,15 @@ func GetWorkingDir(appName string, image string) string {
 	return workDir
 }
 
+func IsComposeInstalled() bool {
+	result, err := CallExecCommand(ExecCommandInput{
+		Command:       DockerBin(),
+		Args:          []string{"info", "--format", "{{range .ClientInfo.Plugins}}{{if eq .Name \"compose\"}}true{{end}}{{end}}')"},
+		CaptureOutput: true,
+	})
+	return err == nil && result.ExitCode == 0
+}
+
 // IsImageCnbBased returns true if app image is based on cnb
 func IsImageCnbBased(image string) bool {
 	if len(image) == 0 {

plugins/logs/functions.go

@@ -5,11 +5,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"html/template"
 	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/dokku/dokku/plugins/common"
 	"github.com/joncalhoun/qson"
@@ -25,92 +25,164 @@ type vectorSource struct {
 	IncludeLabels []string `json:"include_labels,omitempty"`
 }
 
-type vectorSink map[string]interface{}
-
-const vectorContainerName = "vector"
-
-func killVectorContainer() error {
-	if !common.ContainerExists(vectorContainerName) {
-		return nil
-	}
-
-	if err := stopVectorContainer(); err != nil {
-		return err
-	}
-
-	time.Sleep(10 * time.Second)
-	if err := removeVectorContainer(); err != nil {
-		return err
-	}
-
-	return nil
+type vectorTemplateData struct {
+	DokkuLibRoot string
+	DokkuLogsDir string
+	VectorImage  string
 }
 
-func removeVectorContainer() error {
-	if !common.ContainerExists(vectorContainerName) {
-		return nil
+type vectorSink map[string]interface{}
+
+const vectorContainerName = "vector-vector-1"
+const vectorOldContainerName = "vector"
+
+func getComposeFile() ([]byte, error) {
+	result, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+		Trigger:       "vector-template-source",
+		CaptureOutput: true,
+	})
+	if err == nil && result.ExitCode == 0 && strings.TrimSpace(result.Stdout) != "" {
+		contents, err := os.ReadFile(strings.TrimSpace(result.Stdout))
+		if err != nil {
+			return []byte{}, fmt.Errorf("Unable to read compose template: %s", err)
+		}
+
+		return contents, nil
 	}
 
-	cmd := common.NewShellCmd(strings.Join([]string{
-		common.DockerBin(), "container", "rm", "-f", vectorContainerName}, " "))
+	contents, err := templates.ReadFile("templates/compose.yml.tmpl")
+	if err != nil {
+		return []byte{}, fmt.Errorf("Unable to read compose template: %s", err)
+	}
 
-	return common.SuppressOutput(func() error {
-		if cmd.Execute() {
-			return nil
-		}
-
-		if common.ContainerExists(vectorContainerName) {
-			return errors.New("Unable to remove vector container")
-		}
-
-		return nil
-	})
+	return contents, nil
 }
 
 func startVectorContainer(vectorImage string) error {
-	cmd := common.NewShellCmd(strings.Join([]string{
-		common.DockerBin(),
-		"container",
-		"run", "--detach", "--name", vectorContainerName, common.MustGetEnv("DOKKU_GLOBAL_RUN_ARGS"),
-		"--restart", "unless-stopped",
-		"--volume", "/var/lib/dokku/data/logs:/etc/vector",
-		"--volume", "/var/run/docker.sock:/var/run/docker.sock",
-		"--volume", common.MustGetEnv("DOKKU_LOGS_HOST_DIR") + ":/var/logs/dokku/apps",
-		"--volume", common.MustGetEnv("DOKKU_LOGS_HOST_DIR") + "/apps:/var/log/dokku/apps",
-		vectorImage,
-		"--config", "/etc/vector/vector.json", "--watch-config"}, " "))
-	cmd.ShowOutput = false
+	if !common.IsComposeInstalled() {
+		return errors.New("Required docker compose plugin is not installed")
+	}
 
-	if !cmd.Execute() {
-		return errors.New("Unable to start vector container")
+	if common.ContainerExists(vectorOldContainerName) {
+		return errors.New("Vector container %s already exists in old format, run 'dokku logs:vector-stop' once to remove it")
+	}
+
+	tmpFile, err := os.CreateTemp(os.TempDir(), "vector-compose-*.yml")
+	if err != nil {
+		return fmt.Errorf("Unable to create temporary file: %s", err)
+	}
+	defer os.Remove(tmpFile.Name())
+
+	contents, err := getComposeFile()
+	if err != nil {
+		return fmt.Errorf("Unable to read compose template: %s", err)
+	}
+
+	tmpl, err := template.New("compose.yml").Parse(string(contents))
+	if err != nil {
+		return fmt.Errorf("Unable to parse compose template: %s", err)
+	}
+
+	dokkuLibRoot := os.Getenv("DOKKU_LIB_HOST_ROOT")
+	if dokkuLibRoot == "" {
+		dokkuLibRoot = os.Getenv("DOKKU_LIB_ROOT")
+	}
+
+	dokkuLogsDir := os.Getenv("DOKKU_LOGS_HOST_DIR")
+	if dokkuLogsDir == "" {
+		dokkuLogsDir = os.Getenv("DOKKU_LOGS_DIR")
+	}
+
+	data := vectorTemplateData{
+		DokkuLibRoot: dokkuLibRoot,
+		DokkuLogsDir: dokkuLogsDir,
+		VectorImage:  vectorImage,
+	}
+
+	if err := tmpl.Execute(tmpFile, data); err != nil {
+		return fmt.Errorf("Unable to execute compose template: %s", err)
+	}
+
+	result, err := common.CallExecCommand(common.ExecCommandInput{
+		Command: common.DockerBin(),
+		Args: []string{
+			"compose",
+			"--file", tmpFile.Name(),
+			"--project-name", "vector",
+			"up",
+			"--detach",
+			"--quiet-pull",
+		},
+		StreamStdio: true,
+	})
+	if err != nil || result.ExitCode != 0 {
+		return fmt.Errorf("Unable to start vector container: %s", result.Stderr)
 	}
 
 	return nil
 }
 
 func stopVectorContainer() error {
-	if !common.ContainerExists(vectorContainerName) {
-		return nil
+	if !common.IsComposeInstalled() {
+		return errors.New("Required docker compose plugin is not installed")
 	}
 
-	if !common.ContainerIsRunning(vectorContainerName) {
-		return nil
+	if common.ContainerExists(vectorOldContainerName) {
+		common.ContainerRemove(vectorOldContainerName)
 	}
 
-	cmd := common.NewShellCmd(strings.Join([]string{
-		common.DockerBin(), "container", "stop", vectorContainerName}, " "))
+	tmpFile, err := os.CreateTemp(os.TempDir(), "vector-compose-*.yml")
+	if err != nil {
+		return fmt.Errorf("Unable to create temporary file: %s", err)
+	}
+	defer os.Remove(tmpFile.Name())
 
-	return common.SuppressOutput(func() error {
-		if cmd.Execute() {
-			return nil
-		}
+	contents, err := getComposeFile()
+	if err != nil {
+		return fmt.Errorf("Unable to read compose template: %s", err)
+	}
 
-		if common.ContainerIsRunning(vectorContainerName) {
-			return errors.New("Unable to stop vector container")
-		}
+	tmpl, err := template.New("compose.yml").Parse(string(contents))
+	if err != nil {
+		return fmt.Errorf("Unable to parse compose template: %s", err)
+	}
 
-		return nil
+	dokkuLibRoot := os.Getenv("DOKKU_LIB_HOST_ROOT")
+	if dokkuLibRoot == "" {
+		dokkuLibRoot = os.Getenv("DOKKU_LIB_ROOT")
+	}
+
+	dokkuLogsDir := os.Getenv("DOKKU_LOGS_HOST_DIR")
+	if dokkuLogsDir == "" {
+		dokkuLogsDir = os.Getenv("DOKKU_LOGS_DIR")
+	}
+
+	data := vectorTemplateData{
+		DokkuLibRoot: dokkuLibRoot,
+		DokkuLogsDir: dokkuLogsDir,
+		VectorImage:  VectorImage,
+	}
+
+	if err := tmpl.Execute(tmpFile, data); err != nil {
+		return fmt.Errorf("Unable to execute compose template: %s", err)
+	}
+
+	result, err := common.CallExecCommand(common.ExecCommandInput{
+		Command: common.DockerBin(),
+		Args: []string{
+			"compose",
+			"--file", tmpFile.Name(),
+			"--project-name", "vector",
+			"down",
+			"--remove-orphans",
+		},
+		StreamStdio: true,
 	})
+	if err != nil || result.ExitCode != 0 {
+		return fmt.Errorf("Unable to stop vector container: %s", result.Stderr)
+	}
+
+	return nil
 }
 
 func sinkValueToConfig(appName string, sinkValue string) (vectorSink, error) {
@@ -132,9 +204,7 @@ func sinkValueToConfig(appName string, sinkValue string) (vectorSink, error) {
 	u.Scheme = strings.ReplaceAll(u.Scheme, "-", "_")
 
 	query := u.RawQuery
-	if strings.HasPrefix(query, "&") {
-		query = strings.TrimPrefix(query, "&")
-	}
+	query = strings.TrimPrefix(query, "&")
 
 	b, err := qson.ToJSON(query)
 	if err != nil {

plugins/logs/logs.go

@@ -1,6 +1,7 @@
 package logs
 
 import (
+	"embed"
 	"fmt"
 
 	"github.com/dokku/dokku/plugins/common"
@@ -30,6 +31,9 @@ const VectorImage = "timberio/vector:0.35.X-debian"
 // VectorDefaultSink contains the default sink in use for vector log shipping
 const VectorDefaultSink = "blackhole://?print_interval_secs=1"
 
+//go:embed templates/*
+var templates embed.FS
+
 // GetFailedLogs outputs failed deploy logs for a given app
 func GetFailedLogs(appName string) error {
 	common.LogInfo2Quiet(fmt.Sprintf("%s failed deploy logs", appName))

plugins/logs/subcommands.go

@@ -107,20 +107,9 @@ func CommandVectorStart(vectorImage string) error {
 		vectorImage = common.PropertyGetDefault("logs", "--global", "vector-image", VectorImage)
 	}
 
-	if common.ContainerExists(vectorContainerName) {
-		if common.ContainerIsRunning(vectorContainerName) {
-			common.LogVerbose("Vector container is running")
-			return nil
-		}
-
-		common.LogVerbose("Starting vector container")
-		if !common.ContainerStart(vectorContainerName) {
-			return errors.New("Unable to start vector container")
-		}
-	} else {
-		if err := startVectorContainer(vectorImage); err != nil {
-			return err
-		}
+	common.LogVerbose("Starting vector container")
+	if err := startVectorContainer(vectorImage); err != nil {
+		return err
 	}
 
 	common.LogVerbose("Waiting for 10 seconds")
@@ -134,6 +123,6 @@ func CommandVectorStart(vectorImage string) error {
 
 // CommandVectorStop stops and removes an existing vector container
 func CommandVectorStop() error {
-	common.LogInfo2Quiet("Stopping and removing vector container")
-	return killVectorContainer()
+	common.LogInfo2Quiet("Stopping and removing vector container")
+	return stopVectorContainer()
 }

plugins/logs/templates/compose.yml.tmpl

@@ -0,0 +1,25 @@
+---
+version: "3.7"
+
+services:
+  vector:
+    image: "{{ $.VectorImage }}"
+
+    command:
+      - "--config"
+      - "/etc/vector/vector.json"
+      - "--watch-config"
+
+    labels:
+      dokku: ""
+      org.label-schema.schema-version: "1.0"
+      org.label-schema.vendor: dokku
+
+    network_mode: bridge
+
+    restart: unless-stopped
+
+    volumes:
+      - "{{ $.DokkuLibRoot }}/data/logs:/etc/vector"
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+      - "{{ $.DokkuLogsDir }}/apps:/var/log/dokku/apps"

tests/unit/logs.bats

@@ -553,7 +553,7 @@ teardown() {
   assert_success
   assert_output_contains "Vector container is running"
 
-  run /bin/bash -c "sudo docker inspect --format='{{.HostConfig.RestartPolicy.Name}}' vector"
+  run /bin/bash -c "sudo docker inspect --format='{{.HostConfig.RestartPolicy.Name}}' vector-vector-1"
   echo "output: $output"
   echo "status: $status"
   assert_success
@@ -586,7 +586,7 @@ teardown() {
   assert_output_contains "vector:" 5
   assert_line_count 6
 
-  run /bin/bash -c "docker stop vector"
+  run /bin/bash -c "docker stop vector-vector-1"
   echo "output: $output"
   echo "status: $status"
   assert_success
@@ -602,5 +602,5 @@ teardown() {
   echo "output: $output"
   echo "status: $status"
   assert_success
-  assert_output_contains "Stopping and removing vector container"
+  assert_output_contains "Stopping and removing vector container"
 }