From 518337812f61cad641fb8e746c0a43bfa367a45c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 4 Mar 2021 20:43:37 -0500 Subject: [PATCH 01/36] chore: fix comment --- plugins/builder/report.go | 1 + plugins/builder/subcommands.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/builder/report.go b/plugins/builder/report.go index 3f2f9cb15..0d7772047 100644 --- a/plugins/builder/report.go +++ b/plugins/builder/report.go @@ -38,6 +38,7 @@ func reportComputedSelected(appName string) string { return value } + func reportGlobalSelected(appName string) string { return common.PropertyGet("builder", "--global", "selected") } diff --git a/plugins/builder/subcommands.go b/plugins/builder/subcommands.go index 18e04ac0e..9278f8493 100644 --- a/plugins/builder/subcommands.go +++ b/plugins/builder/subcommands.go @@ -4,7 +4,7 @@ import ( "github.com/dokku/dokku/plugins/common" ) -// CommandReport displays a network report for one or more apps +// CommandReport displays a builder report for one or more apps func CommandReport(appName string, format string, infoFlag string) error { if len(appName) == 0 { apps, err := common.DokkuApps() From a755f69e92cb7f39b41229f43d393a974c3f427e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 4 Mar 2021 20:44:49 -0500 Subject: [PATCH 02/36] feat: first pass at registry plugin --- plugins/registry/.gitignore | 9 ++ plugins/registry/Makefile | 6 + plugins/registry/go.mod | 12 ++ plugins/registry/go.sum | 10 ++ plugins/registry/plugin.toml | 4 + plugins/registry/registry.go | 20 +++ plugins/registry/report.go | 131 ++++++++++++++++++ plugins/registry/src/commands/commands.go | 57 ++++++++ .../registry/src/subcommands/subcommands.go | 58 ++++++++ plugins/registry/src/triggers/triggers.go | 36 +++++ plugins/registry/subcommands.go | 65 +++++++++ plugins/registry/triggers.go | 21 +++ 12 files changed, 429 insertions(+) create mode 100644 plugins/registry/.gitignore create mode 100644 plugins/registry/Makefile create mode 100644 plugins/registry/go.mod create mode 100644 plugins/registry/go.sum create mode 100644 plugins/registry/plugin.toml create mode 100644 plugins/registry/registry.go create mode 100644 plugins/registry/report.go create mode 100644 plugins/registry/src/commands/commands.go create mode 100644 plugins/registry/src/subcommands/subcommands.go create mode 100644 plugins/registry/src/triggers/triggers.go create mode 100644 plugins/registry/subcommands.go create mode 100644 plugins/registry/triggers.go diff --git a/plugins/registry/.gitignore b/plugins/registry/.gitignore new file mode 100644 index 000000000..dee017e8f --- /dev/null +++ b/plugins/registry/.gitignore @@ -0,0 +1,9 @@ +/commands +/core-post-deploy +/subcommands/* +/triggers/* +/triggers +/network-* +/install +/post-* +/report diff --git a/plugins/registry/Makefile b/plugins/registry/Makefile new file mode 100644 index 000000000..bdfd0efe5 --- /dev/null +++ b/plugins/registry/Makefile @@ -0,0 +1,6 @@ +SUBCOMMANDS = subcommands/login subcommands/report subcommands/set +TRIGGERS = triggers/install triggers/post-delete triggers/report +BUILD = commands subcommands triggers +PLUGIN_NAME = registry + +include ../../common.mk diff --git a/plugins/registry/go.mod b/plugins/registry/go.mod new file mode 100644 index 000000000..a0a8fcac0 --- /dev/null +++ b/plugins/registry/go.mod @@ -0,0 +1,12 @@ +module github.com/dokku/dokku/plugins/registry + +go 1.15 + +require ( + github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 + github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 + github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000 + github.com/spf13/pflag v1.0.5 // indirect +) + +replace github.com/dokku/dokku/plugins/common => ../common diff --git a/plugins/registry/go.sum b/plugins/registry/go.sum new file mode 100644 index 000000000..58c904707 --- /dev/null +++ b/plugins/registry/go.sum @@ -0,0 +1,10 @@ +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA= +github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE= +github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 h1:utdYOikI1XjNtTFGCwSM6OmFJblU4ld4gACoJsbadJg= +github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/plugins/registry/plugin.toml b/plugins/registry/plugin.toml new file mode 100644 index 000000000..8d6e20e3f --- /dev/null +++ b/plugins/registry/plugin.toml @@ -0,0 +1,4 @@ +[plugin] +description = "dokku core registry plugin" +version = "0.24.0" +[plugin.config] diff --git a/plugins/registry/registry.go b/plugins/registry/registry.go new file mode 100644 index 000000000..32dcded7b --- /dev/null +++ b/plugins/registry/registry.go @@ -0,0 +1,20 @@ +package registry + +var ( + // DefaultProperties is a map of all valid network properties with corresponding default property values + DefaultProperties = map[string]string{ + "create-repository": "false", + "disable-delete-warning": "false", + "image-repo": "", + "push-on-release": "false", + "server": "", + } + + // GlobalProperties is a map of all valid global network properties + GlobalProperties = map[string]bool{ + "create-repository": true, + "disable-delete-warning": true, + "push-on-release": true, + "server": true, + } +) diff --git a/plugins/registry/report.go b/plugins/registry/report.go new file mode 100644 index 000000000..ad077cde8 --- /dev/null +++ b/plugins/registry/report.go @@ -0,0 +1,131 @@ +package registry + +import ( + "github.com/dokku/dokku/plugins/common" +) + +// ReportSingleApp is an internal function that displays the registry report for one or more apps +func ReportSingleApp(appName string, format string, infoFlag string) error { + if err := common.VerifyAppName(appName); err != nil { + return err + } + + flags := map[string]common.ReportFunc{ + "--registry-computed-create-repository": reportComputedCreateRepository, + "--registry-global-create-repository": reportGlobalCreateRepository, + "--registry-create-repository": reportCreateRepository, + "--registry-computed-disable-delete-warning": reportComputedDisableDeleteWarning, + "--registry-global-disable-delete-warning": reportGlobalDisableDeleteWarning, + "--registry-disable-delete-warning": reportDisableDeleteWarning, + "--registry-image-repo": reportImageRepo, + "--registry-computed-push-on-release": reportComputedPushOnRelease, + "--registry-global-push-on-release": reportGlobalPushOnRelease, + "--registry-push-on-release": reportPushOnRelease, + "--registry-computed-server": reportComputedServer, + "--registry-global-server": reportGlobalServer, + "--registry-server": reportServer, + "--registry-tag-version": reportTagVersion, + } + + flagKeys := []string{} + for flagKey := range flags { + flagKeys = append(flagKeys, flagKey) + } + + trimPrefix := false + uppercaseFirstCharacter := true + infoFlags := common.CollectReport(appName, infoFlag, flags) + return common.ReportSingleApp("registry", appName, infoFlag, infoFlags, flagKeys, format, trimPrefix, uppercaseFirstCharacter) +} + +func reportComputedCreateRepository(appName string) string { + value := reportCreateRepository(appName) + if value == "" { + value = reportGlobalCreateRepository(appName) + } + + if value == "" { + value = DefaultProperties["create-repository"] + } + + return value +} + +func reportGlobalCreateRepository(appName string) string { + return common.PropertyGet("registry", "--global", "create-repository") +} + +func reportCreateRepository(appName string) string { + return common.PropertyGet("registry", appName, "create-repository") +} + +func reportComputedDisableDeleteWarning(appName string) string { + value := reportDisableDeleteWarning(appName) + if value == "" { + value = reportGlobalDisableDeleteWarning(appName) + } + + if value == "" { + value = DefaultProperties["disable-delete-warning"] + } + + return value +} + +func reportGlobalDisableDeleteWarning(appName string) string { + return common.PropertyGet("registry", "--global", "disable-delete-warning") +} + +func reportDisableDeleteWarning(appName string) string { + return common.PropertyGet("registry", appName, "disable-delete-warning") +} + +func reportImageRepo(appName string) string { + return common.PropertyGet("registry", appName, "image-repo") +} + +func reportComputedPushOnRelease(appName string) string { + value := reportPushOnRelease(appName) + if value == "" { + value = reportGlobalPushOnRelease(appName) + } + + if value == "" { + value = DefaultProperties["push-on-release"] + } + + return value +} + +func reportGlobalPushOnRelease(appName string) string { + return common.PropertyGet("registry", "--global", "push-on-release") +} + +func reportPushOnRelease(appName string) string { + return common.PropertyGet("registry", appName, "push-on-release") +} + +func reportComputedServer(appName string) string { + value := reportServer(appName) + if value == "" { + value = reportGlobalServer(appName) + } + + if value == "" { + value = DefaultProperties["server"] + } + + return value +} + +func reportGlobalServer(appName string) string { + return common.PropertyGet("registry", "--global", "server") +} + +func reportServer(appName string) string { + return common.PropertyGet("registry", appName, "server") +} + +func reportTagVersion(appName string) string { + return common.PropertyGet("registry", appName, "tag-version") +} diff --git a/plugins/registry/src/commands/commands.go b/plugins/registry/src/commands/commands.go new file mode 100644 index 000000000..c6259c483 --- /dev/null +++ b/plugins/registry/src/commands/commands.go @@ -0,0 +1,57 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strconv" + "strings" + + "github.com/dokku/dokku/plugins/common" +) + +const ( + helpHeader = `Usage: dokku registry[:COMMAND] + +Manage registry settings for an app + +Additional commands:` + + helpContent = ` + registry:login [--password-stdin] [], Login to a docker registry + registry:report [] [], Displays a registry report for one or more apps + registry:set (), Set or clear a registry property for an app +` +) + +func main() { + flag.Usage = usage + flag.Parse() + + cmd := flag.Arg(0) + switch cmd { + case "registry", "registry:help": + usage() + case "help": + command := common.NewShellCmd(fmt.Sprintf("ps -o command= %d", os.Getppid())) + command.ShowOutput = false + output, err := command.Output() + + if err == nil && strings.Contains(string(output), "--all") { + fmt.Println(helpContent) + } else { + fmt.Print("\n registry, Manage registry settings for an app\n") + } + default: + dokkuNotImplementExitCode, err := strconv.Atoi(os.Getenv("DOKKU_NOT_IMPLEMENTED_EXIT")) + if err != nil { + fmt.Println("failed to retrieve DOKKU_NOT_IMPLEMENTED_EXIT environment variable") + dokkuNotImplementExitCode = 10 + } + os.Exit(dokkuNotImplementExitCode) + } +} + +func usage() { + common.CommandUsage(helpHeader, helpContent) +} diff --git a/plugins/registry/src/subcommands/subcommands.go b/plugins/registry/src/subcommands/subcommands.go new file mode 100644 index 000000000..0ed00197b --- /dev/null +++ b/plugins/registry/src/subcommands/subcommands.go @@ -0,0 +1,58 @@ +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/registry" + + flag "github.com/spf13/pflag" +) + +// main entrypoint to all subcommands +func main() { + parts := strings.Split(os.Args[0], "/") + subcommand := parts[len(parts)-1] + + var err error + switch subcommand { + case "login": + args := flag.NewFlagSet("registry:login", flag.ExitOnError) + passwordStdin := args.Bool("password-stdin", false, "--password-stdin: read password from stdin") + args.Parse(os.Args[2:]) + server := args.Arg(0) + username := args.Arg(1) + password := args.Arg(2) + err = registry.CommandLogin(server, username, password, *passwordStdin) + case "report": + args := flag.NewFlagSet("registry:report", flag.ExitOnError) + format := args.String("format", "stdout", "format: [ stdout | json ]") + osArgs, infoFlag, flagErr := common.ParseReportArgs("registry", os.Args[2:]) + if flagErr == nil { + args.Parse(osArgs) + appName := args.Arg(0) + err = registry.CommandReport(appName, *format, infoFlag) + } + case "set": + args := flag.NewFlagSet("registry:set", flag.ExitOnError) + global := args.Bool("global", false, "--global: set a global property") + args.Parse(os.Args[2:]) + appName := args.Arg(0) + property := args.Arg(1) + value := args.Arg(2) + if *global { + appName = "--global" + property = args.Arg(0) + value = args.Arg(1) + } + err = registry.CommandSet(appName, property, value) + default: + err = fmt.Errorf("Invalid plugin subcommand call: %s", subcommand) + } + + if err != nil { + common.LogFailWithError(err) + } +} diff --git a/plugins/registry/src/triggers/triggers.go b/plugins/registry/src/triggers/triggers.go new file mode 100644 index 000000000..7ab5872b1 --- /dev/null +++ b/plugins/registry/src/triggers/triggers.go @@ -0,0 +1,36 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/registry" +) + +// main entrypoint to all triggers +func main() { + parts := strings.Split(os.Args[0], "/") + trigger := parts[len(parts)-1] + flag.Parse() + + var err error + switch trigger { + case "install": + err = registry.TriggerInstall() + case "post-delete": + appName := flag.Arg(0) + err = registry.TriggerPostDelete(appName) + case "report": + appName := flag.Arg(0) + err = registry.ReportSingleApp(appName, "", "") + default: + err = fmt.Errorf("Invalid plugin trigger call: %s", trigger) + } + + if err != nil { + common.LogFailWithError(err) + } +} diff --git a/plugins/registry/subcommands.go b/plugins/registry/subcommands.go new file mode 100644 index 000000000..6a6493cb0 --- /dev/null +++ b/plugins/registry/subcommands.go @@ -0,0 +1,65 @@ +package registry + +import ( + "errors" + "io" + "io/ioutil" + "os" + "strings" + + "github.com/dokku/dokku/plugins/common" +) + +func CommandLogin(server string, username string, password string, passwordStdin bool) error { + if passwordStdin { + stdin, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return err + } + + password = strings.TrimSpace(string(stdin)) + } + + command := []string{ + common.DockerBin(), + "login", + "--username", + username, + "--password-stdin", + server, + } + + reader, writer := io.Pipe() + writer.Write([]byte(password)) + loginCmd := common.NewShellCmd(strings.Join(command, " ")) + loginCmd.Command.Stdin = reader + if !loginCmd.Execute() { + return errors.New("Failed to log into registry") + } + + return nil +} + +// CommandReport displays a registry report for one or more apps +func CommandReport(appName string, format string, infoFlag string) error { + if len(appName) == 0 { + apps, err := common.DokkuApps() + if err != nil { + return err + } + for _, appName := range apps { + if err := ReportSingleApp(appName, format, infoFlag); err != nil { + return err + } + } + return nil + } + + return ReportSingleApp(appName, format, infoFlag) +} + +// CommandSet set or clear a registry property for an app +func CommandSet(appName string, property string, value string) error { + common.CommandPropertySet("registry", appName, property, value, DefaultProperties, GlobalProperties) + return nil +} diff --git a/plugins/registry/triggers.go b/plugins/registry/triggers.go new file mode 100644 index 000000000..9039bd773 --- /dev/null +++ b/plugins/registry/triggers.go @@ -0,0 +1,21 @@ +package registry + +import ( + "fmt" + + "github.com/dokku/dokku/plugins/common" +) + +// TriggerInstall runs the install step for the registry plugin +func TriggerInstall() error { + if err := common.PropertySetup("registry"); err != nil { + return fmt.Errorf("Unable to install the registry plugin: %s", err.Error()) + } + + return nil +} + +// TriggerPostDelete destroys the registry property for a given app container +func TriggerPostDelete(appName string) error { + return common.PropertyDestroy("registry", appName) +} From 9e6091e665a37dfdf8ec07144b6a6e59f9cf2e35 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 4 Mar 2021 22:15:26 -0500 Subject: [PATCH 03/36] feat: add a new hook for global builder-release --- plugins/builder-dockerfile/builder-release | 1 + plugins/builder-herokuish/builder-release | 1 + plugins/builder-pack/builder-release | 1 + 3 files changed, 3 insertions(+) diff --git a/plugins/builder-dockerfile/builder-release b/plugins/builder-dockerfile/builder-release index 8e92cff53..18dcf49f8 100755 --- a/plugins/builder-dockerfile/builder-release +++ b/plugins/builder-dockerfile/builder-release @@ -17,6 +17,7 @@ trigger-builder-dockerfile-builder-release() { local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") docker-image-labeler --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku "$IMAGE" plugn trigger post-release-dockerfile "$APP" "$IMAGE_TAG" + plugn trigger post-release-builder "$BUILDER_TYPE" "$APP" "$IMAGE" } trigger-builder-dockerfile-builder-release "$@" diff --git a/plugins/builder-herokuish/builder-release b/plugins/builder-herokuish/builder-release index 9609c390f..59d0eca7e 100755 --- a/plugins/builder-herokuish/builder-release +++ b/plugins/builder-herokuish/builder-release @@ -42,6 +42,7 @@ trigger-builder-herokuish-builder-release() { docker-image-labeler --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku "$IMAGE" plugn trigger post-release-buildpack "$APP" "$IMAGE_TAG" + plugn trigger post-release-builder "$BUILDER_TYPE" "$APP" "$IMAGE" } trigger-builder-herokuish-builder-release "$@" diff --git a/plugins/builder-pack/builder-release b/plugins/builder-pack/builder-release index a5819abd0..1cce0c6aa 100755 --- a/plugins/builder-pack/builder-release +++ b/plugins/builder-pack/builder-release @@ -17,6 +17,7 @@ trigger-builder-pack-builder-release() { local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") docker-image-labeler --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku "$IMAGE" plugn trigger post-release-pack "$APP" "$IMAGE_TAG" + plugn trigger post-release-builder "$BUILDER_TYPE" "$APP" "$IMAGE" } trigger-builder-pack-builder-release "$@" From f8edf6b07a2c0433440b047c163665f0be235bfd Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 4 Mar 2021 22:15:52 -0500 Subject: [PATCH 04/36] docs: move schedulers to live under deployment --- .../schedulers/alternate-schedulers.md | 0 docs/{advanced-usage => deployment}/schedulers/docker-local.md | 0 docs/viewdocs.json | 3 +++ 3 files changed, 3 insertions(+) rename docs/{advanced-usage => deployment}/schedulers/alternate-schedulers.md (100%) rename docs/{advanced-usage => deployment}/schedulers/docker-local.md (100%) diff --git a/docs/advanced-usage/schedulers/alternate-schedulers.md b/docs/deployment/schedulers/alternate-schedulers.md similarity index 100% rename from docs/advanced-usage/schedulers/alternate-schedulers.md rename to docs/deployment/schedulers/alternate-schedulers.md diff --git a/docs/advanced-usage/schedulers/docker-local.md b/docs/deployment/schedulers/docker-local.md similarity index 100% rename from docs/advanced-usage/schedulers/docker-local.md rename to docs/deployment/schedulers/docker-local.md diff --git a/docs/viewdocs.json b/docs/viewdocs.json index dcb03d039..8f552acdf 100644 --- a/docs/viewdocs.json +++ b/docs/viewdocs.json @@ -38,6 +38,9 @@ "dokku-events-logs": "advanced-usage/event-logs/", "dokku-storage": "advanced-usage/persistent-storage/", + "advanced-usage/schedulers/alternate-schedulers": "deployment/schedulers/alternate-schedulers", + "advanced-usage/schedulers/docker-local": "deployment/schedulers/docker-local", + "community/tutorials/deploying-with-gitlab-ci": "deployment/continuous-integration/gitlab-ci/", "plugins": "community/plugins/" From 1b325435b83622996b65f046cd73378108154b86 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 4 Mar 2021 22:16:04 -0500 Subject: [PATCH 05/36] docs: add initial registry docs --- docs/deployment/registry-management.md | 67 ++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 docs/deployment/registry-management.md diff --git a/docs/deployment/registry-management.md b/docs/deployment/registry-management.md new file mode 100644 index 000000000..c00712a4d --- /dev/null +++ b/docs/deployment/registry-management.md @@ -0,0 +1,67 @@ +# Registry Management + +> New as of 0.25.0 + +``` +registry:login [--password-stdin] [] # Login to a docker registry +registry:report [] [] # Displays a registry report for one or more apps +registry:set () # Set or clear a registry property for an app +``` + +The registry plugin enables interacting with remote registries, which is useful when either deploying images via `git:from-image` or when interacting with custom schedulers to deploy built image artifacts. + +## Usage + +### Logging into a registry + +The `registry:login` command can be used to log into a docker registry. The following are examples for logging into various common registries: + +```shell +# hub.docker.com +dokku registry:login docker.io $USERNAME $PASSWORD + +# digitalocean +# the username and password are both defined as the same api token +dokku registry:login registry.digitalocean.com $DIGITALOCEAN_API_TOKEN $DIGITALOCEAN_API_TOKEN + +# github container registry +# see the following link for information on retrieving a personal access token +# https://docs.github.com/en/packages/guides/pushing-and-pulling-docker-images#authenticating-to-github-container-registry +dokku registry:login ghcr.io $USERNAME $REGISTRY_PAT_TOKEN + +# quay +# a robot user may be used to login +dokku registry:login quay.io $USERNAME $PASSWORD +``` + +For security reasons, the password may also be specified as stdin by specifying the `--password-stdin` flag. This is supported regardless of the registry being logged into. + +```shell +echo "$PASSWORD" | dokku registry:login docker.io $USERNAME +``` + +For certain Docker registries - such as Amazon ECR or Google's GCR registries - users may instead wish to use a docker credential helper to automatically authenticate against a server; please see the documentation regarding the credential helper in question for further setup instructions. + +### Pushing images on build + +To push the image on release, set the `push-on-release` property to `true` via the `registry:set` command. The default value for this property is `false`. + +```shell +dokku registry:set node-js-app push-on-release true +``` + +This property can be set for a single app or globally via the `--global` flag. When set globally, the app-specific value will always overide the global value. The default global value for this property is `false`. + +```shell +dokku registry:set --global push-on-release true +``` + +Setting the property value to an empty string will reset the value to the system default. Resetting the value can be done per app or globally. + +```shell +# per-app +dokku registry:set node-js-app push-on-release + +# globally +dokku registry:set --global push-on-release +``` From a6e4949264a97be29f16c469d282830678d68fc7 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 4 Mar 2021 22:16:21 -0500 Subject: [PATCH 06/36] refactor: drop property --- plugins/registry/registry.go | 16 ++++++------- plugins/registry/report.go | 46 +++++++++--------------------------- 2 files changed, 18 insertions(+), 44 deletions(-) diff --git a/plugins/registry/registry.go b/plugins/registry/registry.go index 32dcded7b..88cf04481 100644 --- a/plugins/registry/registry.go +++ b/plugins/registry/registry.go @@ -3,18 +3,16 @@ package registry var ( // DefaultProperties is a map of all valid network properties with corresponding default property values DefaultProperties = map[string]string{ - "create-repository": "false", - "disable-delete-warning": "false", - "image-repo": "", - "push-on-release": "false", - "server": "", + "create-repository": "false", + "image-repo": "", + "push-on-release": "false", + "server": "", } // GlobalProperties is a map of all valid global network properties GlobalProperties = map[string]bool{ - "create-repository": true, - "disable-delete-warning": true, - "push-on-release": true, - "server": true, + "create-repository": true, + "push-on-release": true, + "server": true, } ) diff --git a/plugins/registry/report.go b/plugins/registry/report.go index ad077cde8..61d4232cb 100644 --- a/plugins/registry/report.go +++ b/plugins/registry/report.go @@ -11,20 +11,17 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { } flags := map[string]common.ReportFunc{ - "--registry-computed-create-repository": reportComputedCreateRepository, - "--registry-global-create-repository": reportGlobalCreateRepository, - "--registry-create-repository": reportCreateRepository, - "--registry-computed-disable-delete-warning": reportComputedDisableDeleteWarning, - "--registry-global-disable-delete-warning": reportGlobalDisableDeleteWarning, - "--registry-disable-delete-warning": reportDisableDeleteWarning, - "--registry-image-repo": reportImageRepo, - "--registry-computed-push-on-release": reportComputedPushOnRelease, - "--registry-global-push-on-release": reportGlobalPushOnRelease, - "--registry-push-on-release": reportPushOnRelease, - "--registry-computed-server": reportComputedServer, - "--registry-global-server": reportGlobalServer, - "--registry-server": reportServer, - "--registry-tag-version": reportTagVersion, + "--registry-computed-create-repository": reportComputedCreateRepository, + "--registry-global-create-repository": reportGlobalCreateRepository, + "--registry-create-repository": reportCreateRepository, + "--registry-image-repo": reportImageRepo, + "--registry-computed-push-on-release": reportComputedPushOnRelease, + "--registry-global-push-on-release": reportGlobalPushOnRelease, + "--registry-push-on-release": reportPushOnRelease, + "--registry-computed-server": reportComputedServer, + "--registry-global-server": reportGlobalServer, + "--registry-server": reportServer, + "--registry-tag-version": reportTagVersion, } flagKeys := []string{} @@ -59,27 +56,6 @@ func reportCreateRepository(appName string) string { return common.PropertyGet("registry", appName, "create-repository") } -func reportComputedDisableDeleteWarning(appName string) string { - value := reportDisableDeleteWarning(appName) - if value == "" { - value = reportGlobalDisableDeleteWarning(appName) - } - - if value == "" { - value = DefaultProperties["disable-delete-warning"] - } - - return value -} - -func reportGlobalDisableDeleteWarning(appName string) string { - return common.PropertyGet("registry", "--global", "disable-delete-warning") -} - -func reportDisableDeleteWarning(appName string) string { - return common.PropertyGet("registry", appName, "disable-delete-warning") -} - func reportImageRepo(appName string) string { return common.PropertyGet("registry", appName, "image-repo") } From 336e901c1ee25278e3a374552896602a83fb2d5a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 4 Mar 2021 22:16:40 -0500 Subject: [PATCH 07/36] fix: add some simple validation --- plugins/registry/subcommands.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/registry/subcommands.go b/plugins/registry/subcommands.go index 6a6493cb0..4e7fb2c8d 100644 --- a/plugins/registry/subcommands.go +++ b/plugins/registry/subcommands.go @@ -20,6 +20,16 @@ func CommandLogin(server string, username string, password string, passwordStdin password = strings.TrimSpace(string(stdin)) } + if server == "" { + return errors.New("Missing server argument") + } + if username == "" { + return errors.New("Missing username argument") + } + if password == "" { + return errors.New("Missing password argument") + } + command := []string{ common.DockerBin(), "login", From 5d8340d305f7d05960c40739471f712b78a31848 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 4 Mar 2021 22:16:51 -0500 Subject: [PATCH 08/36] feat: stub out a bunch of code --- plugins/registry/functions.go | 57 +++++++++++++++++++++++ plugins/registry/src/triggers/triggers.go | 6 +++ plugins/registry/triggers.go | 20 ++++++++ 3 files changed, 83 insertions(+) create mode 100644 plugins/registry/functions.go diff --git a/plugins/registry/functions.go b/plugins/registry/functions.go new file mode 100644 index 000000000..5adff1eca --- /dev/null +++ b/plugins/registry/functions.go @@ -0,0 +1,57 @@ +package registry + +import ( + "fmt" + "strconv" + + "github.com/dokku/dokku/plugins/common" +) + +func isPushEnabled(appName string) bool { + return reportComputedPushOnRelease(appName) == "true" +} + +func incrementTagVersion(appName string) (string, error) { + tag := common.PropertyGet("registry", appName, "tag-version") + if tag == "" { + tag = "0" + } + + version, err := strconv.Atoi(tag) + if err != nil { + return "", fmt.Errorf("Unable to convert existing tag version (%s) to integer: %v", tag, err) + } + + version += 1 + common.LogVerboseQuiet(fmt.Sprintf("Bumping tag to %d", version)) + if err = common.PropertyWrite("registry", appName, "tag-version", strconv.Itoa(version)); err != nil { + return "", err + } + + return strconv.Itoa(version), nil +} + +func pushToRegistry(appName string, imageTag string) error { + common.LogVerboseQuiet("Retrieving image info for app") + + // DOKKU_REGISTRY_SERVER=$(fn-registry-remote-repository "$APP") + // IMAGE_REPO=$(fn-registry-image-repo "$APP") + // IMAGE_TAG="$(get_running_image_tag "$APP")" + // IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") + // IMAGE_ID=$(docker inspect --format '{{ .Id }}' "$IMAGE") + + // dokku_log_verbose_quiet "Tagging $IMAGE_REPO:$TAG in registry format" + // docker tag "$IMAGE_ID" "${DOKKU_REGISTRY_SERVER}${IMAGE_REPO}:${TAG}" + // docker tag "$IMAGE_ID" "${IMAGE_REPO}:${TAG}" + + // fn-registry-create-repository "$APP" "$DOKKU_REGISTRY_SERVER" "$IMAGE_REPO" + + common.LogVerboseQuiet("Pushing $IMAGE_REPO:$TAG") + // docker push "${DOKKU_REGISTRY_SERVER}${IMAGE_REPO}:${TAG}" + + common.LogVerboseQuiet("Cleaning up") + // fn-registry-image-cleanup "$APP" "$DOKKU_REGISTRY_SERVER" "$IMAGE_REPO" "$IMAGE_TAG" "$TAG" + + common.LogVerboseQuiet("Image $IMAGE_REPO:$TAG pushed") + return nil +} diff --git a/plugins/registry/src/triggers/triggers.go b/plugins/registry/src/triggers/triggers.go index 7ab5872b1..028fbb2eb 100644 --- a/plugins/registry/src/triggers/triggers.go +++ b/plugins/registry/src/triggers/triggers.go @@ -18,11 +18,17 @@ func main() { var err error switch trigger { + case "deployed-app-repository": + appName := flag.Arg(0) + err = registry.TriggerDeployedAppRepository(appName) case "install": err = registry.TriggerInstall() case "post-delete": appName := flag.Arg(0) err = registry.TriggerPostDelete(appName) + case "post-release-builder": + appName := flag.Arg(1) + err = registry.TriggerPostReleaseBuilder(appName) case "report": appName := flag.Arg(0) err = registry.ReportSingleApp(appName, "", "") diff --git a/plugins/registry/triggers.go b/plugins/registry/triggers.go index 9039bd773..47f2b313f 100644 --- a/plugins/registry/triggers.go +++ b/plugins/registry/triggers.go @@ -6,6 +6,11 @@ import ( "github.com/dokku/dokku/plugins/common" ) +// TriggerDeployedAppRepository outputs the associated registry repository to stdout +func TriggerDeployedAppRepository(appName string) error { + // TODO +} + // TriggerInstall runs the install step for the registry plugin func TriggerInstall() error { if err := common.PropertySetup("registry"); err != nil { @@ -19,3 +24,18 @@ func TriggerInstall() error { func TriggerPostDelete(appName string) error { return common.PropertyDestroy("registry", appName) } + +// TriggerPostReleaseBuilder pushes the image to the remote registry +func TriggerPostReleaseBuilder(appName string) error { + if !isPushEnabled(appName) { + return nil + } + + common.LogInfo1("Pushing image to registry") + imageTag, err := incrementTagVersion(appName) + if err != nil { + return err + } + + return pushToRegistry(appName, imageTag) +} From e68c04b9a617f45400c31aace78118eba843efa7 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 19 Mar 2021 16:31:07 -0400 Subject: [PATCH 09/36] fix: handle lint issues --- plugins/registry/functions.go | 2 +- plugins/registry/subcommands.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/registry/functions.go b/plugins/registry/functions.go index 5adff1eca..99c1dbce1 100644 --- a/plugins/registry/functions.go +++ b/plugins/registry/functions.go @@ -22,7 +22,7 @@ func incrementTagVersion(appName string) (string, error) { return "", fmt.Errorf("Unable to convert existing tag version (%s) to integer: %v", tag, err) } - version += 1 + version++ common.LogVerboseQuiet(fmt.Sprintf("Bumping tag to %d", version)) if err = common.PropertyWrite("registry", appName, "tag-version", strconv.Itoa(version)); err != nil { return "", err diff --git a/plugins/registry/subcommands.go b/plugins/registry/subcommands.go index 4e7fb2c8d..c9d424f53 100644 --- a/plugins/registry/subcommands.go +++ b/plugins/registry/subcommands.go @@ -10,6 +10,7 @@ import ( "github.com/dokku/dokku/plugins/common" ) +// CommandLogin logs a user into the specified server func CommandLogin(server string, username string, password string, passwordStdin bool) error { if passwordStdin { stdin, err := ioutil.ReadAll(os.Stdin) From 3848b467a2006e138c1ee0c714d226c1a3bd70f1 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 19 Mar 2021 16:31:13 -0400 Subject: [PATCH 10/36] fix: add missing return --- plugins/registry/triggers.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/registry/triggers.go b/plugins/registry/triggers.go index 47f2b313f..55050b96d 100644 --- a/plugins/registry/triggers.go +++ b/plugins/registry/triggers.go @@ -9,6 +9,7 @@ import ( // TriggerDeployedAppRepository outputs the associated registry repository to stdout func TriggerDeployedAppRepository(appName string) error { // TODO + return nil } // TriggerInstall runs the install step for the registry plugin From 88cbae2a6de6d45562b88e773c3f54f2705f2b3e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 19 Mar 2021 16:50:32 -0400 Subject: [PATCH 11/36] feat: implement more of the registry plugin --- plugins/registry/functions.go | 19 ++++++++++++++++++ plugins/registry/report.go | 11 +---------- plugins/registry/src/triggers/triggers.go | 6 ++++++ plugins/registry/triggers.go | 24 ++++++++++++++++++++++- 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/plugins/registry/functions.go b/plugins/registry/functions.go index 99c1dbce1..c3495f109 100644 --- a/plugins/registry/functions.go +++ b/plugins/registry/functions.go @@ -3,10 +3,29 @@ package registry import ( "fmt" "strconv" + "strings" "github.com/dokku/dokku/plugins/common" ) +func getRegistryServerForApp(appName string) string { + value := common.PropertyGet("registry", appName, "server") + if value == "" { + value = common.PropertyGet("registry", "--global", "server") + } + + if value == "" { + value = DefaultProperties["server"] + } + + value = strings.TrimSuffix(value, "/") + "/" + if value == "hub.docker.com/" || value == "docker.io/" { + value = "" + } + + return value +} + func isPushEnabled(appName string) bool { return reportComputedPushOnRelease(appName) == "true" } diff --git a/plugins/registry/report.go b/plugins/registry/report.go index 61d4232cb..b2789ea95 100644 --- a/plugins/registry/report.go +++ b/plugins/registry/report.go @@ -82,16 +82,7 @@ func reportPushOnRelease(appName string) string { } func reportComputedServer(appName string) string { - value := reportServer(appName) - if value == "" { - value = reportGlobalServer(appName) - } - - if value == "" { - value = DefaultProperties["server"] - } - - return value + return getRegistryServerForApp(appName) } func reportGlobalServer(appName string) string { diff --git a/plugins/registry/src/triggers/triggers.go b/plugins/registry/src/triggers/triggers.go index 028fbb2eb..85211a403 100644 --- a/plugins/registry/src/triggers/triggers.go +++ b/plugins/registry/src/triggers/triggers.go @@ -18,6 +18,12 @@ func main() { var err error switch trigger { + case "deployed-app-image-repo": + appName := flag.Arg(0) + err = registry.TriggerDeployedAppImageRepo(appName) + case "deployed-app-image-tag": + appName := flag.Arg(0) + err = registry.TriggerDeployedAppImageTag(appName) case "deployed-app-repository": appName := flag.Arg(0) err = registry.TriggerDeployedAppRepository(appName) diff --git a/plugins/registry/triggers.go b/plugins/registry/triggers.go index 55050b96d..c6ecda4bf 100644 --- a/plugins/registry/triggers.go +++ b/plugins/registry/triggers.go @@ -6,9 +6,31 @@ import ( "github.com/dokku/dokku/plugins/common" ) +// TriggerDeployedAppImageRepo outputs the associated image repo to stdout +func TriggerDeployedAppImageRepo(appName string) error { + imageRepo := common.PropertyGet("registry", appName, "image-repo") + if imageRepo == "" { + imageRepo = common.GetAppImageRepo(appName) + } + + fmt.Println(imageRepo) + return nil +} + +// TriggerDeployedAppImageTag outputs the associated image tag to stdout +func TriggerDeployedAppImageTag(appName string) error { + tagVersion := common.PropertyGet("registry", appName, "tag-version") + if tagVersion == "" { + tagVersion = "1" + } + + fmt.Println(tagVersion) + return nil +} + // TriggerDeployedAppRepository outputs the associated registry repository to stdout func TriggerDeployedAppRepository(appName string) error { - // TODO + fmt.Println(getRegistryServerForApp(appName)) return nil } From b33cd266e70f80b3201f5184b78704a34f5052ce Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 13 Jul 2021 11:30:54 -0400 Subject: [PATCH 12/36] chore: drop unused code --- plugins/registry/functions.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/registry/functions.go b/plugins/registry/functions.go index c3495f109..279fc8e2c 100644 --- a/plugins/registry/functions.go +++ b/plugins/registry/functions.go @@ -14,10 +14,6 @@ func getRegistryServerForApp(appName string) string { value = common.PropertyGet("registry", "--global", "server") } - if value == "" { - value = DefaultProperties["server"] - } - value = strings.TrimSuffix(value, "/") + "/" if value == "hub.docker.com/" || value == "docker.io/" { value = "" From 6480957c4f2d6d3bfa3142840f4b03ec062eb890 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 13 Jul 2021 11:31:16 -0400 Subject: [PATCH 13/36] feat: add computed image-repo --- plugins/registry/report.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/registry/report.go b/plugins/registry/report.go index b2789ea95..5c38048f6 100644 --- a/plugins/registry/report.go +++ b/plugins/registry/report.go @@ -14,6 +14,7 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { "--registry-computed-create-repository": reportComputedCreateRepository, "--registry-global-create-repository": reportGlobalCreateRepository, "--registry-create-repository": reportCreateRepository, + "--registry-computed-image-repo": reportComputedImageRepo, "--registry-image-repo": reportImageRepo, "--registry-computed-push-on-release": reportComputedPushOnRelease, "--registry-global-push-on-release": reportGlobalPushOnRelease, @@ -56,6 +57,15 @@ func reportCreateRepository(appName string) string { return common.PropertyGet("registry", appName, "create-repository") } +func reportComputedImageRepo(appName string) string { + imageRepo := reportImageRepo(appName) + if imageRepo == "" { + imageRepo = common.GetAppImageRepo(appName) + } + + return imageRepo +} + func reportImageRepo(appName string) string { return common.PropertyGet("registry", appName, "image-repo") } From 93f819609e9fdf20d131845b95fa7d1c45d0ec96 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 13 Jul 2021 11:31:49 -0400 Subject: [PATCH 14/36] feat: filled in more of pushToRegistry --- plugins/registry/functions.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/registry/functions.go b/plugins/registry/functions.go index 279fc8e2c..fe94f1d58 100644 --- a/plugins/registry/functions.go +++ b/plugins/registry/functions.go @@ -46,16 +46,16 @@ func incrementTagVersion(appName string) (string, error) { return strconv.Itoa(version), nil } -func pushToRegistry(appName string, imageTag string) error { +func pushToRegistry(appName string, tag string) error { common.LogVerboseQuiet("Retrieving image info for app") - // DOKKU_REGISTRY_SERVER=$(fn-registry-remote-repository "$APP") - // IMAGE_REPO=$(fn-registry-image-repo "$APP") - // IMAGE_TAG="$(get_running_image_tag "$APP")" - // IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") - // IMAGE_ID=$(docker inspect --format '{{ .Id }}' "$IMAGE") + registryServer := getRegistryServerForApp(appName) + imageRepo := reportComputedImageRepo(appName) + imageTag, _ := common.GetRunningImageTag(appName) + image := common.GetAppImageName(appName, imageTag, "") + imageID, _ := common.DockerInspect(image, "{{ .Id }}") - // dokku_log_verbose_quiet "Tagging $IMAGE_REPO:$TAG in registry format" + common.LogVerboseQuiet(fmt.Sprintf("Tagging $IMAGE_REPO:%s in registry format", tag)) // docker tag "$IMAGE_ID" "${DOKKU_REGISTRY_SERVER}${IMAGE_REPO}:${TAG}" // docker tag "$IMAGE_ID" "${IMAGE_REPO}:${TAG}" From dc04ab912d86db9d05e6d1668b014e83fff1c3c8 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 2 Aug 2021 00:42:54 -0400 Subject: [PATCH 15/36] feat: tag image --- plugins/registry/functions.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/plugins/registry/functions.go b/plugins/registry/functions.go index fe94f1d58..9cf36e696 100644 --- a/plugins/registry/functions.go +++ b/plugins/registry/functions.go @@ -1,10 +1,12 @@ package registry import ( + "errors" "fmt" "strconv" "strings" + "github.com/codeskyblue/go-sh" "github.com/dokku/dokku/plugins/common" ) @@ -56,8 +58,15 @@ func pushToRegistry(appName string, tag string) error { imageID, _ := common.DockerInspect(image, "{{ .Id }}") common.LogVerboseQuiet(fmt.Sprintf("Tagging $IMAGE_REPO:%s in registry format", tag)) - // docker tag "$IMAGE_ID" "${DOKKU_REGISTRY_SERVER}${IMAGE_REPO}:${TAG}" - // docker tag "$IMAGE_ID" "${IMAGE_REPO}:${TAG}" + if !dockerTag(imageID, fmt.Sprintf("%s%s:%s", registryServer, imageRepo, tag)) { + // TODO: better error + return errors.New("Unable to tag image") + } + + if !dockerTag(imageID, fmt.Sprintf("%s:%s", imageRepo, tag)) { + // TODO: better error + return errors.New("Unable to tag image") + } // fn-registry-create-repository "$APP" "$DOKKU_REGISTRY_SERVER" "$IMAGE_REPO" @@ -70,3 +79,14 @@ func pushToRegistry(appName string, tag string) error { common.LogVerboseQuiet("Image $IMAGE_REPO:$TAG pushed") return nil } + +func dockerTag(imageID string, imageTag string) bool { + cmd := sh.Command(common.DockerBin(), "image", "tag", imageID, imageTag) + cmd.Stdout = nil + cmd.Stderr = nil + if err := cmd.Run(); err != nil { + return false + } + + return true +} From 1a99271fa84a3fd924ce0ed0a7503a8172802b32 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 2 Aug 2021 02:45:41 -0400 Subject: [PATCH 16/36] feat: push tag --- plugins/registry/functions.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/plugins/registry/functions.go b/plugins/registry/functions.go index 9cf36e696..83bd0cac8 100644 --- a/plugins/registry/functions.go +++ b/plugins/registry/functions.go @@ -3,6 +3,7 @@ package registry import ( "errors" "fmt" + "os" "strconv" "strings" @@ -71,7 +72,10 @@ func pushToRegistry(appName string, tag string) error { // fn-registry-create-repository "$APP" "$DOKKU_REGISTRY_SERVER" "$IMAGE_REPO" common.LogVerboseQuiet("Pushing $IMAGE_REPO:$TAG") - // docker push "${DOKKU_REGISTRY_SERVER}${IMAGE_REPO}:${TAG}" + if !dockerPush(fmt.Sprintf("%s%s:%s", registryServer, imageRepo, tag)) { + // TODO: better error + return errors.New("Unable to push image") + } common.LogVerboseQuiet("Cleaning up") // fn-registry-image-cleanup "$APP" "$DOKKU_REGISTRY_SERVER" "$IMAGE_REPO" "$IMAGE_TAG" "$TAG" @@ -90,3 +94,14 @@ func dockerTag(imageID string, imageTag string) bool { return true } + +func dockerPush(imageTag string) bool { + cmd := sh.Command(common.DockerBin(), "image", "push", imageTag) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return false + } + + return true +} From 36c96be0916716df7cdf041b7ce6c171952ea593 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 3 Aug 2021 16:20:19 -0400 Subject: [PATCH 17/36] fix: add missing event hook --- plugins/20_events/post-release-builder | 1 + 1 file changed, 1 insertion(+) create mode 120000 plugins/20_events/post-release-builder diff --git a/plugins/20_events/post-release-builder b/plugins/20_events/post-release-builder new file mode 120000 index 000000000..5178a749f --- /dev/null +++ b/plugins/20_events/post-release-builder @@ -0,0 +1 @@ +hook \ No newline at end of file From 9a845664e10224ed1245fd52bb3077fe904ee6fe Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 3 Aug 2021 16:22:57 -0400 Subject: [PATCH 18/36] feat: implement image cleanup --- plugins/common/docker.go | 58 +++++++++++++++++------------------ plugins/registry/functions.go | 44 +++++++++++++++++++------- 2 files changed, 62 insertions(+), 40 deletions(-) diff --git a/plugins/common/docker.go b/plugins/common/docker.go index fe9dfa4bd..9d914e365 100644 --- a/plugins/common/docker.go +++ b/plugins/common/docker.go @@ -210,7 +210,7 @@ func DockerCleanup(appName string, forceCleanup bool) error { } // delete dangling images - imageIDs, _ := listDanglingImages(appName) + imageIDs, _ := ListDanglingImages(appName) if len(imageIDs) > 0 { RemoveImages(imageIDs) } @@ -301,6 +301,34 @@ func IsImageHerokuishBased(image string, appName string) bool { return output != "" } +func ListDanglingImages(appName string) ([]string, error) { + command := []string{ + DockerBin(), + "image", + "list", + "--quiet", + "--filter", + "dangling=true", + } + + if appName != "" { + command = append(command, []string{"--filter", fmt.Sprintf("label=com.dokku.app-name=%v", appName)}...) + } + + var stderr bytes.Buffer + listCmd := NewShellCmd(strings.Join(command, " ")) + listCmd.ShowOutput = false + listCmd.Command.Stderr = &stderr + b, err := listCmd.Output() + + if err != nil { + return []string{}, errors.New(strings.TrimSpace(stderr.String())) + } + + output := strings.Split(strings.TrimSpace(string(b[:])), "\n") + return output, nil +} + // RemoveImages removes images by ID func RemoveImages(imageIDs []string) { command := []string{ @@ -356,34 +384,6 @@ func listContainers(status string, appName string) ([]string, error) { return output, nil } -func listDanglingImages(appName string) ([]string, error) { - command := []string{ - DockerBin(), - "image", - "list", - "--quiet", - "--filter", - "dangling=true", - } - - if appName != "" { - command = append(command, []string{"--filter", fmt.Sprintf("label=com.dokku.app-name=%v", appName)}...) - } - - var stderr bytes.Buffer - listCmd := NewShellCmd(strings.Join(command, " ")) - listCmd.ShowOutput = false - listCmd.Command.Stderr = &stderr - b, err := listCmd.Output() - - if err != nil { - return []string{}, errors.New(strings.TrimSpace(stderr.String())) - } - - output := strings.Split(strings.TrimSpace(string(b[:])), "\n") - return output, nil -} - func pruneUnusedImages(appName string) { command := []string{ DockerBin(), diff --git a/plugins/registry/functions.go b/plugins/registry/functions.go index 83bd0cac8..14eb40dc1 100644 --- a/plugins/registry/functions.go +++ b/plugins/registry/functions.go @@ -29,7 +29,7 @@ func isPushEnabled(appName string) bool { return reportComputedPushOnRelease(appName) == "true" } -func incrementTagVersion(appName string) (string, error) { +func incrementTagVersion(appName string) (int, error) { tag := common.PropertyGet("registry", appName, "tag-version") if tag == "" { tag = "0" @@ -37,19 +37,19 @@ func incrementTagVersion(appName string) (string, error) { version, err := strconv.Atoi(tag) if err != nil { - return "", fmt.Errorf("Unable to convert existing tag version (%s) to integer: %v", tag, err) + return 0, fmt.Errorf("Unable to convert existing tag version (%s) to integer: %v", tag, err) } version++ common.LogVerboseQuiet(fmt.Sprintf("Bumping tag to %d", version)) if err = common.PropertyWrite("registry", appName, "tag-version", strconv.Itoa(version)); err != nil { - return "", err + return 0, err } - return strconv.Itoa(version), nil + return version, nil } -func pushToRegistry(appName string, tag string) error { +func pushToRegistry(appName string, tag int) error { common.LogVerboseQuiet("Retrieving image info for app") registryServer := getRegistryServerForApp(appName) @@ -58,27 +58,29 @@ func pushToRegistry(appName string, tag string) error { image := common.GetAppImageName(appName, imageTag, "") imageID, _ := common.DockerInspect(image, "{{ .Id }}") - common.LogVerboseQuiet(fmt.Sprintf("Tagging $IMAGE_REPO:%s in registry format", tag)) - if !dockerTag(imageID, fmt.Sprintf("%s%s:%s", registryServer, imageRepo, tag)) { + common.LogVerboseQuiet(fmt.Sprintf("Tagging $IMAGE_REPO:%d in registry format", tag)) + if !dockerTag(imageID, fmt.Sprintf("%s%s:%d", registryServer, imageRepo, tag)) { // TODO: better error return errors.New("Unable to tag image") } - if !dockerTag(imageID, fmt.Sprintf("%s:%s", imageRepo, tag)) { + if !dockerTag(imageID, fmt.Sprintf("%s:%d", imageRepo, tag)) { // TODO: better error return errors.New("Unable to tag image") } - // fn-registry-create-repository "$APP" "$DOKKU_REGISTRY_SERVER" "$IMAGE_REPO" + // For the future, we should also add the ability to create the remote repository + // This is only really important for registries that do not support creation on push + // Examples include AWS and Quay.io common.LogVerboseQuiet("Pushing $IMAGE_REPO:$TAG") - if !dockerPush(fmt.Sprintf("%s%s:%s", registryServer, imageRepo, tag)) { + if !dockerPush(fmt.Sprintf("%s%s:%d", registryServer, imageRepo, tag)) { // TODO: better error return errors.New("Unable to push image") } common.LogVerboseQuiet("Cleaning up") - // fn-registry-image-cleanup "$APP" "$DOKKU_REGISTRY_SERVER" "$IMAGE_REPO" "$IMAGE_TAG" "$TAG" + imageCleanup(appName, registryServer, imageRepo, imageTag, tag) common.LogVerboseQuiet("Image $IMAGE_REPO:$TAG pushed") return nil @@ -105,3 +107,23 @@ func dockerPush(imageTag string) bool { return true } + +func imageCleanup(appName string, registryServer string, imageRepo string, imageTag string, tag int) { + // # keep last two images in place + oldTag := tag - 1 + tenImagesAgoTag := tag - 12 + + imagesToRemove := []string{} + for oldTag > 0 { + imagesToRemove = append(imagesToRemove, fmt.Sprintf("%s%s:%d", registryServer, imageRepo, oldTag)) + imagesToRemove = append(imagesToRemove, fmt.Sprintf("%s:%d", imageRepo, oldTag)) + oldTag = oldTag - 1 + if tenImagesAgoTag == oldTag { + break + } + } + + imageIDs, _ := common.ListDanglingImages(appName) + imagesToRemove = append(imagesToRemove, imageIDs...) + common.RemoveImages(imagesToRemove) +} From 455a80e09d8ce274a2066e166fdf1659ad5cfb2d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 3 Aug 2021 16:23:10 -0400 Subject: [PATCH 19/36] chore: drop unimplemented code --- plugins/registry/registry.go | 12 +++++------ plugins/registry/report.go | 42 ++++++++---------------------------- 2 files changed, 14 insertions(+), 40 deletions(-) diff --git a/plugins/registry/registry.go b/plugins/registry/registry.go index 88cf04481..301219437 100644 --- a/plugins/registry/registry.go +++ b/plugins/registry/registry.go @@ -3,16 +3,14 @@ package registry var ( // DefaultProperties is a map of all valid network properties with corresponding default property values DefaultProperties = map[string]string{ - "create-repository": "false", - "image-repo": "", - "push-on-release": "false", - "server": "", + "image-repo": "", + "push-on-release": "false", + "server": "", } // GlobalProperties is a map of all valid global network properties GlobalProperties = map[string]bool{ - "create-repository": true, - "push-on-release": true, - "server": true, + "push-on-release": true, + "server": true, } ) diff --git a/plugins/registry/report.go b/plugins/registry/report.go index 5c38048f6..b02b25ef2 100644 --- a/plugins/registry/report.go +++ b/plugins/registry/report.go @@ -11,18 +11,15 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { } flags := map[string]common.ReportFunc{ - "--registry-computed-create-repository": reportComputedCreateRepository, - "--registry-global-create-repository": reportGlobalCreateRepository, - "--registry-create-repository": reportCreateRepository, - "--registry-computed-image-repo": reportComputedImageRepo, - "--registry-image-repo": reportImageRepo, - "--registry-computed-push-on-release": reportComputedPushOnRelease, - "--registry-global-push-on-release": reportGlobalPushOnRelease, - "--registry-push-on-release": reportPushOnRelease, - "--registry-computed-server": reportComputedServer, - "--registry-global-server": reportGlobalServer, - "--registry-server": reportServer, - "--registry-tag-version": reportTagVersion, + "--registry-computed-image-repo": reportComputedImageRepo, + "--registry-image-repo": reportImageRepo, + "--registry-computed-push-on-release": reportComputedPushOnRelease, + "--registry-global-push-on-release": reportGlobalPushOnRelease, + "--registry-push-on-release": reportPushOnRelease, + "--registry-computed-server": reportComputedServer, + "--registry-global-server": reportGlobalServer, + "--registry-server": reportServer, + "--registry-tag-version": reportTagVersion, } flagKeys := []string{} @@ -36,27 +33,6 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { return common.ReportSingleApp("registry", appName, infoFlag, infoFlags, flagKeys, format, trimPrefix, uppercaseFirstCharacter) } -func reportComputedCreateRepository(appName string) string { - value := reportCreateRepository(appName) - if value == "" { - value = reportGlobalCreateRepository(appName) - } - - if value == "" { - value = DefaultProperties["create-repository"] - } - - return value -} - -func reportGlobalCreateRepository(appName string) string { - return common.PropertyGet("registry", "--global", "create-repository") -} - -func reportCreateRepository(appName string) string { - return common.PropertyGet("registry", appName, "create-repository") -} - func reportComputedImageRepo(appName string) string { imageRepo := reportImageRepo(appName) if imageRepo == "" { From 9930c0493d1446ddd6a3799b1bc5070ab0776836 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 3 Aug 2021 16:23:33 -0400 Subject: [PATCH 20/36] docs: update docs for new features --- docs/appendices/0.25.0-migration-guide.md | 16 +++++- docs/deployment/registry-management.md | 60 +++++++++++++++++++++++ docs/development/plugin-triggers.md | 24 +++++++++ 3 files changed, 98 insertions(+), 2 deletions(-) diff --git a/docs/appendices/0.25.0-migration-guide.md b/docs/appendices/0.25.0-migration-guide.md index 9ad58a916..f61892a97 100644 --- a/docs/appendices/0.25.0-migration-guide.md +++ b/docs/appendices/0.25.0-migration-guide.md @@ -1,16 +1,28 @@ # 0.25.0 Migration Guide -## Changes +## Registry Plugin + +The [dokku-registry](https://github.com/dokku/dokku-registry) plugin is now built-in. This comes with a few changes: + +- Builder plugins should call `post-release-builder` at the end of the build. +- The `push` and `pull` command are not implemented. Users wishing to deploy a remote image should use `git:from-image`. Image pushing is not available at this time. +- At this time, remote docker repositories are not automatically created for AWS, and users must create those repositories for their applications as necessary. This may be implemented in the future. +- Docker images are only pushed when configured to do so. See the [registry management documentation](/docs/deployment/registry-management.md) for more details. + +## Other + +### Changes - The network plugin can now set an `initial-network` for all containers on creation. This is a replacement for specifying the `--network` flag via the `docker-options` plugin. Please see the [network documentation](/docs/networking/network.md#attaching-an-app-to-a-network) for more information. - The `dokku run` command now always removes the ephemeral container on exit. Users that need a persistent container should instead specify a `console` process type in their `Procfile` specifying an available shell (usually either `bash` or `sh`) and scale that container appropriately. -## Deprecations +### Deprecations - In previous versions of Dokku, the only way to specify a custom `Dockerfile` was to use the `docker-options` plugin to set the `--file` flag for a docker build. As of 0.25.0, the `builder-dockerfile:set` command should be used instead, as outlined in the [docs here](/docs/deployment/builders/dockerfiles.md#changingthe-dockerfile-location). Usage of the old method should be migrated to the new method. - The `--rm` and ``--rm-container` flags may be specified but no longer have any effect on `dokku run`. - The `--detach` flag is deprecated in favor of the `run:detached` command. - The `DOKKU_SCALE` file is deprecated. Please see the [process management documentation](/docs/processes/process-management.md#manually-managing-process-scaling) for more information on it's replacement with the `formations` key of the `app.json` file. +- The hooks `post-release-buildpack`, `post-release-dockerfile`, and `post-release-pack` are deprecated in favor of `post-release-builder`. See the [plugin triggers documentation](https://dokku.com/docs/development/plugin-triggers/#post-release-builder) for more details. ## Removals diff --git a/docs/deployment/registry-management.md b/docs/deployment/registry-management.md index c00712a4d..5e0da5623 100644 --- a/docs/deployment/registry-management.md +++ b/docs/deployment/registry-management.md @@ -42,6 +42,65 @@ echo "$PASSWORD" | dokku registry:login docker.io $USERNAME For certain Docker registries - such as Amazon ECR or Google's GCR registries - users may instead wish to use a docker credential helper to automatically authenticate against a server; please see the documentation regarding the credential helper in question for further setup instructions. +### Setting a remote server + +To specify a remote server registry for pushes, set the `server` property via the `registry:set` command. The default value for this property is empty string. + +```shell +dokku registry:set node-js-app server docker.io +``` + +This property can be set for a single app or globally via the `--global` flag. When set globally, the app-specific value will always overide the global value. The default global value for this property is empty string. + +```shell +dokku registry:set --global server docker.io +``` + +Setting the property value to an empty string will reset the value to the system default. Resetting the value can be done per app or globally. + +```shell +# per-app +dokku registry:set node-js-app server + +# globally +dokku registry:set --global server +``` + +The following are the values that should be used for common remote servers: + +- Amazon Elastic Container Registry: + - value: `$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com` + - notes: The `$AWS_ACCOUNT_ID` and `$AWS_REGION` should match the values for your account and region, respectively. Additionally, an IAM profile that allows `push` access to the repository specified by `image-repo` should be attached to your Dokku server. +- Azure Container Registry: + - value `$REGISTRY_NAME.azurecr.io` + - notes: The `$AKS_REGISTRY_NAME` should match the name of the registry created on your account. +- Docker Hub: + - value: `docker.io` + - notes: Requires owning the namespace used in the `image-repo` value. +- Digitalocean: + - value: `registry.digitalocean.com` + - notes: Requires setting the correct `image-repo` value for your registry. +- Github Container Registry: + - value: `ghcr.io` + - notes: Requires that the authenticated user has access to the namespace used in the `image-repo` value. +- Quay.io: + - value: `quay.io` + +### Specifying an image repository name + +By default, Dokku uses the value `dokku/$APP_NAME` as the image repository that is pushed and deployed. For certain registries, the `dokku` namespace may not be available to your user. In these cases, the value can be set by changing the value of the `image-repo` property via the `registry:set` command. + +```shell +dokku registry:set node-js-app image-repo my-awesome-prefix/node-js-app +``` + +Setting the property value to an empty string will reset the value to the system default. Resetting the value has to be done per-app. + +```shell +# per-app +dokku registry:set node-js-app push-on-release +``` + ### Pushing images on build To push the image on release, set the `push-on-release` property to `true` via the `registry:set` command. The default value for this property is `false`. @@ -65,3 +124,4 @@ dokku registry:set node-js-app push-on-release # globally dokku registry:set --global push-on-release ``` + diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 1181a9cff..ed02204b2 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1345,8 +1345,28 @@ APP="$1" haproxy-build-config "$APP" ``` +### `post-release-builder` + +> Warning: Image mutation in this trigger may result in an invalid run state, and is heavily discouraged. + +- Description: Invokes a command after the build process is complete. +- Invoked by: builder plugins +- Arguments: `$BUILDER_TYPE $APP $IMAGE` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +BUILDER_TYPE="$1"; APP="$2"; IMAGE=$3 + +# TODO +``` + ### `post-release-buildpack` +> Warning: Deprecated, please use `docker-args-process-build` instead > Warning: Image mutation in this trigger may result in an invalid run state, and is heavily discouraged. - Description: Allows you to run commands after environment variables are set for the release step of the deploy. Only applies to apps using buildpacks. @@ -1366,6 +1386,8 @@ APP="$1"; IMAGE_TAG="$2"; IMAGE=$(get_app_image_name $APP $IMAGE_TAG) ### `post-release-pack` +> Warning: Deprecated, please use `docker-args-process-build` instead + > Warning: The pack plugin trigger apis are under development and may change > between minor releases until the 1.0 release. @@ -1388,6 +1410,8 @@ APP="$1"; IMAGE_TAG="$2"; IMAGE=$(get_app_image_name $APP $IMAGE_TAG) ### `post-release-dockerfile` +> Warning: Deprecated, please use `docker-args-process-build` instead + > Warning: Image mutation in this trigger may result in an invalid run state, and is heavily discouraged. - Description: Allows you to run commands after environment variables are set for the release step of the deploy. Only applies to apps using a dockerfile. From 93bd3cd9e3420e4dea60e1eadce304deca1ad35e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 3 Aug 2021 16:24:59 -0400 Subject: [PATCH 21/36] fix: add godoc --- plugins/common/docker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/common/docker.go b/plugins/common/docker.go index 9d914e365..ca2a19340 100644 --- a/plugins/common/docker.go +++ b/plugins/common/docker.go @@ -301,6 +301,7 @@ func IsImageHerokuishBased(image string, appName string) bool { return output != "" } +// ListDanglingImages lists all dangling image ids for a given app func ListDanglingImages(appName string) ([]string, error) { command := []string{ DockerBin(), From 1f2e2ba61dbf962789387640cdba716552f93597 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 01:11:50 -0400 Subject: [PATCH 22/36] docs: update registry docs for clarity --- docs/deployment/registry-management.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/deployment/registry-management.md b/docs/deployment/registry-management.md index 5e0da5623..0e6c56e52 100644 --- a/docs/deployment/registry-management.md +++ b/docs/deployment/registry-management.md @@ -37,14 +37,14 @@ dokku registry:login quay.io $USERNAME $PASSWORD For security reasons, the password may also be specified as stdin by specifying the `--password-stdin` flag. This is supported regardless of the registry being logged into. ```shell -echo "$PASSWORD" | dokku registry:login docker.io $USERNAME +echo "$PASSWORD" | dokku registry:login --password-stdin docker.io $USERNAME ``` For certain Docker registries - such as Amazon ECR or Google's GCR registries - users may instead wish to use a docker credential helper to automatically authenticate against a server; please see the documentation regarding the credential helper in question for further setup instructions. ### Setting a remote server -To specify a remote server registry for pushes, set the `server` property via the `registry:set` command. The default value for this property is empty string. +To specify a remote server registry for pushes, set the `server` property via the `registry:set` command. The default value for this property is empty string. Setting the value to `docker.io` or `hub.docker.com` will result in the computed value being empty string (as that is the default, implicit registry), while any non-zero length value will have a `/` appended to it if there is not one already. ```shell dokku registry:set node-js-app server docker.io @@ -69,22 +69,22 @@ dokku registry:set --global server The following are the values that should be used for common remote servers: - Amazon Elastic Container Registry: - - value: `$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com` + - value: `$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/` - notes: The `$AWS_ACCOUNT_ID` and `$AWS_REGION` should match the values for your account and region, respectively. Additionally, an IAM profile that allows `push` access to the repository specified by `image-repo` should be attached to your Dokku server. - Azure Container Registry: - - value `$REGISTRY_NAME.azurecr.io` + - value `$REGISTRY_NAME.azurecr.io/` - notes: The `$AKS_REGISTRY_NAME` should match the name of the registry created on your account. - Docker Hub: - - value: `docker.io` + - value: `docker.io/` - notes: Requires owning the namespace used in the `image-repo` value. - Digitalocean: - - value: `registry.digitalocean.com` + - value: `registry.digitalocean.com/` - notes: Requires setting the correct `image-repo` value for your registry. - Github Container Registry: - - value: `ghcr.io` + - value: `ghcr.io/` - notes: Requires that the authenticated user has access to the namespace used in the `image-repo` value. - Quay.io: - - value: `quay.io` + - value: `quay.io/` ### Specifying an image repository name @@ -103,7 +103,7 @@ dokku registry:set node-js-app push-on-release ### Pushing images on build -To push the image on release, set the `push-on-release` property to `true` via the `registry:set` command. The default value for this property is `false`. +To push the image on release, set the `push-on-release` property to `true` via the `registry:set` command. The default value for this property is `false`. Setting the property to `true` will result in the imag being tagged with an ID that is incremented with every release. This tag will be what is used for running app code. ```shell dokku registry:set node-js-app push-on-release true From c07b3161fccd858b999cc23e597f59460d87c068 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 01:11:59 -0400 Subject: [PATCH 23/36] fix: build all registry triggers --- plugins/registry/.gitignore | 1 + plugins/registry/Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/registry/.gitignore b/plugins/registry/.gitignore index dee017e8f..7b4bc330f 100644 --- a/plugins/registry/.gitignore +++ b/plugins/registry/.gitignore @@ -7,3 +7,4 @@ /install /post-* /report +/deployed-* diff --git a/plugins/registry/Makefile b/plugins/registry/Makefile index bdfd0efe5..4953058e4 100644 --- a/plugins/registry/Makefile +++ b/plugins/registry/Makefile @@ -1,5 +1,5 @@ SUBCOMMANDS = subcommands/login subcommands/report subcommands/set -TRIGGERS = triggers/install triggers/post-delete triggers/report +TRIGGERS = triggers/deployed-app-image-repo triggers/deployed-app-image-tag triggers/deployed-app-repository triggers/install triggers/post-delete triggers/post-release-builder triggers/report BUILD = commands subcommands triggers PLUGIN_NAME = registry From 13608df0efb53f9fdbe1d3c98ea27fd222610188 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 01:12:10 -0400 Subject: [PATCH 24/36] fix: properly pass in password to docker login command --- plugins/registry/subcommands.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/registry/subcommands.go b/plugins/registry/subcommands.go index c9d424f53..beabc002a 100644 --- a/plugins/registry/subcommands.go +++ b/plugins/registry/subcommands.go @@ -1,8 +1,8 @@ package registry import ( + "bytes" "errors" - "io" "io/ioutil" "os" "strings" @@ -40,10 +40,11 @@ func CommandLogin(server string, username string, password string, passwordStdin server, } - reader, writer := io.Pipe() - writer.Write([]byte(password)) + buffer := bytes.Buffer{} + buffer.Write([]byte(password + "\n")) + loginCmd := common.NewShellCmd(strings.Join(command, " ")) - loginCmd.Command.Stdin = reader + loginCmd.Command.Stdin = &buffer if !loginCmd.Execute() { return errors.New("Failed to log into registry") } From d1728a4653c84bf4b858feb962d6d21d714e187b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 01:12:30 -0400 Subject: [PATCH 25/36] fix: avoid returning a numbered image tag if push is not enabled --- plugins/registry/triggers.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/registry/triggers.go b/plugins/registry/triggers.go index c6ecda4bf..00ddcb6c7 100644 --- a/plugins/registry/triggers.go +++ b/plugins/registry/triggers.go @@ -19,6 +19,10 @@ func TriggerDeployedAppImageRepo(appName string) error { // TriggerDeployedAppImageTag outputs the associated image tag to stdout func TriggerDeployedAppImageTag(appName string) error { + if !isPushEnabled(appName) { + return nil + } + tagVersion := common.PropertyGet("registry", appName, "tag-version") if tagVersion == "" { tagVersion = "1" From 39af6359be6f539b9bfac035514dce6f8ee399ce Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 01:12:40 -0400 Subject: [PATCH 26/36] feat: add a Start() helper function --- plugins/common/subprocess.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/common/subprocess.go b/plugins/common/subprocess.go index f7da71901..2f970474d 100644 --- a/plugins/common/subprocess.go +++ b/plugins/common/subprocess.go @@ -61,6 +61,13 @@ func (sc *ShellCmd) Execute() bool { return true } +// Start is a wrapper around exec.Command.Start() +func (sc *ShellCmd) Start() error { + sc.setup() + + return sc.Command.Start() +} + // Output is a lightweight wrapper around exec.Command.Output() func (sc *ShellCmd) Output() ([]byte, error) { sc.setup() From d727dd76ce50500dc862a055a05449950412d3bb Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 01:13:36 -0400 Subject: [PATCH 27/36] fix: ensure we push and cleanup the correct images --- plugins/registry/functions.go | 24 +++++++++++------------ plugins/registry/src/triggers/triggers.go | 3 ++- plugins/registry/triggers.go | 18 +++++++++++++++-- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/plugins/registry/functions.go b/plugins/registry/functions.go index 14eb40dc1..991981248 100644 --- a/plugins/registry/functions.go +++ b/plugins/registry/functions.go @@ -49,17 +49,16 @@ func incrementTagVersion(appName string) (int, error) { return version, nil } -func pushToRegistry(appName string, tag int) error { +func pushToRegistry(appName string, tag int, imageID string, imageRepo string) error { common.LogVerboseQuiet("Retrieving image info for app") registryServer := getRegistryServerForApp(appName) - imageRepo := reportComputedImageRepo(appName) imageTag, _ := common.GetRunningImageTag(appName) - image := common.GetAppImageName(appName, imageTag, "") - imageID, _ := common.DockerInspect(image, "{{ .Id }}") - common.LogVerboseQuiet(fmt.Sprintf("Tagging $IMAGE_REPO:%d in registry format", tag)) - if !dockerTag(imageID, fmt.Sprintf("%s%s:%d", registryServer, imageRepo, tag)) { + fullImage := fmt.Sprintf("%s%s:%d", registryServer, imageRepo, tag) + + common.LogVerboseQuiet(fmt.Sprintf("Tagging %s:%d in registry format", imageRepo, tag)) + if !dockerTag(imageID, fullImage) { // TODO: better error return errors.New("Unable to tag image") } @@ -73,16 +72,17 @@ func pushToRegistry(appName string, tag int) error { // This is only really important for registries that do not support creation on push // Examples include AWS and Quay.io - common.LogVerboseQuiet("Pushing $IMAGE_REPO:$TAG") - if !dockerPush(fmt.Sprintf("%s%s:%d", registryServer, imageRepo, tag)) { + common.LogVerboseQuiet(fmt.Sprintf("Pushing %s", fullImage)) + if !dockerPush(fullImage) { // TODO: better error return errors.New("Unable to push image") } common.LogVerboseQuiet("Cleaning up") - imageCleanup(appName, registryServer, imageRepo, imageTag, tag) + imageCleanup(appName, fmt.Sprintf("%s%s", registryServer, imageRepo), imageTag, tag) + imageCleanup(appName, imageRepo, imageTag, tag) - common.LogVerboseQuiet("Image $IMAGE_REPO:$TAG pushed") + common.LogVerboseQuiet(fmt.Sprintf("Image %s pushed", fullImage)) return nil } @@ -108,14 +108,14 @@ func dockerPush(imageTag string) bool { return true } -func imageCleanup(appName string, registryServer string, imageRepo string, imageTag string, tag int) { +func imageCleanup(appName string, imageRepo string, imageTag string, tag int) { // # keep last two images in place oldTag := tag - 1 tenImagesAgoTag := tag - 12 imagesToRemove := []string{} for oldTag > 0 { - imagesToRemove = append(imagesToRemove, fmt.Sprintf("%s%s:%d", registryServer, imageRepo, oldTag)) + common.LogInfo1(fmt.Sprintf("Removing image: %s:%d", imageRepo, oldTag)) imagesToRemove = append(imagesToRemove, fmt.Sprintf("%s:%d", imageRepo, oldTag)) oldTag = oldTag - 1 if tenImagesAgoTag == oldTag { diff --git a/plugins/registry/src/triggers/triggers.go b/plugins/registry/src/triggers/triggers.go index 85211a403..28d907ff6 100644 --- a/plugins/registry/src/triggers/triggers.go +++ b/plugins/registry/src/triggers/triggers.go @@ -34,7 +34,8 @@ func main() { err = registry.TriggerPostDelete(appName) case "post-release-builder": appName := flag.Arg(1) - err = registry.TriggerPostReleaseBuilder(appName) + image := flag.Arg(2) + err = registry.TriggerPostReleaseBuilder(appName, image) case "report": appName := flag.Arg(0) err = registry.ReportSingleApp(appName, "", "") diff --git a/plugins/registry/triggers.go b/plugins/registry/triggers.go index 00ddcb6c7..547955150 100644 --- a/plugins/registry/triggers.go +++ b/plugins/registry/triggers.go @@ -1,7 +1,9 @@ package registry import ( + "errors" "fmt" + "strings" "github.com/dokku/dokku/plugins/common" ) @@ -53,7 +55,19 @@ func TriggerPostDelete(appName string) error { } // TriggerPostReleaseBuilder pushes the image to the remote registry -func TriggerPostReleaseBuilder(appName string) error { +func TriggerPostReleaseBuilder(appName string, image string) error { + imageID, _ := common.DockerInspect(image, "{{ .Id }}") + imageRepo := common.GetAppImageRepo(appName) + computedImageRepo := reportComputedImageRepo(appName) + newImage := strings.Replace(image, imageRepo+":", computedImageRepo+":", 1) + + if computedImageRepo != imageRepo { + if !dockerTag(imageID, newImage) { + // TODO: better error + return errors.New("Unable to tag image") + } + } + if !isPushEnabled(appName) { return nil } @@ -64,5 +78,5 @@ func TriggerPostReleaseBuilder(appName string) error { return err } - return pushToRegistry(appName, imageTag) + return pushToRegistry(appName, imageTag, imageID, computedImageRepo) } From 30a15aaa7ac2b553681e455343e414b3b9111f76 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 04:16:49 -0400 Subject: [PATCH 28/36] fix: respect pre-deploy task image manipulation This requires a bc-incompatible break in terms of what triggers the pre-deploy hook (dokku internals vs schedulers). --- docs/appendices/0.25.0-migration-guide.md | 1 + plugins/app-json/triggers.go | 6 +----- plugins/common/functions | 3 ++- plugins/config/docker-args-deploy | 2 +- plugins/ps/triggers.go | 5 +---- plugins/scheduler-docker-local/pre-deploy | 2 +- plugins/scheduler-docker-local/scheduler-deploy | 1 - 7 files changed, 7 insertions(+), 13 deletions(-) diff --git a/docs/appendices/0.25.0-migration-guide.md b/docs/appendices/0.25.0-migration-guide.md index f61892a97..76dd350f8 100644 --- a/docs/appendices/0.25.0-migration-guide.md +++ b/docs/appendices/0.25.0-migration-guide.md @@ -15,6 +15,7 @@ The [dokku-registry](https://github.com/dokku/dokku-registry) plugin is now buil - The network plugin can now set an `initial-network` for all containers on creation. This is a replacement for specifying the `--network` flag via the `docker-options` plugin. Please see the [network documentation](/docs/networking/network.md#attaching-an-app-to-a-network) for more information. - The `dokku run` command now always removes the ephemeral container on exit. Users that need a persistent container should instead specify a `console` process type in their `Procfile` specifying an available shell (usually either `bash` or `sh`) and scale that container appropriately. +- The `pre-deploy` plugin trigger is now called internally by Dokku. Scheduler plugins should avoid calling this trigger, as any image changes introduced by subsequent trigger calls will be ignored. ### Deprecations diff --git a/plugins/app-json/triggers.go b/plugins/app-json/triggers.go index 708a5583b..a18707640 100644 --- a/plugins/app-json/triggers.go +++ b/plugins/app-json/triggers.go @@ -47,11 +47,7 @@ func TriggerPostDeploy(appName string, imageTag string) error { // TriggerPreDeploy is a trigger to execute predeploy and release deployment tasks func TriggerPreDeploy(appName string, imageTag string) error { - image, err := common.GetDeployingAppImageName(appName, imageTag, "") - if err != nil { - return err - } - + image := common.GetAppImageName(appName, imageTag, "") if err := refreshAppJSON(appName, image); err != nil { return err } diff --git a/plugins/common/functions b/plugins/common/functions index 8191c2773..dfad49333 100755 --- a/plugins/common/functions +++ b/plugins/common/functions @@ -665,7 +665,7 @@ release_and_deploy() { source "$PLUGIN_AVAILABLE_PATH/config/functions" local APP="$1" - local IMAGE_TAG="$2" + local IMAGE_TAG="${2:-latest}" local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") local DOKKU_DOCKERFILE_PORTS @@ -687,6 +687,7 @@ release_and_deploy() { local DOKKU_SKIP_DEPLOY=${DOKKU_APP_SKIP_DEPLOY:="$DOKKU_GLOBAL_SKIP_DEPLOY"} dokku_log_info1 "Releasing $APP..." + plugn trigger pre-deploy "$APP" "$IMAGE_TAG" dokku_release "$APP" "$IMAGE_SOURCE_TYPE" "$IMAGE_TAG" if [[ "$DOKKU_SKIP_DEPLOY" != "true" ]]; then diff --git a/plugins/config/docker-args-deploy b/plugins/config/docker-args-deploy index c429e3a83..2f4008227 100755 --- a/plugins/config/docker-args-deploy +++ b/plugins/config/docker-args-deploy @@ -10,7 +10,7 @@ trigger-config-docker-args() { declare APP="$1" IMAGE_TAG="$2" local ENV_ARGS IMAGE STDIN trigger - IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") + IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") STDIN=$(cat) trigger="$0 config_docker_args" diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 4723ee9af..646c127be 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -224,10 +224,7 @@ func TriggerPostStop(appName string) error { // TriggerPreDeploy ensures an app has an up to date scale parameters func TriggerPreDeploy(appName string, imageTag string) error { - image, err := common.GetDeployingAppImageName(appName, imageTag, "") - if err != nil { - return err - } + image := common.GetAppImageName(appName, imageTag, "") if err := removeProcfile(appName); err != nil { return err diff --git a/plugins/scheduler-docker-local/pre-deploy b/plugins/scheduler-docker-local/pre-deploy index 3336bc627..dbd480362 100755 --- a/plugins/scheduler-docker-local/pre-deploy +++ b/plugins/scheduler-docker-local/pre-deploy @@ -65,7 +65,7 @@ scheduler-docker-local-pre-deploy-chown-app() { scheduler-docker-local-pre-deploy-precheck() { declare desc="Outputs the checks messages if necessary" declare APP="$1" IMAGE_TAG="$2" - local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") + local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") local CHECKS_FILE=$(mktemp "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX") trap "rm -rf '$CHECKS_FILE' >/dev/null" RETURN INT TERM EXIT copy_from_image "$IMAGE" "CHECKS" "$CHECKS_FILE" 2>/dev/null || true diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index ab3a1c281..3605794a6 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -22,7 +22,6 @@ trigger-scheduler-docker-local-scheduler-deploy() { DOKKU_HEROKUISH=false DOKKU_CNB=false IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") - plugn trigger pre-deploy "$APP" "$IMAGE_TAG" is_image_cnb_based "$IMAGE" && DOKKU_CNB=true is_image_herokuish_based "$IMAGE" "$APP" && DOKKU_HEROKUISH=true From 0a0891180039162eb10c4ab6e52cac143ec4c5bc Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 04:17:37 -0400 Subject: [PATCH 29/36] fix: do not aggressively clean up when using the docker-local scheduler --- plugins/registry/functions.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/plugins/registry/functions.go b/plugins/registry/functions.go index 991981248..3b5ecf13a 100644 --- a/plugins/registry/functions.go +++ b/plugins/registry/functions.go @@ -78,9 +78,15 @@ func pushToRegistry(appName string, tag int, imageID string, imageRepo string) e return errors.New("Unable to push image") } - common.LogVerboseQuiet("Cleaning up") - imageCleanup(appName, fmt.Sprintf("%s%s", registryServer, imageRepo), imageTag, tag) - imageCleanup(appName, imageRepo, imageTag, tag) + // Only clean up when the scheduler is not docker-local + // other schedulers do not retire local images + if common.GetAppScheduler(appName) != "docker-local" { + common.LogVerboseQuiet("Cleaning up") + imageCleanup(appName, fmt.Sprintf("%s%s", registryServer, imageRepo), imageTag, tag) + if fmt.Sprintf("%s%s", registryServer, imageRepo) != imageRepo { + imageCleanup(appName, imageRepo, imageTag, tag) + } + } common.LogVerboseQuiet(fmt.Sprintf("Image %s pushed", fullImage)) return nil @@ -110,12 +116,11 @@ func dockerPush(imageTag string) bool { func imageCleanup(appName string, imageRepo string, imageTag string, tag int) { // # keep last two images in place - oldTag := tag - 1 + oldTag := tag - 2 tenImagesAgoTag := tag - 12 imagesToRemove := []string{} for oldTag > 0 { - common.LogInfo1(fmt.Sprintf("Removing image: %s:%d", imageRepo, oldTag)) imagesToRemove = append(imagesToRemove, fmt.Sprintf("%s:%d", imageRepo, oldTag)) oldTag = oldTag - 1 if tenImagesAgoTag == oldTag { From d4ec8d3ab5072005d7a835c20cd3c9162521e0c6 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 04:20:22 -0400 Subject: [PATCH 30/36] tests: add tests for registry plugin --- tests/unit/registry.bats | 291 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 tests/unit/registry.bats diff --git a/tests/unit/registry.bats b/tests/unit/registry.bats new file mode 100644 index 000000000..e740cadc6 --- /dev/null +++ b/tests/unit/registry.bats @@ -0,0 +1,291 @@ +#!/usr/bin/env bats + +load test_helper + +setup() { + global_setup + create_app + dokku config:set $TEST_APP DOKKU_WAIT_TO_RETIRE=30 +} + +teardown() { + destroy_app + global_teardown +} + +@test "(registry) registry:help" { + run /bin/bash -c "dokku registry" + echo "output: $output" + echo "status: $status" + assert_output_contains "Manage registry settings for an app" + help_output="$output" + + run /bin/bash -c "dokku registry:help" + echo "output: $output" + echo "status: $status" + assert_output_contains "Manage registry settings for an app" + assert_output "$help_output" +} + +@test "(registry) registry:login" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + run /bin/bash -c "dokku registry:login docker.io $DOCKERHUB_USERNAME $DOCKERHUB_TOKEN" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "Login Succeeded" + + run /bin/bash -c "echo $DOCKERHUB_TOKEN | dokku registry:login docker.io --password-stdin $DOCKERHUB_USERNAME" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "Login Succeeded" +} + +@test "(registry) registry:set server" { + run /bin/bash -c "dokku registry:set --global server ghcr.io" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku registry:report $TEST_APP --registry-server" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_exists + + run /bin/bash -c "dokku registry:report $TEST_APP --registry-global-server" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "ghcr.io" + + run /bin/bash -c "dokku registry:report $TEST_APP --registry-computed-server" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "ghcr.io/" + + run /bin/bash -c "dokku registry:set --global server docker.io" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku registry:report $TEST_APP --registry-server" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_exists + + run /bin/bash -c "dokku registry:report $TEST_APP --registry-global-server" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "docker.io" + + run /bin/bash -c "dokku registry:report $TEST_APP --registry-computed-server" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_exists + + run /bin/bash -c "dokku registry:set $TEST_APP server docker.io" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku registry:report $TEST_APP --registry-computed-server" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_exists + + run /bin/bash -c "dokku registry:report $TEST_APP --registry-global-server" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "docker.io" + + run /bin/bash -c "dokku registry:report $TEST_APP --registry-server" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "docker.io" + + run /bin/bash -c "dokku registry:set $TEST_APP server ghcr.io" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku registry:report $TEST_APP --registry-computed-server" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "ghcr.io/" + + run /bin/bash -c "dokku registry:report $TEST_APP --registry-global-server" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "docker.io" + + run /bin/bash -c "dokku registry:report $TEST_APP --registry-server" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "ghcr.io" +} + +@test "(registry) registry:set image-repo" { + run /bin/bash -c "docker images" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku registry:set $TEST_APP image-repo heroku/$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect heroku/$TEST_APP:latest" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker images" + echo "output: $output" + echo "status: $status" + assert_success +} + +@test "(registry) registry:set push-on-release" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + run /bin/bash -c "dokku registry:set $TEST_APP push-on-release true" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku registry:set $TEST_APP image-repo dokku/test-app" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app + echo "output: $output" + echo "status: $status" + assert_success + + sleep 60 + + run /bin/bash -c "dokku ps:retire" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker container ls -a" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker image ls" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + sleep 60 + + run /bin/bash -c "dokku ps:retire" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker container ls -a" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker image ls" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP key=VALUE" + echo "output: $output" + echo "status: $status" + assert_success + + sleep 60 + + run /bin/bash -c "dokku ps:retire" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker container ls -a" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker image ls" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + sleep 60 + + run /bin/bash -c "dokku ps:retire" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker container ls -a" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker image ls" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker image inspect dokku/test-app:1" + echo "output: $output" + echo "status: $status" + assert_failure + + run /bin/bash -c "docker image inspect dokku/test-app:2" + echo "output: $output" + echo "status: $status" + assert_failure + + run /bin/bash -c "docker image inspect dokku/test-app:3" + echo "output: $output" + echo "status: $status" + assert_failure + + run /bin/bash -c "docker image inspect dokku/test-app:4" + echo "output: $output" + echo "status: $status" + assert_success +} From 8f98bbc66fea986e1f000aad3c8d0ee62f9a9558 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 12:04:18 -0400 Subject: [PATCH 31/36] fix: do not append trailing slash on empty server values --- docs/deployment/registry-management.md | 1 - plugins/registry/functions.go | 8 ++++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/deployment/registry-management.md b/docs/deployment/registry-management.md index 0e6c56e52..c55b22fce 100644 --- a/docs/deployment/registry-management.md +++ b/docs/deployment/registry-management.md @@ -124,4 +124,3 @@ dokku registry:set node-js-app push-on-release # globally dokku registry:set --global push-on-release ``` - diff --git a/plugins/registry/functions.go b/plugins/registry/functions.go index 3b5ecf13a..62d85812a 100644 --- a/plugins/registry/functions.go +++ b/plugins/registry/functions.go @@ -17,11 +17,15 @@ func getRegistryServerForApp(appName string) string { value = common.PropertyGet("registry", "--global", "server") } - value = strings.TrimSuffix(value, "/") + "/" - if value == "hub.docker.com/" || value == "docker.io/" { + value = strings.TrimSuffix(value, "/") + if value == "hub.docker.com" || value == "docker.io" { value = "" } + if value != "" { + value = value + "/" + } + return value } From d5697b93992c32025ab17667c18faa2ed136fdf9 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 16:48:22 -0400 Subject: [PATCH 32/36] tests: inject docker credentials for tests --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8360a6b27..2fee872b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,8 @@ jobs: timeout-minutes: 30 run: sudo -E ./.github/commands/ci-run ${{ matrix.index }} env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} SYNC_GITHUB_PASSWORD: ${{ secrets.SYNC_GITHUB_PASSWORD }} SYNC_GITHUB_USERNAME: ${{ secrets.SYNC_GITHUB_USERNAME }} From 5a19cebfd770aac59c6b3d42943f033cad45a4ba Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 17:48:09 -0400 Subject: [PATCH 33/36] fix: do not mark images dead when they still have tags --- plugins/scheduler-docker-local/internal-functions | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/scheduler-docker-local/internal-functions b/plugins/scheduler-docker-local/internal-functions index cf07b1dd2..d11e966fb 100755 --- a/plugins/scheduler-docker-local/internal-functions +++ b/plugins/scheduler-docker-local/internal-functions @@ -184,8 +184,14 @@ fn-scheduler-docker-local-retire-images() { fi if echo "$RM_OUTPUT" | grep -q "image has dependent child images"; then - dokku_log_warn "Image ${IMAGE_ID} has children images, skipping rm and marking dead" - DEAD_IMAGES+=("$IMAGE_ID") + TAG_COUNT="$(docker inspect "$IMAGE_ID" --format '{{ json .RepoTags }}' | jq '. | length')" + if [[ "$TAG_COUNT" -eq 0 ]]; then + dokku_log_warn "Image ${IMAGE_ID} has children images and is untagged, skipping rm and marking dead" + DEAD_IMAGES+=("$IMAGE_ID") + continue + fi + + dokku_log_warn "Image ${IMAGE_ID} has children images and has $TAG_COUNT tags, skipping rm" continue fi From f0a125f0706ffbd6b9cb81fc124c6ab5167a5817 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 18:58:33 -0400 Subject: [PATCH 34/36] fix: remove the test-app:2 image The dokku-retire call needs two passes because a later image was a child image of test-app:2. --- tests/unit/registry.bats | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/registry.bats b/tests/unit/registry.bats index e740cadc6..67bb6adf3 100644 --- a/tests/unit/registry.bats +++ b/tests/unit/registry.bats @@ -259,6 +259,11 @@ teardown() { echo "status: $status" assert_success + run /bin/bash -c "dokku ps:retire" + echo "output: $output" + echo "status: $status" + assert_success + run /bin/bash -c "docker container ls -a" echo "output: $output" echo "status: $status" From f973239379915afd81a4948b003ca804302002dd Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 20:14:20 -0400 Subject: [PATCH 35/36] docs: link to registry management docs --- docs/{deployment => advanced-usage}/registry-management.md | 0 docs/template.html | 1 + 2 files changed, 1 insertion(+) rename docs/{deployment => advanced-usage}/registry-management.md (100%) diff --git a/docs/deployment/registry-management.md b/docs/advanced-usage/registry-management.md similarity index 100% rename from docs/deployment/registry-management.md rename to docs/advanced-usage/registry-management.md diff --git a/docs/template.html b/docs/template.html index 067144620..1f9cccf33 100644 --- a/docs/template.html +++ b/docs/template.html @@ -153,6 +153,7 @@ Backup and Recovery Deployment Tasks Docker Container Options + Docker Registry Management Event Logs Persistent Storage Plugin Management From b02f8ba88bb73349b0038564689beb6547f110f7 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Aug 2021 20:14:30 -0400 Subject: [PATCH 36/36] docs: fix deprecation warning --- docs/development/plugin-triggers.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index ed02204b2..a3bf42e4d 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1366,7 +1366,7 @@ BUILDER_TYPE="$1"; APP="$2"; IMAGE=$3 ### `post-release-buildpack` -> Warning: Deprecated, please use `docker-args-process-build` instead +> Warning: Deprecated, please use `post-release-builder` instead > Warning: Image mutation in this trigger may result in an invalid run state, and is heavily discouraged. - Description: Allows you to run commands after environment variables are set for the release step of the deploy. Only applies to apps using buildpacks. @@ -1386,7 +1386,7 @@ APP="$1"; IMAGE_TAG="$2"; IMAGE=$(get_app_image_name $APP $IMAGE_TAG) ### `post-release-pack` -> Warning: Deprecated, please use `docker-args-process-build` instead +> Warning: Deprecated, please use `post-release-builder` instead > Warning: The pack plugin trigger apis are under development and may change > between minor releases until the 1.0 release. @@ -1410,7 +1410,7 @@ APP="$1"; IMAGE_TAG="$2"; IMAGE=$(get_app_image_name $APP $IMAGE_TAG) ### `post-release-dockerfile` -> Warning: Deprecated, please use `docker-args-process-build` instead +> Warning: Deprecated, please use `post-release-builder` instead > Warning: Image mutation in this trigger may result in an invalid run state, and is heavily discouraged.