Merge pull request #5603 from dokku/4559-git-load-image

Add ability to deploy images generated via docker save with git:load-image
2025-12-29 00:25:08 +01:00 · 2023-01-21 17:25:12 -05:00
parent 5b416eaf52 4e98e17c4f
commit 2a6e42a79c
7 changed files with 398 additions and 2 deletions
--- a/docs/deployment/methods/git.md
+++ b/docs/deployment/methods/git.md
@@ -7,6 +7,7 @@ git:allow-host <host>                             # Adds a host to known_hosts
 git:auth <host> [<username> <password>]           # Configures netrc authentication for a given git server
 git:from-archive [--archive-type ARCHIVE_TYPE] <app> <archive-url> [<git-username> <git-email>] # Updates an app's git repository with a given archive file
 git:from-image [--build-dir DIRECTORY] <app> <docker-image> [<git-username> <git-email>] # Updates an app's git repository with a given docker image
+git:load-image [--build-dir DIRECTORY] <app> <docker-image> [<git-username> <git-email>] # Updates an app's git repository with a docker image loaded from stdin
 git:sync [--build] <app> <repository> [<git-ref>] # Clone or fetch an app from remote git repo
 git:initialize <app>                              # Initialize a git repository for an app
 git:public-key                                    # Outputs the dokku public deploy key
@@ -211,6 +212,61 @@ Finally, if the archive url is specified as `--`, the archive will be fetched fr
 curl -sSL https://github.com/dokku/smoke-test-app/releases/download/2.0.0/smoke-test-app.tar | dokku git:from-archive node-js-app  --
 ```

+### Initializing an app repository from a remote image without a registry
+
+> New as of 0.30.0
+
+A Dokku app repository can be initialized or updated from the contents of an image archive tar file  via the `git:load-image` command. This method can be used when a Docker Registry is unavailable to act as an intermediary for storing an image, such as when building an image in CI and deploying directly from that image.
+
+```shell
+docker image save dokku/node-js-getting-started:latest | ssh dokku@dokku.me git:load-image node-js-app dokku/node-js-getting-started:latest
+```
+
+In the above example, we are saving the image to a tar file via `docker image save`, streaming that to the Dokku host, and then running `git:load-image` on the incoming stream. Dokku will build the app as if the repository contained _only_ a `Dockerfile` with the following content:
+
+```Dockerfile
+FROM dokku/node-js-getting-started:latest
+```
+
+When deploying an app via `git:load-image`, it is highly recommended to use a unique image tag when building the image. Not doing so will result in Dokku exiting `0` early as there will be no changes detected. If the image tag is reused but the underlying image is different, it is recommended to use the image digest instead of the tag. This can be retrieved via the following command:
+
+```shell
+docker inspect --format='{{index .RepoDigests 0}}' $IMAGE_NAME
+```
+
+The resulting `git:load-image` call would then be:
+
+```shell
+# where the image sha is: sha256:9d187c3025d03c033dcc71e3a284fee53be88cc4c0356a19242758bc80cab673
+docker image save dokku/node-js-getting-started:latest | ssh dokku@dokku.me git:load-image node-js-app dokku/node-js-getting-started@sha256:9d187c3025d03c033dcc71e3a284fee53be88cc4c0356a19242758bc80cab673
+```
+
+The `git:load-image` command can optionally take a git `user.name` and `user.email` argument (in that order) to customize the author. If the arguments are left empty, they will fallback to `Dokku` and `automated@dokku.sh`, respectively.
+
+```shell
+docker image save dokku/node-js-getting-started:latest | ssh dokku@dokku.me git:load-image node-js-app dokku/node-js-getting-started:latest "Camila" "camila@example.com"
+```
+
+Building an app from an image will result in the following files being extracted from the source image (with all custom paths specified for each file being respected):
+
+- nginx.conf.sigil
+- Procfile
+
+In the case where the repository is later modified to manually add any of the above files and deployed via `git push`, the files will still be extracted from the initial source image. To avoid this, please clear the `source-image` git property. It will be set back to the original source image on any subsequent `git:load-image` calls.
+
+```shell
+# sets an empty value
+dokku git:set node-js-app source-image
+```
+
+Finally, certain images may require a custom build context in order for `ONBUILD ADD` and `ONBUILD COPY` statements to succeed. A custom build context can be specified via the `--build-dir` flag. All files in the specified `build-dir` will be copied into the repository for use within the `docker build` process. The build context _must_ be specified on each deploy, and is not otherwise persisted between builds.
+
+```shell
+docker image save dokku/node-js-getting-started:latest | ssh dokku@dokku.me git:load-image --build-dir path/to/build node-js-app dokku/node-js-getting-started:latest "Camila" "camila@example.com"
+```
+
+See the [dockerfile documentation](/docs/deployment/builders/dockerfiles.md) to learn about the different ways to configure Dockerfile-based deploys.
+
 ### Initializing an app repository from a remote repository

 > New as of 0.23.0
--- a/plugins/builder-dockerfile/builder-release
+++ b/plugins/builder-dockerfile/builder-release
@@ -12,6 +12,7 @@ trigger-builder-dockerfile-builder-release() {
    return
  fi

+  local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG")
  local DOCKER_BUILD_LABEL_ARGS=("--label=org.label-schema.schema-version=1.0" "--label=org.label-schema.vendor=dokku" "--label=com.dokku.app-name=$APP" "--label=com.dokku.image-stage=release" "--label=dokku")

  plugn trigger pre-release-dockerfile "$APP" "$IMAGE_TAG"
@@ -19,7 +20,6 @@ trigger-builder-dockerfile-builder-release() {
  TMP_WORK_DIR="$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")"
  trap "rm -rf '$TMP_WORK_DIR' >/dev/null" RETURN INT TERM EXIT

-  local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG")
  if ! suppress_output "$DOCKER_BIN" image build "${DOCKER_BUILD_LABEL_ARGS[@]}" $DOKKU_GLOBAL_BUILD_ARGS -f "$PLUGIN_AVAILABLE_PATH/builder-dockerfile/dockerfiles/builder-release.Dockerfile" --build-arg APP_IMAGE="$IMAGE" -t "$IMAGE" "$TMP_WORK_DIR"; then
    dokku_log_warn "Failure injecting docker labels on image"
    return 1
--- a/plugins/builder-herokuish/builder-release
+++ b/plugins/builder-herokuish/builder-release
@@ -13,7 +13,6 @@ trigger-builder-herokuish-builder-release() {
    return
  fi

-  local CID
  local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG")
  local DOCKER_BUILD_LABEL_ARGS=("--label=org.label-schema.schema-version=1.0" "--label=org.label-schema.vendor=dokku" "--label=com.dokku.app-name=$APP" "--label=com.dokku.image-stage=release" "--label=dokku")

--- a/plugins/git/help-functions
+++ b/plugins/git/help-functions
@@ -31,6 +31,7 @@ fn-help-content() {
    git:auth <host> [<username> <password>], Configures netrc authentication for a given git server
    git:from-archive <app> <archive-url> [<git-username> <git-email>], Updates an app's git repository with a given archive file
    git:from-image <app> <docker-image> [<git-username> <git-email>], Updates an app's git repository with a given docker image
+    git:load-image <app> <docker-image> [<git-username> <git-email>], Updates an app's git repository with a docker image loaded from stdin
    git:sync [--build] <app> <repository> [<git-ref>], Clone or fetch an app from remote git repo
    git:initialize <app>, Initialize a git repository for an app
    git:public-key, Outputs the dokku public deploy key
--- a/plugins/git/internal-functions
+++ b/plugins/git/internal-functions
@@ -78,6 +78,51 @@ cmd-git-auth() {
  netrc set "$HOST" "$USERNAME" "$PASSWORD"
 }

+cmd-git-load-image() {
+  declare desc="updates an app's git repository with a docker image loaded from stdin"
+  local cmd="git:load-image"
+  [[ "$1" == "$cmd" ]] && shift 1
+  declare APP DOCKER_IMAGE USER_NAME USER_EMAIL
+  local BUILD_DIR
+
+  ARGS=()
+  skip=false
+  for arg in "$@"; do
+    if [[ "$arg" == "--build-dir" ]]; then
+      skip=true
+      continue
+    fi
+
+    if [[ "$skip" == "true" ]]; then
+      BUILD_DIR="$arg"
+      skip=false
+      continue
+    fi
+
+    ARGS+=("$arg")
+  done
+
+  APP="${ARGS[0]}"
+  DOCKER_IMAGE="${ARGS[1]}"
+  USER_NAME="${ARGS[2]}"
+  USER_EMAIL="${ARGS[3]}"
+
+  verify_app_name "$APP"
+  [[ -z "$DOCKER_IMAGE" ]] && dokku_log_fail "Please specify a docker image"
+  [[ ! -t 0 ]] || dokku_log_fail "Expecting tar archive containing docker image on STDIN"
+
+  cat | docker load
+
+  if ! verify_image "$DOCKER_IMAGE"; then
+    dokku_log_fail "Loaded image tarball but the specified docker image was not found: $DOCKER_IMAGE"
+  fi
+
+  if ! plugn trigger git-from-image "$APP" "$DOCKER_IMAGE" "$BUILD_DIR" "$USER_NAME" "$USER_EMAIL"; then
+    return 1
+  fi
+  plugn trigger deploy-source-set "$APP" "docker-image" "$DOCKER_IMAGE"
+}
+
 cmd-git-from-image() {
  declare desc="updates an app's git repository with a given docker image"
  local cmd="git:from-image"
--- a/plugins/git/subcommands/load-image
+++ b/plugins/git/subcommands/load-image
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+source "$PLUGIN_AVAILABLE_PATH/git/internal-functions"
+set -eo pipefail
+[[ $DOKKU_TRACE ]] && set -x
+
+cmd-git-load-image "$@"
--- a/tests/unit/git_7.bats
+++ b/tests/unit/git_7.bats
@@ -0,0 +1,289 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+setup() {
+  global_setup
+  create_app
+  touch /home/dokku/.ssh/known_hosts
+  chown dokku:dokku /home/dokku/.ssh/known_hosts
+}
+
+teardown() {
+  docker image rm linuxserver/foldingathome:7.5.1-ls1 || true
+  docker image rm dokku/node-js-getting-started:latest || true
+  docker image rm dokku-test/$TEST_APP:latest || true
+  docker image rm dokku-test/$TEST_APP:v2 || true
+  docker image rm gliderlabs/logspout:v3.2.13 || true
+  rm -f /tmp/image.tar /tmp/image-2.tar
+  rm -f /home/dokku/.ssh/id_rsa.pub || true
+  destroy_app
+  global_teardown
+}
+
+@test "(git) git:load-image [normal]" {
+  run /bin/bash -c "docker image pull linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image save -o /tmp/image.tar linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image rm linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /tmp/image.tar | dokku git:load-image $TEST_APP linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+}
+
+@test "(git) git:load-image [normal-git-init]" {
+  run rm -rf "/home/dokku/$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run mkdir "/home/dokku/$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run chown -R dokku:dokku "/home/dokku/$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image pull linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image save -o /tmp/image.tar linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image rm linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /tmp/image.tar | dokku git:load-image $TEST_APP linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+}
+
+@test "(git) git:load-image [normal-cnb]" {
+  run /bin/bash -c "docker image pull dokku/node-js-getting-started:latest"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image save -o /tmp/image.tar dokku/node-js-getting-started:latest"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image rm dokku/node-js-getting-started:latest"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /tmp/image.tar | dokku git:load-image $TEST_APP dokku/node-js-getting-started:latest"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+}
+
+@test "(git) git:load-image [failing deploy]" {
+  local CUSTOM_TMP=$(mktemp -d "/tmp/dokku.me.XXXXX")
+  trap 'popd &>/dev/null || true; rm -rf "$CUSTOM_TMP"' INT TERM
+  rmdir "$CUSTOM_TMP" && cp -r "${BATS_TEST_DIRNAME}/../../tests/apps/python" "$CUSTOM_TMP"
+
+  run /bin/bash -c "docker image build -t dokku-test/$TEST_APP:latest -f $CUSTOM_TMP/alt.Dockerfile $CUSTOM_TMP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image build -t dokku-test/$TEST_APP:v2 --build-arg BUILD_ARG=value -f $CUSTOM_TMP/alt.Dockerfile $CUSTOM_TMP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image save -o /tmp/image.tar dokku-test/$TEST_APP:latest"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image save -o /tmp/image-2.tar dokku-test/$TEST_APP:v2"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image rm dokku-test/$TEST_APP:latest dokku-test/$TEST_APP:v2"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=true"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /tmp/image.tar | dokku git:load-image $TEST_APP dokku-test/$TEST_APP:latest"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  run /bin/bash -c "dokku config:get $TEST_APP FAIL_ON_STARTUP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "true"
+
+  run /bin/bash -c "dokku git:status $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+  assert_output "fatal: this operation must be run in a work tree"
+
+  run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=false"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /tmp/image.tar | dokku git:load-image $TEST_APP dokku-test/$TEST_APP:latest"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=true"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /tmp/image-2.tar | dokku git:load-image $TEST_APP dokku-test/$TEST_APP:v2"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=false"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /tmp/image-2.tar | dokku git:load-image $TEST_APP dokku-test/$TEST_APP:v2"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "No changes detected, skipping git commit" 0
+}
+
+@test "(git) git:load-image [onbuild]" {
+  local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX")
+  trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM
+
+  run /bin/bash -c "dokku storage:mount $TEST_APP /var/run/docker.sock:/var/run/docker.sock"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image pull gliderlabs/logspout:v3.2.13"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image save -o /tmp/image.tar gliderlabs/logspout:v3.2.13"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image rm gliderlabs/logspout:v3.2.13"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /tmp/image.tar | dokku git:load-image $TEST_APP gliderlabs/logspout:v3.2.13"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  cat <<EOF >"$TMP/build.sh"
+#!/bin/sh
+set -e
+apk add --update go build-base git mercurial ca-certificates
+cd /src
+go build -ldflags "-X main.Version=\$1" -o /bin/logspout
+apk del go git mercurial build-base
+rm -rf /root/go /var/cache/apk/*
+
+# backwards compatibility
+ln -fs /tmp/docker.sock /var/run/docker.sock
+EOF
+
+  cat <<EOF >"$TMP/modules.go"
+package main
+
+import (
+  _ "github.com/gliderlabs/logspout/adapters/multiline"
+  _ "github.com/gliderlabs/logspout/adapters/raw"
+  _ "github.com/gliderlabs/logspout/adapters/syslog"
+  _ "github.com/gliderlabs/logspout/healthcheck"
+  _ "github.com/gliderlabs/logspout/httpstream"
+  _ "github.com/gliderlabs/logspout/routesapi"
+  _ "github.com/gliderlabs/logspout/transports/tcp"
+  _ "github.com/gliderlabs/logspout/transports/tls"
+  _ "github.com/gliderlabs/logspout/transports/udp"
+)
+EOF
+
+  run sudo chown -R dokku:dokku "$TMP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /tmp/image.tar | dokku git:load-image --build-dir $TMP $TEST_APP gliderlabs/logspout:v3.2.13"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /tmp/image.tar | dokku git:load-image $TEST_APP gliderlabs/logspout:v3.2.13"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+}
+
+@test "(git) git:load-image labels correctly" {
+  run /bin/bash -c "docker image pull linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image save -o /tmp/image.tar linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image rm linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /tmp/image.tar | dokku git:load-image $TEST_APP linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker image inspect dokku/$TEST_APP:latest --format '{{ index .Config.Labels \"com.dokku.docker-image-labeler/alternate-tags\" }}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "linuxserver/foldingathome:7.5.1-ls1"
+}