Merge pull request #4450 from dokku/4296-git-from-image

Simplify docker image deploys via git:from-image

.github/commands/bump-azure

@@ -45,7 +45,7 @@ main() {
   echo "=====> Creating upstream pull request"
   ../$GH_FOLDER/bin/gh pr create --head dokku:dokku-$VERSION --repo Azure/azure-quickstart-templates --title 'Update dokku-vm dokku version to 0.23.7' --body ''
 
-  popd >/dev/null
+  popd &>/dev/null
 }
 
 main "$@"

.github/commands/ci-run

@@ -7,7 +7,7 @@ main() {
   mkdir -p test-results/bats
   pushd tests/unit >/dev/null
   bats --report-formatter junit --output ../../test-results/bats "$FILE_NAME.bats"
-  popd >/dev/null
+  popd &>/dev/null
 
   ls -lah test-results/bats
 }

docs/appendices/0.24.0-migration-guide.md

@@ -8,3 +8,4 @@
 ## Deprecations
 
 - The 1.0.0 release of Dokku will no longer select buildpack builders over dockerfile builders if both builders match. Instead, Dokku will choose the first builder that responds to the `builder-detect` trigger. Users that wish to use a specific builder may set a builder using the `builder:set` command, which will force Dokku to use the specified builder regardless of what might be auto-detected.
+- The `tags` plugin is deprecated in favor of the [`git:from-image`](/docs/deployment/methods/git.md#initializing-an-app-repository-from-a-docker-image) command. It will be removed in a future release, and is considered unmaintained. Users are highly encouraged to switch their workflows to `git:from-image`.

docs/deployment/methods/git.md

@@ -3,13 +3,14 @@
 > Subcommands new as of 0.12.0
 
 ```
-git:allow-host <host>                              # Adds a host to known_hosts
-git:auth <host> [<username> <password>]            # Configures netrc authentication for a given git server
-git:sync [--build] <app> <repository> [<git-ref>]  # Clone or fetch an app from remote git repo
-git:initialize <app>                               # Initialize a git repository for an app
-git:public-key                                     # Outputs the dokku public deploy key
-git:report [<app>] [<flag>]                        # Displays a git report for one or more apps
-git:set <app> <key> (<value>)                      # Set or clear a git property for an app
+git:allow-host <host>                             # Adds a host to known_hosts
+git:auth <host> [<username> <password>]           # Configures netrc authentication for a given git server
+git:from-image [--build-dir DIRECTORY] <app> <docker-image> [<git-username> <git-email>] # Updates a app's git repository with a given docker image
+git:sync [--build] <app> <repository> [<git-ref>] # Clone or fetch an app from remote git repo
+git:initialize <app>                              # Initialize a git repository for an app
+git:public-key                                    # Outputs the dokku public deploy key
+git:report [<app>] [<flag>]                       # Displays a git report for one or more apps
+git:set <app> <key> (<value>)                     # Set or clear a git property for an app
 ```
 
 Git-based deployment has been the traditional method of deploying applications in Dokku. As of v0.12.0, Dokku introduces a few ways to customize the experience of deploying via `git push`. A Git-based deployment currently supports building applications via:
@@ -114,6 +115,38 @@ dokku git:set node-js-app keep-git-dir ""
 
 Please keep in mind that setting `keep-git-dir` to `true` may result in unstaged changes shown within the built container due to the build process generating application changes within the built app directory.
 
+### Initializing an app repository from a docker image
+
+> New as of 0.24.0
+
+A Dokku app repository can be initialized or updated from a Docker image via the `git:from-image` command. This command will either initialize the app repository or update it to include the specified Docker image via a `FROM` stanza. This is an excellent way of tracking changes when deploying only a given docker image, especially if deploying an image from a remote CI/CD pipeline.
+
+```shell
+dokku git:from-image node-js-app dokku/node-js-getting-started:latest
+```
+
+In the above example, Dokku will build the app as if the repository contained _only_ a `Dockerfile` with the following content:
+
+```Dockerfile
+FROM dokku/node-js-getting-started:latest
+```
+
+Triggering a build with the same arguments multiple times will result in Dokku exiting `0` early as there is no build to perform.
+
+The `git:from-image` command can optionally take a git `user.name` and `user.email` argument (in that order) to customize the author. If the arguments are left empty, they will fallback to `Dokku` and `automated@dokku.sh`, respectively.
+
+```shell
+dokku git:from-image node-js-app dokku/node-js-getting-started:latest "Camila" "camila@example.com"
+```
+
+Finally, certain images may require a custom build context in order for `ONBUILD ADD` and `ONBUILD COPY` statements to succeed. A custom build context can be specified via the `--build-dir` flag. All files in the specified `build-dir` will be copied into the repository for use within the `docker build` process. The build context _must_ be specified on each deploy, and is not otherwise persisted between builds.
+
+```shell
+dokku git:from-image --build-dir path/to/build node-js-app dokku/node-js-getting-started:latest "Camila" "camila@example.com"
+```
+
+See the [dockerfile documentation](/docs/deployment/methods/dockerfiles.md) to learn about the different ways to configure Dockerfile-based deploys.
+
 ### Initializing an app repository from a remote repository
 
 > New as of 0.23.0

docs/deployment/methods/images.md

@@ -1,5 +1,7 @@
 # Docker Image Tag Deployment
 
+> Warning: As of 0.24.0, this functionality is deprecated in favor of the [`git:from-image`](/docs/deployment/methods/git.md#initializing-an-app-repository-from-a-docker-image) command. It will be removed in a future release, and is considered unmaintained. Users are highly encouraged to switch their workflows to `git:from-image`.
+>
 > New as of 0.4.0
 
 ```

docs/development/plugin-triggers.md

@@ -1284,7 +1284,7 @@ REV="$3" # optional, may not be sent for tar-based builds
 pushd "$TMP_WORK_DIR" >/dev/null
 touch Procfile
 echo "clock: some-command" >> Procfile
-popd >/dev/null
+popd &>/dev/null
 ```
 
 ### `post-proxy-ports-update`

plugins/builder-herokuish/post-app-clone-setup

@@ -14,7 +14,7 @@ trigger-builder-herokuish-post-app-clone-setup() {
 
   pushd "$DOKKU_ROOT/$OLD_APP/." >/dev/null
   find ./* \( -name ".cache" -o -name "cache" \) -prune -o -print | cpio -pdmu --quiet "$DOKKU_ROOT/$NEW_APP"
-  popd >/dev/null 2>&1 || pushd "/tmp" >/dev/null
+  popd &>/dev/null || pushd "/tmp" >/dev/null
 
   if [[ -d "$NEW_CACHE_DIR" ]] && ! rmdir "$NEW_CACHE_DIR"; then
     local DOCKER_RUN_LABEL_ARGS="--label=com.dokku.app-name=$NEW_APP"

plugins/builder-herokuish/post-app-rename-setup

@@ -20,7 +20,7 @@ trigger-builder-herokuish-post-app-clone-setup() {
 
   pushd "$DOKKU_ROOT/$OLD_APP/." >/dev/null
   find ./* \( -name ".cache" -o -name "cache" \) -prune -o -print | cpio -pdmu --quiet "$DOKKU_ROOT/$NEW_APP"
-  popd >/dev/null 2>&1 || pushd "/tmp" >/dev/null
+  popd &>/dev/null || pushd "/tmp" >/dev/null
 }
 
 trigger-builder-herokuish-post-app-clone-setup "$@"

plugins/git/git-from-directory

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions"
+source "$PLUGIN_AVAILABLE_PATH/git/functions"
+source "$PLUGIN_AVAILABLE_PATH/git/internal-functions"
+set -eo pipefail
+[[ $DOKKU_TRACE ]] && set -x
+
+trigger-git-git-from-directory() {
+  declare desc="updates a repository from a directory"
+  declare trigger="git-from-directory"
+  declare APP="$1" SOURCECODE_WORK_DIR="$2" USER_NAME="${3:-Dokku}" USER_EMAIL="${4:-automated@dokku.sh}"
+
+  local APP_ROOT="$DOKKU_ROOT/$APP"
+  local DOKKU_DEPLOY_BRANCH="$(fn-git-deploy-branch "$APP")"
+  local REV
+  local TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")
+  local TMP_WORK_DIR_2=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")
+  # shellcheck disable=SC2086
+  trap "rm -rf '$TMP_WORK_DIR' '$TMP_WORK_DIR_2' >/dev/null" RETURN INT TERM EXIT
+
+  dokku_log_info1 "Updating git repository with specified build context"
+  if [[ "$(fn-git-cmd "$APP_ROOT" count-objects)" == "0 objects, 0 kilobytes" ]]; then
+    # setup new repo
+    cp -rT "$SOURCECODE_WORK_DIR" "$TMP_WORK_DIR"
+    suppress_output fn-git-cmd "$TMP_WORK_DIR" init
+    suppress_output fn-git-cmd "$TMP_WORK_DIR" config user.name "$USER_NAME"
+    suppress_output fn-git-cmd "$TMP_WORK_DIR" config user.email "$USER_EMAIL"
+    suppress_output fn-git-cmd "$TMP_WORK_DIR" add --all
+    suppress_output fn-git-cmd "$TMP_WORK_DIR" commit -m "Initial commit"
+
+    REV="$(fn-git-cmd "$TMP_WORK_DIR" rev-parse HEAD)"
+    rsync -a "$TMP_WORK_DIR/.git/" "$APP_ROOT"
+    fn-git-create-hook "$APP"
+  else
+    # update existing repo
+    GIT_TERMINAL_PROMPT=0 suppress_output git clone "$APP_ROOT" "$TMP_WORK_DIR_2"
+    cp -rT "$SOURCECODE_WORK_DIR" "$TMP_WORK_DIR"
+    mv "$TMP_WORK_DIR_2/.git" "$TMP_WORK_DIR/.git"
+    suppress_output fn-git-cmd "$TMP_WORK_DIR" config user.name "$USER_NAME"
+    suppress_output fn-git-cmd "$TMP_WORK_DIR" config user.email "$USER_EMAIL"
+    suppress_output fn-git-cmd "$TMP_WORK_DIR" add --all
+    suppress_output fn-git-cmd "$TMP_WORK_DIR" update-index --refresh
+    if suppress_output fn-git-cmd "$TMP_WORK_DIR" diff-index --quiet HEAD --; then
+      dokku_log_warn "No changes detected, aborting git update"
+      return
+    fi
+
+    suppress_output fn-git-cmd "$TMP_WORK_DIR" commit -m "Automated commit @ $(date +%s)"
+
+    REV="$(fn-git-cmd "$TMP_WORK_DIR" rev-parse HEAD)"
+    fn-git-fetch "$APP" "$TMP_WORK_DIR" "$REV"
+  fi
+
+  git_receive_app "$APP" "$REV"
+}
+
+trigger-git-git-from-directory "$@"

plugins/git/help-functions

@@ -29,6 +29,7 @@ fn-help-content() {
   cat <<help_content
     git:allow-host <host>, Adds a host to known_hosts
     git:auth <host> [<username> <password>], Configures netrc authentication for a given git server
+    git:from-image <app> <docker-image> [<git-username> <git-email>], Updates a app's git repository with a given docker image
     git:sync [--build] <app> <repository> [<git-ref>], Clone or fetch an app from remote git repo
     git:initialize <app>, Initialize a git repository for an app
     git:public-key, Outputs the dokku public deploy key

plugins/git/internal-functions

@@ -36,6 +36,58 @@ cmd-git-auth() {
   netrc set "$HOST" "$USERNAME" "$PASSWORD"
 }
 
+cmd-git-from-image() {
+  declare desc="updates a app's git repository with a given docker image"
+  local cmd="git:from-image"
+  [[ "$1" == "$cmd" ]] && shift 1
+  declare APP DOCKER_IMAGE USER_NAME USER_EMAIL
+  local BUILD_DIR
+
+  ARGS=()
+  skip=false
+  for arg in "$@"; do
+    if [[ "$arg" == "--build-dir" ]]; then
+      skip=true
+      continue
+    fi
+
+    if [[ "$skip" == "true" ]]; then
+      BUILD_DIR="$arg"
+      skip=false
+      continue
+    fi
+
+    ARGS+=("$arg")
+  done
+
+  APP="${ARGS[0]}"
+  DOCKER_IMAGE="${ARGS[1]}"
+  USER_NAME="${ARGS[2]}"
+  USER_EMAIL="${ARGS[3]}"
+
+  verify_app_name "$APP"
+  [[ -z "$DOCKER_IMAGE" ]] && dokku_log_fail "Please specify a docker image"
+
+  local TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")
+  trap "rm -rf '$TMP_WORK_DIR' '$TMP_WORK_DIR_2' >/dev/null" RETURN INT TERM EXIT
+
+  dokku_log_info1 "Generating build context"
+  if [[ -n "$BUILD_DIR" ]]; then
+    if [[ ! -d "$BUILD_DIR" ]]; then
+      dokku_log_fail "Invalid BUILD_DIR specified for docker build context"
+    fi
+
+    dokku_log_verbose "Syncing build directory context"
+    rsync -a "$BUILD_DIR/" "$TMP_WORK_DIR"
+  fi
+
+  dokku_log_verbose "Setting Dockerfile"
+  touch "$TMP_WORK_DIR/Dockerfile"
+  echo "FROM $DOCKER_IMAGE" >"$TMP_WORK_DIR/Dockerfile"
+
+  plugn trigger git-from-directory "$APP" "$TMP_WORK_DIR" "$USER_NAME" "$USER_EMAIL"
+}
+
 cmd-git-sync() {
   declare desc="clone or fetch an app from remote git repo"
   local cmd="git:sync"
@@ -245,7 +297,7 @@ fn-git-cmd() {
   pushd "$GIT_DIR" >/dev/null
   git "$@"
   exit_code="$?"
-  popd >/dev/null 2>&1 || pushd "/tmp" >/dev/null
+  popd &>/dev/null || pushd "/tmp" >/dev/null
   return $exit_code
 }
 
@@ -383,7 +435,7 @@ fn-git-setup-build-dir-submodules() {
   if [[ "$DOKKU_KEEP_GIT_DIR" != "true" ]]; then
     pushd "$GIT_WORKDIR" >/dev/null
     find "$GIT_WORKDIR" -name .git -prune -exec rm -rf {} \; >/dev/null
-    popd >/dev/null 2>&1 || pushd "/tmp" >/dev/null
+    popd &>/dev/null || pushd "/tmp" >/dev/null
   fi
 }
 

plugins/git/subcommands/from-image

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+source "$PLUGIN_AVAILABLE_PATH/git/internal-functions"
+set -eo pipefail
+[[ $DOKKU_TRACE ]] && set -x
+
+cmd-git-from-image "$@"

tests/unit/git_4.bats

@@ -0,0 +1,96 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+setup() {
+  global_setup
+  create_app
+  touch /home/dokku/.ssh/known_hosts
+  chown dokku:dokku /home/dokku/.ssh/known_hosts
+}
+
+teardown() {
+  rm -f /home/dokku/.ssh/id_rsa.pub || true
+  destroy_app
+  global_teardown
+}
+
+@test "(git) git:from-image [missing]" {
+  run /bin/bash -c "dokku git:from-image $TEST_APP dokku/python:missing-tag"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+}
+
+@test "(git) git:from-image [normal]" {
+  run /bin/bash -c "dokku git:from-image $TEST_APP linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+}
+
+@test "(git) git:from-image [normal-cnb]" {
+  run /bin/bash -c "dokku git:from-image $TEST_APP dokku/node-js-getting-started:latest"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+}
+
+@test "(git) git:from-image [onbuild]" {
+  local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX")
+  trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM
+
+  run /bin/bash -c "dokku storage:mount $TEST_APP /var/run/docker.sock:/var/run/docker.sock"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:from-image $TEST_APP gliderlabs/logspout:v3.2.13"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  cat <<EOF >"$TMP/build.sh"
+#!/bin/sh
+set -e
+apk add --update go build-base git mercurial ca-certificates
+cd /src
+go build -ldflags "-X main.Version=\$1" -o /bin/logspout
+apk del go git mercurial build-base
+rm -rf /root/go /var/cache/apk/*
+
+# backwards compatibility
+ln -fs /tmp/docker.sock /var/run/docker.sock
+EOF
+
+  cat <<EOF >"$TMP/modules.go"
+package main
+
+import (
+  _ "github.com/gliderlabs/logspout/adapters/multiline"
+  _ "github.com/gliderlabs/logspout/adapters/raw"
+  _ "github.com/gliderlabs/logspout/adapters/syslog"
+  _ "github.com/gliderlabs/logspout/healthcheck"
+  _ "github.com/gliderlabs/logspout/httpstream"
+  _ "github.com/gliderlabs/logspout/routesapi"
+  _ "github.com/gliderlabs/logspout/transports/tcp"
+  _ "github.com/gliderlabs/logspout/transports/tls"
+  _ "github.com/gliderlabs/logspout/transports/udp"
+)
+EOF
+
+  run sudo chown -R dokku:dokku "$TMP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:from-image --build-dir $TMP $TEST_APP gliderlabs/logspout:v3.2.13"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:from-image $TEST_APP gliderlabs/logspout:v3.2.13"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+}