From 5804cd11bc6087a7dac78d947e73fa5645076f04 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 1 Mar 2021 00:30:34 -0500 Subject: [PATCH 1/2] chore: standardize popd calls --- .github/commands/bump-azure | 2 +- .github/commands/ci-run | 2 +- docs/development/plugin-triggers.md | 2 +- plugins/builder-herokuish/post-app-clone-setup | 2 +- plugins/builder-herokuish/post-app-rename-setup | 2 +- plugins/git/internal-functions | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/commands/bump-azure b/.github/commands/bump-azure index 087873f17..fb6a186d2 100755 --- a/.github/commands/bump-azure +++ b/.github/commands/bump-azure @@ -45,7 +45,7 @@ main() { echo "=====> Creating upstream pull request" ../$GH_FOLDER/bin/gh pr create --head dokku:dokku-$VERSION --repo Azure/azure-quickstart-templates --title 'Update dokku-vm dokku version to 0.23.7' --body '' - popd >/dev/null + popd &>/dev/null } main "$@" diff --git a/.github/commands/ci-run b/.github/commands/ci-run index aad5ed2f2..88256ef92 100755 --- a/.github/commands/ci-run +++ b/.github/commands/ci-run @@ -7,7 +7,7 @@ main() { mkdir -p test-results/bats pushd tests/unit >/dev/null bats --report-formatter junit --output ../../test-results/bats "$FILE_NAME.bats" - popd >/dev/null + popd &>/dev/null ls -lah test-results/bats } diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index da4ac32ef..bc540b7f0 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1284,7 +1284,7 @@ REV="$3" # optional, may not be sent for tar-based builds pushd "$TMP_WORK_DIR" >/dev/null touch Procfile echo "clock: some-command" >> Procfile -popd >/dev/null +popd &>/dev/null ``` ### `post-proxy-ports-update` diff --git a/plugins/builder-herokuish/post-app-clone-setup b/plugins/builder-herokuish/post-app-clone-setup index 5d79dda88..3abb15417 100755 --- a/plugins/builder-herokuish/post-app-clone-setup +++ b/plugins/builder-herokuish/post-app-clone-setup @@ -14,7 +14,7 @@ trigger-builder-herokuish-post-app-clone-setup() { pushd "$DOKKU_ROOT/$OLD_APP/." >/dev/null find ./* \( -name ".cache" -o -name "cache" \) -prune -o -print | cpio -pdmu --quiet "$DOKKU_ROOT/$NEW_APP" - popd >/dev/null 2>&1 || pushd "/tmp" >/dev/null + popd &>/dev/null || pushd "/tmp" >/dev/null if [[ -d "$NEW_CACHE_DIR" ]] && ! rmdir "$NEW_CACHE_DIR"; then local DOCKER_RUN_LABEL_ARGS="--label=com.dokku.app-name=$NEW_APP" diff --git a/plugins/builder-herokuish/post-app-rename-setup b/plugins/builder-herokuish/post-app-rename-setup index 8c8d8ddfe..2c96153a8 100755 --- a/plugins/builder-herokuish/post-app-rename-setup +++ b/plugins/builder-herokuish/post-app-rename-setup @@ -20,7 +20,7 @@ trigger-builder-herokuish-post-app-clone-setup() { pushd "$DOKKU_ROOT/$OLD_APP/." >/dev/null find ./* \( -name ".cache" -o -name "cache" \) -prune -o -print | cpio -pdmu --quiet "$DOKKU_ROOT/$NEW_APP" - popd >/dev/null 2>&1 || pushd "/tmp" >/dev/null + popd &>/dev/null || pushd "/tmp" >/dev/null } trigger-builder-herokuish-post-app-clone-setup "$@" diff --git a/plugins/git/internal-functions b/plugins/git/internal-functions index 0c956dfba..bd43827a7 100755 --- a/plugins/git/internal-functions +++ b/plugins/git/internal-functions @@ -245,7 +245,7 @@ fn-git-cmd() { pushd "$GIT_DIR" >/dev/null git "$@" exit_code="$?" - popd >/dev/null 2>&1 || pushd "/tmp" >/dev/null + popd &>/dev/null || pushd "/tmp" >/dev/null return $exit_code } @@ -383,7 +383,7 @@ fn-git-setup-build-dir-submodules() { if [[ "$DOKKU_KEEP_GIT_DIR" != "true" ]]; then pushd "$GIT_WORKDIR" >/dev/null find "$GIT_WORKDIR" -name .git -prune -exec rm -rf {} \; >/dev/null - popd >/dev/null 2>&1 || pushd "/tmp" >/dev/null + popd &>/dev/null || pushd "/tmp" >/dev/null fi } From 0a31f6fe3ad2615cee3e90baf1a394dcad9f20dd Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 1 Mar 2021 02:03:11 -0500 Subject: [PATCH 2/2] feat: add git:from-image support The new command superscedes the previous tags plugin, and integrates docker image deployment with the general build process. While `docker image load` is not supported, this otherwise completely handles all previous workflows supported by the `tags:deploy` command, while doing so in a much easier to use interface. Closes #4296 --- docs/appendices/0.24.0-migration-guide.md | 1 + docs/deployment/methods/git.md | 47 +++++++++-- docs/deployment/methods/images.md | 2 + plugins/git/git-from-directory | 57 ++++++++++++++ plugins/git/help-functions | 1 + plugins/git/internal-functions | 52 ++++++++++++ plugins/git/subcommands/from-image | 6 ++ tests/unit/git_4.bats | 96 +++++++++++++++++++++++ 8 files changed, 255 insertions(+), 7 deletions(-) create mode 100755 plugins/git/git-from-directory create mode 100755 plugins/git/subcommands/from-image create mode 100644 tests/unit/git_4.bats diff --git a/docs/appendices/0.24.0-migration-guide.md b/docs/appendices/0.24.0-migration-guide.md index 4815728f8..3a538eed4 100644 --- a/docs/appendices/0.24.0-migration-guide.md +++ b/docs/appendices/0.24.0-migration-guide.md @@ -8,3 +8,4 @@ ## Deprecations - The 1.0.0 release of Dokku will no longer select buildpack builders over dockerfile builders if both builders match. Instead, Dokku will choose the first builder that responds to the `builder-detect` trigger. Users that wish to use a specific builder may set a builder using the `builder:set` command, which will force Dokku to use the specified builder regardless of what might be auto-detected. +- The `tags` plugin is deprecated in favor of the [`git:from-image`](/docs/deployment/methods/git.md#initializing-an-app-repository-from-a-docker-image) command. It will be removed in a future release, and is considered unmaintained. Users are highly encouraged to switch their workflows to `git:from-image`. diff --git a/docs/deployment/methods/git.md b/docs/deployment/methods/git.md index fc5626cfb..4559d9b37 100644 --- a/docs/deployment/methods/git.md +++ b/docs/deployment/methods/git.md @@ -3,13 +3,14 @@ > Subcommands new as of 0.12.0 ``` -git:allow-host # Adds a host to known_hosts -git:auth [ ] # Configures netrc authentication for a given git server -git:sync [--build] [] # Clone or fetch an app from remote git repo -git:initialize # Initialize a git repository for an app -git:public-key # Outputs the dokku public deploy key -git:report [] [] # Displays a git report for one or more apps -git:set () # Set or clear a git property for an app +git:allow-host # Adds a host to known_hosts +git:auth [ ] # Configures netrc authentication for a given git server +git:from-image [--build-dir DIRECTORY] [ ] # Updates a app's git repository with a given docker image +git:sync [--build] [] # Clone or fetch an app from remote git repo +git:initialize # Initialize a git repository for an app +git:public-key # Outputs the dokku public deploy key +git:report [] [] # Displays a git report for one or more apps +git:set () # Set or clear a git property for an app ``` Git-based deployment has been the traditional method of deploying applications in Dokku. As of v0.12.0, Dokku introduces a few ways to customize the experience of deploying via `git push`. A Git-based deployment currently supports building applications via: @@ -114,6 +115,38 @@ dokku git:set node-js-app keep-git-dir "" Please keep in mind that setting `keep-git-dir` to `true` may result in unstaged changes shown within the built container due to the build process generating application changes within the built app directory. +### Initializing an app repository from a docker image + +> New as of 0.24.0 + +A Dokku app repository can be initialized or updated from a Docker image via the `git:from-image` command. This command will either initialize the app repository or update it to include the specified Docker image via a `FROM` stanza. This is an excellent way of tracking changes when deploying only a given docker image, especially if deploying an image from a remote CI/CD pipeline. + +```shell +dokku git:from-image node-js-app dokku/node-js-getting-started:latest +``` + +In the above example, Dokku will build the app as if the repository contained _only_ a `Dockerfile` with the following content: + +```Dockerfile +FROM dokku/node-js-getting-started:latest +``` + +Triggering a build with the same arguments multiple times will result in Dokku exiting `0` early as there is no build to perform. + +The `git:from-image` command can optionally take a git `user.name` and `user.email` argument (in that order) to customize the author. If the arguments are left empty, they will fallback to `Dokku` and `automated@dokku.sh`, respectively. + +```shell +dokku git:from-image node-js-app dokku/node-js-getting-started:latest "Camila" "camila@example.com" +``` + +Finally, certain images may require a custom build context in order for `ONBUILD ADD` and `ONBUILD COPY` statements to succeed. A custom build context can be specified via the `--build-dir` flag. All files in the specified `build-dir` will be copied into the repository for use within the `docker build` process. The build context _must_ be specified on each deploy, and is not otherwise persisted between builds. + +```shell +dokku git:from-image --build-dir path/to/build node-js-app dokku/node-js-getting-started:latest "Camila" "camila@example.com" +``` + +See the [dockerfile documentation](/docs/deployment/methods/dockerfiles.md) to learn about the different ways to configure Dockerfile-based deploys. + ### Initializing an app repository from a remote repository > New as of 0.23.0 diff --git a/docs/deployment/methods/images.md b/docs/deployment/methods/images.md index 33bfbf81d..c82d23f68 100644 --- a/docs/deployment/methods/images.md +++ b/docs/deployment/methods/images.md @@ -1,5 +1,7 @@ # Docker Image Tag Deployment +> Warning: As of 0.24.0, this functionality is deprecated in favor of the [`git:from-image`](/docs/deployment/methods/git.md#initializing-an-app-repository-from-a-docker-image) command. It will be removed in a future release, and is considered unmaintained. Users are highly encouraged to switch their workflows to `git:from-image`. +> > New as of 0.4.0 ``` diff --git a/plugins/git/git-from-directory b/plugins/git/git-from-directory new file mode 100755 index 000000000..a1504257d --- /dev/null +++ b/plugins/git/git-from-directory @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +source "$PLUGIN_AVAILABLE_PATH/git/functions" +source "$PLUGIN_AVAILABLE_PATH/git/internal-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-git-git-from-directory() { + declare desc="updates a repository from a directory" + declare trigger="git-from-directory" + declare APP="$1" SOURCECODE_WORK_DIR="$2" USER_NAME="${3:-Dokku}" USER_EMAIL="${4:-automated@dokku.sh}" + + local APP_ROOT="$DOKKU_ROOT/$APP" + local DOKKU_DEPLOY_BRANCH="$(fn-git-deploy-branch "$APP")" + local REV + local TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX") + local TMP_WORK_DIR_2=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX") + # shellcheck disable=SC2086 + trap "rm -rf '$TMP_WORK_DIR' '$TMP_WORK_DIR_2' >/dev/null" RETURN INT TERM EXIT + + dokku_log_info1 "Updating git repository with specified build context" + if [[ "$(fn-git-cmd "$APP_ROOT" count-objects)" == "0 objects, 0 kilobytes" ]]; then + # setup new repo + cp -rT "$SOURCECODE_WORK_DIR" "$TMP_WORK_DIR" + suppress_output fn-git-cmd "$TMP_WORK_DIR" init + suppress_output fn-git-cmd "$TMP_WORK_DIR" config user.name "$USER_NAME" + suppress_output fn-git-cmd "$TMP_WORK_DIR" config user.email "$USER_EMAIL" + suppress_output fn-git-cmd "$TMP_WORK_DIR" add --all + suppress_output fn-git-cmd "$TMP_WORK_DIR" commit -m "Initial commit" + + REV="$(fn-git-cmd "$TMP_WORK_DIR" rev-parse HEAD)" + rsync -a "$TMP_WORK_DIR/.git/" "$APP_ROOT" + fn-git-create-hook "$APP" + else + # update existing repo + GIT_TERMINAL_PROMPT=0 suppress_output git clone "$APP_ROOT" "$TMP_WORK_DIR_2" + cp -rT "$SOURCECODE_WORK_DIR" "$TMP_WORK_DIR" + mv "$TMP_WORK_DIR_2/.git" "$TMP_WORK_DIR/.git" + suppress_output fn-git-cmd "$TMP_WORK_DIR" config user.name "$USER_NAME" + suppress_output fn-git-cmd "$TMP_WORK_DIR" config user.email "$USER_EMAIL" + suppress_output fn-git-cmd "$TMP_WORK_DIR" add --all + suppress_output fn-git-cmd "$TMP_WORK_DIR" update-index --refresh + if suppress_output fn-git-cmd "$TMP_WORK_DIR" diff-index --quiet HEAD --; then + dokku_log_warn "No changes detected, aborting git update" + return + fi + + suppress_output fn-git-cmd "$TMP_WORK_DIR" commit -m "Automated commit @ $(date +%s)" + + REV="$(fn-git-cmd "$TMP_WORK_DIR" rev-parse HEAD)" + fn-git-fetch "$APP" "$TMP_WORK_DIR" "$REV" + fi + + git_receive_app "$APP" "$REV" +} + +trigger-git-git-from-directory "$@" diff --git a/plugins/git/help-functions b/plugins/git/help-functions index f007b1146..3fc05a844 100755 --- a/plugins/git/help-functions +++ b/plugins/git/help-functions @@ -29,6 +29,7 @@ fn-help-content() { cat <, Adds a host to known_hosts git:auth [ ], Configures netrc authentication for a given git server + git:from-image [ ], Updates a app's git repository with a given docker image git:sync [--build] [], Clone or fetch an app from remote git repo git:initialize , Initialize a git repository for an app git:public-key, Outputs the dokku public deploy key diff --git a/plugins/git/internal-functions b/plugins/git/internal-functions index bd43827a7..3a7cfec45 100755 --- a/plugins/git/internal-functions +++ b/plugins/git/internal-functions @@ -36,6 +36,58 @@ cmd-git-auth() { netrc set "$HOST" "$USERNAME" "$PASSWORD" } +cmd-git-from-image() { + declare desc="updates a app's git repository with a given docker image" + local cmd="git:from-image" + [[ "$1" == "$cmd" ]] && shift 1 + declare APP DOCKER_IMAGE USER_NAME USER_EMAIL + local BUILD_DIR + + ARGS=() + skip=false + for arg in "$@"; do + if [[ "$arg" == "--build-dir" ]]; then + skip=true + continue + fi + + if [[ "$skip" == "true" ]]; then + BUILD_DIR="$arg" + skip=false + continue + fi + + ARGS+=("$arg") + done + + APP="${ARGS[0]}" + DOCKER_IMAGE="${ARGS[1]}" + USER_NAME="${ARGS[2]}" + USER_EMAIL="${ARGS[3]}" + + verify_app_name "$APP" + [[ -z "$DOCKER_IMAGE" ]] && dokku_log_fail "Please specify a docker image" + + local TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX") + trap "rm -rf '$TMP_WORK_DIR' '$TMP_WORK_DIR_2' >/dev/null" RETURN INT TERM EXIT + + dokku_log_info1 "Generating build context" + if [[ -n "$BUILD_DIR" ]]; then + if [[ ! -d "$BUILD_DIR" ]]; then + dokku_log_fail "Invalid BUILD_DIR specified for docker build context" + fi + + dokku_log_verbose "Syncing build directory context" + rsync -a "$BUILD_DIR/" "$TMP_WORK_DIR" + fi + + dokku_log_verbose "Setting Dockerfile" + touch "$TMP_WORK_DIR/Dockerfile" + echo "FROM $DOCKER_IMAGE" >"$TMP_WORK_DIR/Dockerfile" + + plugn trigger git-from-directory "$APP" "$TMP_WORK_DIR" "$USER_NAME" "$USER_EMAIL" +} + cmd-git-sync() { declare desc="clone or fetch an app from remote git repo" local cmd="git:sync" diff --git a/plugins/git/subcommands/from-image b/plugins/git/subcommands/from-image new file mode 100755 index 000000000..1a47823d3 --- /dev/null +++ b/plugins/git/subcommands/from-image @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/git/internal-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-git-from-image "$@" diff --git a/tests/unit/git_4.bats b/tests/unit/git_4.bats new file mode 100644 index 000000000..dd308b786 --- /dev/null +++ b/tests/unit/git_4.bats @@ -0,0 +1,96 @@ +#!/usr/bin/env bats + +load test_helper + +setup() { + global_setup + create_app + touch /home/dokku/.ssh/known_hosts + chown dokku:dokku /home/dokku/.ssh/known_hosts +} + +teardown() { + rm -f /home/dokku/.ssh/id_rsa.pub || true + destroy_app + global_teardown +} + +@test "(git) git:from-image [missing]" { + run /bin/bash -c "dokku git:from-image $TEST_APP dokku/python:missing-tag" + echo "output: $output" + echo "status: $status" + assert_failure +} + +@test "(git) git:from-image [normal]" { + run /bin/bash -c "dokku git:from-image $TEST_APP linuxserver/foldingathome:7.5.1-ls1" + echo "output: $output" + echo "status: $status" + assert_success +} + +@test "(git) git:from-image [normal-cnb]" { + run /bin/bash -c "dokku git:from-image $TEST_APP dokku/node-js-getting-started:latest" + echo "output: $output" + echo "status: $status" + assert_success +} + +@test "(git) git:from-image [onbuild]" { + local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX") + trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM + + run /bin/bash -c "dokku storage:mount $TEST_APP /var/run/docker.sock:/var/run/docker.sock" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku git:from-image $TEST_APP gliderlabs/logspout:v3.2.13" + echo "output: $output" + echo "status: $status" + assert_failure + + cat <"$TMP/build.sh" +#!/bin/sh +set -e +apk add --update go build-base git mercurial ca-certificates +cd /src +go build -ldflags "-X main.Version=\$1" -o /bin/logspout +apk del go git mercurial build-base +rm -rf /root/go /var/cache/apk/* + +# backwards compatibility +ln -fs /tmp/docker.sock /var/run/docker.sock +EOF + + cat <"$TMP/modules.go" +package main + +import ( + _ "github.com/gliderlabs/logspout/adapters/multiline" + _ "github.com/gliderlabs/logspout/adapters/raw" + _ "github.com/gliderlabs/logspout/adapters/syslog" + _ "github.com/gliderlabs/logspout/healthcheck" + _ "github.com/gliderlabs/logspout/httpstream" + _ "github.com/gliderlabs/logspout/routesapi" + _ "github.com/gliderlabs/logspout/transports/tcp" + _ "github.com/gliderlabs/logspout/transports/tls" + _ "github.com/gliderlabs/logspout/transports/udp" +) +EOF + + run sudo chown -R dokku:dokku "$TMP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku git:from-image --build-dir $TMP $TEST_APP gliderlabs/logspout:v3.2.13" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku git:from-image $TEST_APP gliderlabs/logspout:v3.2.13" + echo "output: $output" + echo "status: $status" + assert_failure +}