From 26e9cc9b8c18b52ad3d4de743984cbd8526ff59b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 5 Aug 2023 12:28:11 -0400 Subject: [PATCH] feat: add support for non-web healthchecks via app.json This change converts existing CHECKS files into a healthchecks key that is understood by 'docker-container-healthchecker'. This tool supports a number of different types of container healthchecks - command, http, uptime - and can perform healthchecks against non-web processes. The use of the old CHECKS file is now deprecated, and will be removed in the next minor version. Users can use the 'docker-container-healthchecker' to convert existing CHECKS files to the new format automatically. Closes #2760 --- Makefile | 9 +- .../digitalocean/in_parts/012-dokku-packages | 3 +- debian/control | 2 +- docker/etc/sudoers.d/dokku-docker | 2 +- .../local/bin/docker-container-healthchecker | 20 ++ docs/appendices/0.31.0-migration-guide.md | 2 + docs/deployment/zero-downtime-deploys.md | 293 ++---------------- docs/development/plugin-triggers.md | 15 + docs/processes/process-management.md | 1 - plugins/20_events/app-json-get-content | 1 + plugins/app-json/Makefile | 2 +- plugins/app-json/src/triggers/triggers.go | 3 + plugins/app-json/triggers.go | 22 ++ plugins/scheduler-docker-local/check-deploy | 171 ++++------ tests/ci/setup.sh | 7 + tests/unit/checks.bats | 2 +- 16 files changed, 171 insertions(+), 384 deletions(-) create mode 100755 docker/usr/local/bin/docker-container-healthchecker create mode 120000 plugins/20_events/app-json-get-content diff --git a/Makefile b/Makefile index 7e91b9560..19455ec34 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ DOKKU_VERSION ?= master TARGETARCH ?= amd64 DOCKER_IMAGE_LABELER_VERSION ?= 0.5.0 +DOCKER_CONTAINER_HEALTHCHECKER_VERSION ?= 0.6.4 HEROKUISH_VERSION ?= 0.6.0 LAMBDA_BUILDER_VERSION ?= 0.4.0 NETRC_VERSION ?= 0.6.0 @@ -11,6 +12,7 @@ PROCFILE_VERSION ?= 0.15.0 SIGIL_VERSION ?= 0.9.0 SSHCOMMAND_VERSION ?= 0.16.0 DOCKER_IMAGE_LABELER_URL ?= https://github.com/dokku/docker-image-labeler/releases/download/v${DOCKER_IMAGE_LABELER_VERSION}/docker-image-labeler_${DOCKER_IMAGE_LABELER_VERSION}_linux_${TARGETARCH}.tgz +DOCKER_CONTAINER_HEALTHCHECKER_URL ?= https://github.com/dokku/docker-container-healthchecker/releases/download/v${DOCKER_CONTAINER_HEALTHCHECKER_VERSION}/docker-container-healthchecker_${DOCKER_CONTAINER_HEALTHCHECKER_VERSION}_linux_${TARGETARCH}.tgz LAMBDA_BUILDER_URL ?= https://github.com/dokku/lambda-builder/releases/download/v${LAMBDA_BUILDER_VERSION}/lambda-builder_${LAMBDA_BUILDER_VERSION}_linux_${TARGETARCH}.tgz NETRC_URL ?= https://github.com/dokku/netrc/releases/download/v${NETRC_VERSION}/netrc_${NETRC_VERSION}_linux_${TARGETARCH}.tgz PLUGN_URL ?= https://github.com/dokku/plugn/releases/download/v${PLUGN_VERSION}/plugn_${PLUGN_VERSION}_linux_${TARGETARCH}.tgz @@ -135,7 +137,7 @@ plugin-dependencies: plugn procfile-util plugins: plugn procfile-util docker sudo -E dokku plugin:install --core -dependencies: apt-update docker-image-labeler lambda-builder netrc sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq parallel +dependencies: apt-update docker-image-labeler docker-container-healthchecker lambda-builder netrc sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq parallel $(MAKE) -e stack apt-update: @@ -161,6 +163,11 @@ docker-image-labeler: tar xzf /tmp/docker-image-labeler_latest.tgz -C /usr/local/bin mv /usr/local/bin/docker-image-labeler-${TARGETARCH} /usr/local/bin/docker-image-labeler +docker-container-healthchecker: + wget -qO /tmp/docker-container-healthchecker_latest.tgz ${DOCKER_CONTAINER_HEALTHCHECKER_URL} + tar xzf /tmp/docker-container-healthchecker_latest.tgz -C /usr/local/bin + mv /usr/local/bin/docker-container-healthchecker-${TARGETARCH} /usr/local/bin/docker-container-healthchecker + lambda-builder: wget -qO /tmp/lambda-builder_latest.tgz ${LAMBDA_BUILDER_URL} tar xzf /tmp/lambda-builder_latest.tgz -C /usr/local/bin diff --git a/contrib/images/digitalocean/in_parts/012-dokku-packages b/contrib/images/digitalocean/in_parts/012-dokku-packages index 768c44924..193327852 100755 --- a/contrib/images/digitalocean/in_parts/012-dokku-packages +++ b/contrib/images/digitalocean/in_parts/012-dokku-packages @@ -33,7 +33,8 @@ pkgs=(apache2-utils debconf ) -dokku_pkgs=(docker-image-labeler +dokku_pkgs=(docker-docker-container-healthchecker + docker-image-labeler gliderlabs-sigil lambda-builder netrc diff --git a/debian/control b/debian/control index 146b6c40d..df54edcc9 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.30.11 Section: web Priority: optional Architecture: amd64 -Depends: apache2-utils, locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-compose-plugin | moby-compose, docker-image-labeler (>= 0.2.2), lambda-builder, net-tools, netrc, software-properties-common, parallel, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsync, rsyslog, dos2unix, jq, unzip +Depends: apache2-utils, locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-compose-plugin | moby-compose, docker-container-healthchecker (>= 0.6.4), docker-image-labeler (>= 0.2.2), lambda-builder, net-tools, netrc, software-properties-common, parallel, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsync, rsyslog, dos2unix, jq, unzip Recommends: herokuish (>= 0.3.4), bash-completion, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn (>= 0.3.0), sudo, python3, debconf Maintainer: Jose Diaz-Gonzalez diff --git a/docker/etc/sudoers.d/dokku-docker b/docker/etc/sudoers.d/dokku-docker index 8150da4a5..6dfc5d175 100644 --- a/docker/etc/sudoers.d/dokku-docker +++ b/docker/etc/sudoers.d/dokku-docker @@ -1 +1 @@ -dokku ALL=NOPASSWD:SETENV:/usr/bin/docker,/usr/bin/docker-image-labeler,/usr/bin/pack,/usr/bin/crontab +dokku ALL=NOPASSWD:SETENV:/usr/bin/docker,/usr/bin/docker-container-healthchecker,/usr/bin/docker-image-labeler,/usr/bin/pack,/usr/bin/crontab diff --git a/docker/usr/local/bin/docker-container-healthchecker b/docker/usr/local/bin/docker-container-healthchecker new file mode 100755 index 000000000..8453942f2 --- /dev/null +++ b/docker/usr/local/bin/docker-container-healthchecker @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -eo pipefail +[[ $TRACE ]] && set -x + +main() { + declare desc="re-runs docker-container-healthchecker commands as sudo" + local DOCKER_CONTAINER_HEALTHCHECKER_BIN="" + if [[ -x "/usr/bin/docker-container-healthchecker" ]]; then + DOCKER_CONTAINER_HEALTHCHECKER_BIN="/usr/bin/docker-container-healthchecker" + fi + + if [[ -z "$DOCKER_CONTAINER_HEALTHCHECKER_BIN" ]]; then + echo "! No docker-container-healthchecker binary found" 1>&2 + exit 1 + fi + + sudo -E "$DOCKER_CONTAINER_HEALTHCHECKER_BIN" "$@" +} + +main "$@" diff --git a/docs/appendices/0.31.0-migration-guide.md b/docs/appendices/0.31.0-migration-guide.md index 7ed392dc8..95a76181d 100644 --- a/docs/appendices/0.31.0-migration-guide.md +++ b/docs/appendices/0.31.0-migration-guide.md @@ -8,6 +8,7 @@ - Users no longer need to clear the `source-image` git property when transitioning from image-based deploys (`git:from-image` and `git:load-image`) to other deployment methods (git push, `git:from-archive`, `git:sync`). - For deploys via the `git:from-image` and `git:load-image` commands, the `CHECKS` file is now extracted from the configured `WORKDIR` property of the image. For all other deploys - git push, `git:from-archive`, `git:sync` - will have the `CHECKS` extracted directly from the source code. The filename in both cases is `CHECKS` and cannot be modified. - Port mappings are now auto-detected during the build process. Users may override detected port mappings via the `ports` plugin. A default port mapping of `http:80:5000` will be used if none is otherwise specified. +- Users building docker images that run Dokku will need to use a new sudoer wrapper for the `docker-container-healthchecker` binary to work correctly. A reference version has been placed in the `docker` skeleton directory. This should only impact platform developers, and users of our Docker image will already have the file available. ## Deprecations @@ -17,6 +18,7 @@ - The `common#get_dockerfile_exposed_ports()` function is deprecated and will be removed in the next release. There is no replacement for this as it's only use in Dokku core was in the `builder-dockerfile` plugin. - The `common#get_exposed_ports_from_image()` function is deprecated and will be removed in the next release. There is no replacement for this as it's only use in Dokku core was during the build process. - The environment variable `DOKKU_PROXY_PORT_MAP` has been migrated to the properties system. Direct changes to the value will be ignored, and users should interact with port maps via the `ports` plugin. +- The `CHECKS` file is deprecated in favor of defining healthchecks in the `app.json` file. The [docker-container-healthchecker](https://github.com/dokku/docker-container-healthchecker) tool can be used to generate healthcheck entries in `app.json` format from existing `CHECKS` files. See the [zero-downtime deploy documentation](/docs/deployment/zero-downtime-deploys.md) for more information on how the new zero downtime check format works. ## Un-Deprecations diff --git a/docs/deployment/zero-downtime-deploys.md b/docs/deployment/zero-downtime-deploys.md index 1d6bc3f88..57738dbaa 100644 --- a/docs/deployment/zero-downtime-deploys.md +++ b/docs/deployment/zero-downtime-deploys.md @@ -13,7 +13,7 @@ checks:skip [process-type(s)] Skip zero-downtime checks for all proc By default, Dokku will wait `10` seconds after starting each container before assuming it is up and proceeding with the deploy. Once this has occurred for all containers started by an application, traffic will be switched to point to your new containers. Dokku will also wait a further `60` seconds *after* the deploy is complete before terminating old containers in order to give time for long running connections to terminate. In either case, you may have more than one container running for a given application. -You may both create user-defined checks for web processes using a `CHECKS` file, as well as customize any and all parts of this experience using the checks plugin. +You may both create user-defined checks for web processes using the `healthchecks` key in the `app.json` file, as well as customize any and all parts of this experience using the checks plugin. > Web checks are performed via `curl` on Dokku host. Some application code - such > as the Django framework - checks for specific hostnames or header values, these @@ -42,7 +42,7 @@ There are certain settings that can be configured via environment variables: - `DOKKU_DEFAULT_CHECKS_WAIT`: (default: `10`) If no user-defined checks are specified - or if the process being checked is not a `web` process - this is the period of time Dokku will wait before checking that a container is still running. - `DOKKU_DOCKER_STOP_TIMEOUT`: (default: `10`) Configurable grace period given to the `docker stop` command. If a container has not stopped by this time, a `kill -9` signal or equivalent is sent in order to force-terminate the container. Both the `ps:stop` and `apps:destroy` commands *also* respect this value. If not specified, the Docker defaults for the [`docker stop` command](https://docs.docker.com/engine/reference/commandline/stop/) will be used. -The following settings may also be specified in the `CHECKS` file, though are available as environment variables in order to ease application reuse. +The following settings may also be specified in the `app.json` file, though are available as environment variables in order to ease application reuse. - `DOKKU_CHECKS_WAIT`: (default: `5`) Wait this many seconds for the container to start before running checks. - `DOKKU_CHECKS_TIMEOUT`: (default: `30`) Wait this many seconds for each response before marking it as a failure. @@ -135,95 +135,45 @@ dokku checks:report node-js-app --checks-disabled-list ## Customizing checks +> New as of 0.31.0 + If your application needs a longer period to boot up - perhaps to load data into memory, or because of slow boot time - you may also use Dokku's `checks` functionality to more precisely check whether an application can serve traffic or not. -Checks are run against the detected `web` process from your application's `Procfile`. For non-web processes, Dokku will fallback to the aforementioned process uptime check. +Healthchecks are run against all process from your application's `Procfile`. When no healthcheck is defined, Dokku will fallback to a process uptime check. -For deploys via the `git:from-image` and `git:load-image` commands, the `CHECKS` file is extracted from the configured `WORKDIR` property of the image. For all other deploys - git push, `git:from-archive`, `git:sync` - will have the `CHECKS` extracted directly from the source code. The filename in both cases is `CHECKS` and cannot be modified. The `CHECKS` file should be plain text and may contain: +One or more healthchecks can be defined in the `app.json` file - see the [deployment task documentation](/docs/advanced-usage/deployment-tasks.md) for more information on how this is extracted - under the `healthchecks.web` path: -- check instructions -- settings (NAME=VALUE) -- comments (lines starting with #) -- empty lines - -### Check instructions - -The format of a check instruction is a path or relative URL, optionally followed by the expected content: - -``` -/about Our Amazing Team +```json +{ + "healthchecks": { + "web": [ + { + "type": "startup", + "name": "web check", + "description": "Checking if the app responds to the /health/ready endpoint", + "path": "/health/ready", + "attempts": 3 + } + ] +} ``` -The `CHECKS` file can contain multiple checks: +A healthcheck entry takes the following properties: -``` -/ My Amazing App -/stylesheets/index.css .body -/scripts/index.js $(function() -/images/logo.png -``` +- `attempts`: (default: `3` seconds) Number of retry attempts to perform on failure. +- `command`: (default `''` - empty string) Command to execute within container. +- `content`: (default: `''` - empty string) Content to search in http path check output. +initialDelay default: 0, unit: seconds) Number of seconds to wait after a container has started before triggering the healthcheck. +- `name`: (default: autogenerated) The name of the healthcheck. If unspecified, it will be autogenerated from the rest of the healthcheck information. +- `path`: (default: `/` - for http checks): An http path to check. +- `timeout`: (default: `5` seconds): Number of seconds to wait before timing out a healthcheck. +- `type`: (default: `""` - none): Type of the healthcheck. Options: liveness, readiness, startup. +- `uptime`: (default: `""` - none): Amount of time the container must be alive before the container is considered healthy. Any restarts will cause this to check to fail, and this check does not respect retries. +- `wait`: (default: `5` seconds): Number of seconds to wait between healthcheck attempts. -To check an application that supports multiple hostnames, use relative URLs that include the hostname: +> Warning: Healthchecks are implemented by specific scheduler plugins, and not all plugins support all options. Please consult the scheduler documentation for further details on what is supported. -``` -//admin.dokku.me Admin Dashboard -//static.dokku.me/logo.png -``` - -You can also specify the protocol to explicitly check HTTPS requests: - -``` -https://admin.dokku.me Admin Dashboard -https://static.dokku.me/logo.png -``` - -While a full URL may be used in order to invoke checks, if you are using relative URLs, the port *must* be omitted. - -> Changed as of 0.22.5 - -Please note that dollar sign bracket characters (`{` and `}`) must be escaped when used within a `CHECKS` file. Escaping follows golang template rules. The proper way to do this is via one of the following methods: - -``` -# escaping the `{` character -# using double-quotes -{{"{"}} - -# using raw string constants -{{`{`}} - -# escaping the `}` character -# using double-quotes -{{"}"}} - -# using raw string constants -{{`}`}} -``` - -### Templating Checks Files - -> New as of 0.22.5 - -An app's `CHECKS` file is sent through a single pass of the [`sigil`](https://github.com/gliderlabs/sigil/) templating tool. This enables usage of Golang templating within application `CHECKS` files. In addition to general templating access, access to app environment variables is also allowed via the `var` function: - -``` -{{ var "SOME_ENV_VAR" }} -``` - -This may be useful if certain zero-downtime checks require access to an app-specific value, such as a domain name. - -### Check settings - -The default behavior is to wait for `5` seconds before running the checks, to timeout the checks after `30` seconds, and to attempt the checks `5` times. If the checks fail `5` times, the deployment is considered failed and the old container will continue serving traffic. - -You can change the default behavior by setting `WAIT`, `TIMEOUT`, and `ATTEMPTS` to different values in the `CHECKS` file: - -``` -WAIT=30 # Wait 1/2 minute -TIMEOUT=60 # Timeout after a minute -ATTEMPTS=10 # Attempt checks 10 times - -/ My Amazing App -``` +See the [docker-container-healthchecker](https://github.com/dokku/docker-container-healthchecker) documentation for more details on how healthchecks are interpreted. ## Manually invoking checks @@ -310,182 +260,3 @@ dokku checks:run node-js-app web.3 -----> Running pre-flight checks Invalid container id specified (APP.web.3) ``` - -## Example: Successful Rails deployment - -In this example, a Rails application is successfully deployed to Dokku. The initial round of checks fails while the server is starting, but once it starts they succeed and the deployment is successful. `WAIT` is set to `10` because our application takes a while to boot up. `ATTEMPTS` is set to `6`, but the third attempt succeeds. - -### Successful `CHECKS` file - -``` -WAIT=10 -ATTEMPTS=6 -/check.txt simple_check -``` - -For this check to work, we've added a line to `config/routes.rb` that simply returns a string: - -``` -get '/check.txt', to: proc {[200, {}, ['simple_check']]} -``` - -### Successful deploy output - -> Note: The output has been trimmed for brevity. - -```shell -git push dokku master -``` - -``` ------> Cleaning up... ------> Building node-js-app from herokuish... ------> Adding BUILD_ENV to build environment... ------> Ruby app detected ------> Compiling Ruby/Rails ------> Using Ruby version: ruby-2.0.0 - -..... - ------> Discovering process types - Procfile declares types -> web ------> Releasing node-js-app... ------> Deploying node-js-app... ------> Running pre-flight checks ------> Attempt 1/6 Waiting for 10 seconds ... - CHECKS expected result: - http://localhost/check.txt => "simple_check" - ! -curl: (7) Failed to connect to 172.17.0.155 port 5000: Connection refused - ! Check attempt 1/6 failed. ------> Attempt 2/6 Waiting for 10 seconds ... - CHECKS expected result: - http://localhost/check.txt => "simple_check" - ! -curl: (7) Failed to connect to 172.17.0.155 port 5000: Connection refused - ! Check attempt 2/6 failed. ------> Attempt 3/6 Waiting for 10 seconds ... - CHECKS expected result: - http://localhost/check.txt => "simple_check" ------> All checks successful! -=====> node-js-app container output: - => Booting Thin - => Rails 4.2.0 application starting in production on http://0.0.0.0:5000 - => Run `rails server -h` for more startup options - => Ctrl-C to shutdown server - Thin web server (v1.6.3 codename Protein Powder) - Maximum connections set to 1024 - Listening on 0.0.0.0:5000, CTRL+C to stop -=====> end node-js-app container output ------> Running post-deploy ------> Configuring myapp.dokku.me... ------> Creating http nginx.conf - Reloading nginx ------> Shutting down old container in 60 seconds -=====> Application deployed: - http://myapp.dokku.me -``` - -## Example: Failing Rails deployment - -In this example, a Rails application fails to deploy. The reason for the failure is that the PostgreSQL database connection fails. The initial checks will fail while we wait for the server to start up, just like in the above example. However, once the server does start accepting connections, we will see an error 500 due to the PostgreSQL database connection failure. - -Once the attempts have been exceeded, the deployment fails and we see the container output, which shows the PostgreSQL connection errors. - -### Failing `CHECKS` file - -``` -WAIT=10 -ATTEMPTS=6 -/ -``` - -> The check to the root url `/` would normally access the database. - -### Failing deploy output - -> Note: The output has been trimmed for brevity. - -```shell -git push dokku master -``` - -``` ------> Cleaning up... ------> Building node-js-app from herokuish... ------> Adding BUILD_ENV to build environment... ------> Ruby app detected ------> Compiling Ruby/Rails ------> Using Ruby version: ruby-2.0.0 - -..... - -Discovering process types -Procfile declares types -> web -Releasing node-js-app... -Deploying node-js-app... -Running pre-flight checks ------> Attempt 1/6 Waiting for 10 seconds ... - CHECKS expected result: - http://localhost/ => "" - ! -curl: (7) Failed to connect to 172.17.0.188 port 5000: Connection refused - ! Check attempt 1/6 failed. ------> Attempt 2/6 Waiting for 10 seconds ... - CHECKS expected result: - http://localhost/ => "" - ! -curl: (7) Failed to connect to 172.17.0.188 port 5000: Connection refused - ! Check attempt 2/6 failed. ------> Attempt 3/6 Waiting for 10 seconds ... - CHECKS expected result: - http://localhost/ => "" - ! -curl: (22) The requested URL returned error: 500 Internal Server Error - ! Check attempt 3/6 failed. ------> Attempt 4/6 Waiting for 10 seconds ... - CHECKS expected result: - http://localhost/ => "" - ! -curl: (22) The requested URL returned error: 500 Internal Server Error - ! Check attempt 4/6 failed. ------> Attempt 5/6 Waiting for 10 seconds ... - CHECKS expected result: - http://localhost/ => "" - ! -curl: (22) The requested URL returned error: 500 Internal Server Error - ! Check attempt 5/6 failed. ------> Attempt 6/6 Waiting for 10 seconds ... - CHECKS expected result: - http://localhost/ => "" - ! -curl: (22) The requested URL returned error: 500 Internal Server Error -Could not start due to 1 failed checks. - ! Check attempt 6/6 failed. -=====> node-js-app container output: - => Booting Thin - => Rails 4.2.0 application starting in production on http://0.0.0.0:5000 - => Run `rails server -h` for more startup options - => Ctrl-C to shutdown server - Thin web server (v1.6.3 codename Protein Powder) - Maximum connections set to 1024 - Listening on 0.0.0.0:5000, CTRL+C to stop - Started GET "/" for 172.17.42.1 at 2015-03-26 21:36:47 +0000 - Is the server running on host "172.17.42.1" and accepting - TCP/IP connections on port 5431? - PG::ConnectionBad (could not connect to server: Connection refused - Is the server running on host "172.17.42.1" and accepting - TCP/IP connections on port 5431? - ): - vendor/bundle/ruby/2.0.0/gems/activerecord-4.2.0/lib/active_record/connection_adapters/postgresql_adapter.rb:651:in `initialize' - vendor/bundle/ruby/2.0.0/gems/activerecord-4.2.0/lib/active_record/connection_adapters/postgresql_adapter.rb:651:in `new' - vendor/bundle/ruby/2.0.0/gems/activerecord-4.2.0/lib/active_record/connection_adapters/postgresql_adapter.rb:651:in `connect' - vendor/bundle/ruby/2.0.0/gems/activerecord-4.2.0/lib/active_record/connection_adapters/postgresql_adapter.rb:242:in `initialize' - vendor/bundle/ruby/2.0.0/gems/activerecord-4.2.0/lib/active_record/connection_adapters/postgresql_adapter.rb:44:in `new' - vendor/bundle/ruby/2.0.0/gems/activerecord-4.2.0/lib/active_record/connection_adapters/postgresql_adapter.rb:44:in `postgresql_connection -=====> end node-js-app container output -/usr/bin/dokku: line 49: 23409 Killed dokku deploy "$APP" -To dokku@dokku.me:myapp - ! [remote rejected] dokku -> master (pre-receive hook declined) -error: failed to push some refs to 'dokku@dokku.me:myapp' -``` diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 8c4ad2461..1f75aef43 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -104,6 +104,21 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x # TODO ``` +### `app-json-get-content` + +- Description: Outputs the contents of the app-json file, if any +- Invoked by: Deployment checks +- Arguments: `$APP` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +# TODO +``` + ### `app-maybe-create` - Description: Creates an app (gated by whether this is globally enabled or not) diff --git a/docs/processes/process-management.md b/docs/processes/process-management.md index 1527ccb29..9d3e5fbae 100644 --- a/docs/processes/process-management.md +++ b/docs/processes/process-management.md @@ -140,7 +140,6 @@ For initial app deploys, Dokku will default to starting a single `web` process f There are also a few other exceptions for the `web` process. -- Custom checks defined by a `CHECKS` file only apply to the `web` process type. - By default, the built-in nginx proxy implementation only proxies the `web` process (others may be handled via a custom `nginx.conf.sigil`). - See the [nginx request proxying documentation](/docs/networking/proxies/nginx.md#request-proxying) for more information on how nginx handles proxied requests. - Only the `web` process may be bound to an external port. diff --git a/plugins/20_events/app-json-get-content b/plugins/20_events/app-json-get-content new file mode 120000 index 000000000..5178a749f --- /dev/null +++ b/plugins/20_events/app-json-get-content @@ -0,0 +1 @@ +hook \ No newline at end of file diff --git a/plugins/app-json/Makefile b/plugins/app-json/Makefile index db0133b5d..bff87b1a8 100644 --- a/plugins/app-json/Makefile +++ b/plugins/app-json/Makefile @@ -1,5 +1,5 @@ SUBCOMMANDS = subcommands/report subcommands/set -TRIGGERS = triggers/app-json-process-deploy-parallelism triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/post-deploy triggers/pre-deploy triggers/report +TRIGGERS = triggers/app-json-process-deploy-parallelism triggers/app-json-get-content triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/post-deploy triggers/pre-deploy triggers/report BUILD = commands subcommands triggers PLUGIN_NAME = app-json diff --git a/plugins/app-json/src/triggers/triggers.go b/plugins/app-json/src/triggers/triggers.go index 1b49fdc4d..04580d2e2 100644 --- a/plugins/app-json/src/triggers/triggers.go +++ b/plugins/app-json/src/triggers/triggers.go @@ -22,6 +22,9 @@ func main() { appName := flag.Arg(0) processType := flag.Arg(1) err = appjson.TriggerAppJSONProcessDeployParallelism(appName, processType) + case "app-json-get-content": + appName := flag.Arg(0) + err = appjson.TriggerAppJSONGetContent(appName) case "core-post-deploy": appName := flag.Arg(0) err = appjson.TriggerCorePostDeploy(appName) diff --git a/plugins/app-json/triggers.go b/plugins/app-json/triggers.go index b6e117162..4f33d12c8 100644 --- a/plugins/app-json/triggers.go +++ b/plugins/app-json/triggers.go @@ -37,6 +37,28 @@ func TriggerAppJSONProcessDeployParallelism(appName string, processType string) return nil } +// TriggerAppJSONGetContent outputs the contents of the app-json file, if any +func TriggerAppJSONGetContent(appName string) error { + if !hasAppJSON(appName) { + fmt.Print("{}") + return nil + } + + b, err := os.ReadFile(getProcessSpecificAppJSONPath(appName)) + if err != nil { + return fmt.Errorf("Cannot read app.json file: %v", err) + } + + content := strings.TrimSpace(string(b)) + if content == "" { + fmt.Print("{}") + return nil + } + + fmt.Print(content) + return nil +} + // TriggerCorePostDeploy sets a property to // allow the app to be restored on boot func TriggerCorePostDeploy(appName string) error { diff --git a/plugins/scheduler-docker-local/check-deploy b/plugins/scheduler-docker-local/check-deploy index 678ae2677..cdf2c7610 100755 --- a/plugins/scheduler-docker-local/check-deploy +++ b/plugins/scheduler-docker-local/check-deploy @@ -44,6 +44,7 @@ trigger-scheduler-docker-local-check-deploy() { declare desc="scheduler-docker-local check-deploy plugin trigger" declare trigger="check-deploy" declare APP="$1" DOKKU_APP_CONTAINER_ID="$2" DOKKU_APP_CONTAINER_TYPE="$3" DOKKU_APP_LISTEN_PORT="$4" DOKKU_APP_LISTEN_IP="$5" CONTAINER_INDEX="$6" + local content local DOKKU_SCHEDULER=$(get_app_scheduler "$APP") if [[ "$DOKKU_SCHEDULER" != "docker-local" ]]; then @@ -53,9 +54,6 @@ trigger-scheduler-docker-local-check-deploy() { if [[ -z "$DOKKU_APP_LISTEN_PORT" ]] && [[ -f "$DOKKU_ROOT/$APP/PORT" ]]; then local DOKKU_APP_LISTEN_PORT=$(<"$DOKKU_ROOT/$APP/PORT") fi - if [[ -z "$DOKKU_APP_LISTEN_IP" ]] && [[ -f "$DOKKU_ROOT/$APP/IP" ]]; then - local DOKKU_APP_LISTEN_IP=$(<"$DOKKU_ROOT/$APP/IP") - fi if [[ -z "$DOKKU_APP_CONTAINER_ID" ]]; then local DOKKU_APP_CIDS=($(get_app_container_ids "$APP")) local DOKKU_APP_CONTAINER_ID=${DOKKU_APP_CIDS[0]} @@ -81,6 +79,34 @@ trigger-scheduler-docker-local-check-deploy() { local IMAGE_TAG="$(get_running_image_tag "$APP")" local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") + local TMP_APP_JSON_OUTPUT=$(mktemp "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX") + trap "rm -rf '$TMP_APP_JSON_OUTPUT' >/dev/null" RETURN INT TERM EXIT + + plugn trigger app-json-get-content "$APP" >"$TMP_APP_JSON_OUTPUT" + if [[ -s "${CHECKS_FILENAME}" ]]; then + # Reads name/value pairs, sets the WAIT and TIMEOUT variables + exec <"$CHECKS_FILENAME" + local line + local NAME + local VALUE + while read -r line; do + line=$(strip_inline_comments "$line") + # Name/value pair + if [[ "$line" =~ ^.+= ]]; then + NAME=${line%=*} + VALUE=${line#*=} + [[ "$NAME" == "WAIT" ]] && local WAIT=$VALUE + [[ "$NAME" == "TIMEOUT" ]] && local TIMEOUT=$VALUE + [[ "$NAME" == "ATTEMPTS" ]] && local ATTEMPTS=$VALUE + fi + done + + dokku_log_warn "Deprecated: Usage of the CHECKS file is deprecated in favor of healthchecks in app.json" + dokku_log_warn "Please move your healthchecks to app.json." + content="$(docker-container-healthchecker convert "$CHECKS_FILENAME" --app-json "$TMP_APP_JSON_OUTPUT" --pretty)" + echo "$content" >"$TMP_APP_JSON_OUTPUT" + fi + checks_check_deploy_cleanup() { declare desc="print container output" local id="$1" @@ -93,127 +119,40 @@ trigger-scheduler-docker-local-check-deploy() { } trap "checks_check_deploy_cleanup $DOKKU_APP_CONTAINER_ID" RETURN INT TERM EXIT - # We allow custom check for web instances only - if [[ ! -s "${CHECKS_FILENAME}" ]] || [[ "$DOKKU_APP_CONTAINER_TYPE" != "web" ]]; then - # simple default check to see if the container stuck around - local DOKKU_DEFAULT_CHECKS_WAIT="${DOKKU_DEFAULT_CHECKS_WAIT:-10}" - dokku_log_verbose "Waiting for $DOKKU_DEFAULT_CHECKS_WAIT seconds ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" - sleep "$DOKKU_DEFAULT_CHECKS_WAIT" + local DOKKU_DEFAULT_CHECKS_WAIT="${DOKKU_DEFAULT_CHECKS_WAIT:-10}" + content="$(docker-container-healthchecker add "$DOKKU_APP_CONTAINER_TYPE" --app-json "$TMP_APP_JSON_OUTPUT" --if-empty --pretty --uptime "$DOKKU_DEFAULT_CHECKS_WAIT")" + echo "$content" >"$TMP_APP_JSON_OUTPUT" - ! (is_container_status "$DOKKU_APP_CONTAINER_ID" "Running") && dokku_log_fail "App container failed to start ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" - local container_restarts="$("$DOCKER_BIN" container inspect --format "{{ .RestartCount }}" "$DOKKU_APP_CONTAINER_ID")" - if [[ $container_restarts -ne 0 ]]; then - "$DOCKER_BIN" container update --restart=no "$DOKKU_APP_CONTAINER_ID" &>/dev/null || true - "$DOCKER_BIN" container stop "$DOKKU_APP_CONTAINER_ID" || true - dokku_log_fail "App container failed to start ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" - fi + local FAILEDCHECKS=0 + local SSL="$DOKKU_ROOT/$APP/tls" + declare -a ARG_ARRAY + ARG_ARRAY+=("--app-json") + ARG_ARRAY+=("$TMP_APP_JSON_OUTPUT") + ARG_ARRAY+=("--process-type") + ARG_ARRAY+=("$DOKKU_APP_CONTAINER_TYPE") - trap - EXIT - dokku_log_verbose "Default container check successful ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" && exit 0 + if [[ -e "$SSL/server.crt" && -e "$SSL/server.key" ]]; then + ARG_ARRAY+=("--header") + ARG_ARRAY+=("X-Forwarded-Proto: https") fi - # Reads name/value pairs, sets the WAIT and TIMEOUT variables - exec <"$CHECKS_FILENAME" - local line - local NAME - local VALUE - while read -r line; do - line=$(strip_inline_comments "$line") - # Name/value pair - if [[ "$line" =~ ^.+= ]]; then - NAME=${line%=*} - VALUE=${line#*=} - [[ "$NAME" == "WAIT" ]] && local WAIT=$VALUE - [[ "$NAME" == "TIMEOUT" ]] && local TIMEOUT=$VALUE - [[ "$NAME" == "ATTEMPTS" ]] && local ATTEMPTS=$VALUE - fi - done + if [[ -n "$DOKKU_APP_LISTEN_IP" ]]; then + ARG_ARRAY+=("--ip-address") + ARG_ARRAY+=("$DOKKU_APP_LISTEN_IP") + fi - exec <"$CHECKS_FILENAME" - local CHECK_URL - local EXPECTED - local FAILEDCHECKS=0 - while read -r CHECK_URL EXPECTED; do - # Ignore empty lines and lines starting with # - [[ -z "$CHECK_URL" || "$CHECK_URL" =~ ^\# ]] && continue - # Ignore if it's not a URL in a supported format - ! [[ "$CHECK_URL" =~ ^(http(s)?:)?\/.* ]] && continue + if [[ -n "$DOKKU_APP_LISTEN_PORT" ]]; then + ARG_ARRAY+=("--port") + ARG_ARRAY+=("$DOKKU_APP_LISTEN_PORT") + fi - if [[ "$CHECK_URL" =~ ^https?: ]]; then - local URL_PROTOCOL=${CHECK_URL%:*} - local CHECK_URL=${CHECK_URL#*:} - else - local URL_PROTOCOL="http" - fi - - if [[ "$CHECK_URL" =~ ^//.+ ]]; then - # To test a URL with specific host name, we still make request to localhost, - # but we set Host header to $SEND_HOST. - # - # The pattern is - # //SEND_HOST/PATHNAME - local UNPREFIXED=${CHECK_URL#//} - local URL_HOSTNAME=${UNPREFIXED%%/*} - local URL_PATHNAME=${UNPREFIXED#$URL_HOSTNAME} - - local HEADERS="-H Host:$URL_HOSTNAME" - else - local URL_HOSTNAME=localhost - local URL_PATHNAME=$CHECK_URL - fi - - # -q Do not use .curlrc (must come first) - # --compressed Test compression handled correctly - # --fail Fail on server errors (4xx, 5xx) - # --location Follow redirects - # --noproxy Do not use http_proxy env variable - local CURL_OPTIONS="-q --compressed --fail --location --noproxy $DOKKU_APP_LISTEN_IP --max-time $TIMEOUT" - - # Set X-Forwarded-Proto header if TLS is enabled. - local SSL="$DOKKU_ROOT/$APP/tls" - if [[ -e "$SSL/server.crt" && -e "$SSL/server.key" ]]; then - local CURL_OPTIONS+=" -H X-Forwarded-Proto:https" - fi - - # LOG_URL is used in verbose output below - local LOG_URL="$URL_PROTOCOL://$URL_HOSTNAME$URL_PATHNAME" - # CURL_ARGS includes the behinds the scenes options for hitting the container service directly - local CURL_ARGS="$CURL_OPTIONS $URL_PROTOCOL://$DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT$URL_PATHNAME $HEADERS" - - dokku_log_verbose "CHECKS expected result: $LOG_URL => \"$EXPECTED\" ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" - - local ATTEMPT=0 - local SUCCESS=0 - until [[ $SUCCESS == 1 || $ATTEMPT -ge $ATTEMPTS ]]; do - local ATTEMPT=$((ATTEMPT + 1)) - dokku_log_verbose "Attempt $ATTEMPT/$ATTEMPTS. Waiting for $WAIT seconds ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" - sleep "$WAIT" - - # Capture HTTP response or CURL error message - if OUTPUT=$(curl -# $CURL_ARGS 2>&1); then - # OUTPUT contains the HTTP response - # shellcheck disable=SC2076 - if [[ "$OUTPUT" =~ "$EXPECTED" ]]; then - SUCCESS=1 - break - fi - dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\" ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" - else - # Failed to connect/no response, OUTPUT contains error message - dokku_log_warn "$OUTPUT" - fi - - dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" - done - - if [[ $SUCCESS -ne 1 ]]; then - FAILEDCHECKS=$((FAILEDCHECKS + 1)) - fi - done + docker-container-healthchecker check "$DOKKU_APP_CONTAINER_ID" "${ARG_ARRAY[@]}" || FAILEDCHECKS="$?" if [[ $FAILEDCHECKS -gt 0 ]]; then - dokku_log_fail "Could not start due to $FAILEDCHECKS failed checks ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" - exit 1 + "$DOCKER_BIN" container update --restart=no "$DOKKU_APP_CONTAINER_ID" &>/dev/null || true + "$DOCKER_BIN" container stop "$DOKKU_APP_CONTAINER_ID" || true + dokku_log_warn "Could not start due to $FAILEDCHECKS failed checks ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" + return 1 fi trap - EXIT diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh index 31bdfbaff..7e2699fcc 100755 --- a/tests/ci/setup.sh +++ b/tests/ci/setup.sh @@ -14,6 +14,12 @@ install_dependencies() { curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/focal/docker-image-labeler_${DOCKER_IMAGE_LABELER_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${DOCKER_IMAGE_LABELER_PACKAGE_NAME}" fi + DOCKER_CONTAINER_HEALTHCHECKER_VERSION=$(grep DOCKER_CONTAINER_HEALTHCHECKER_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3) + DOCKER_CONTAINER_HEALTHCHECKER_PACKAGE_NAME="docker-container-healthchecker_${DOCKER_CONTAINER_HEALTHCHECKER_VERSION}_amd64.deb" + if [[ ! -f "$ROOT_DIR/build/${DOCKER_CONTAINER_HEALTHCHECKER_PACKAGE_NAME}" ]]; then + curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/focal/docker-container-healthchecker_${DOCKER_CONTAINER_HEALTHCHECKER_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${DOCKER_CONTAINER_HEALTHCHECKER_PACKAGE_NAME}" + fi + HEROKUISH_VERSION=$(grep HEROKUISH_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3) HEROKUISH_PACKAGE_NAME="herokuish_${HEROKUISH_VERSION}_amd64.deb" if [[ ! -f "$ROOT_DIR/build/${HEROKUISH_PACKAGE_NAME}" ]]; then @@ -62,6 +68,7 @@ install_dependencies() { ls -lah "${ROOT_DIR}/build/" sudo dpkg -i \ + "${ROOT_DIR}/build/$DOCKER_CONTAINER_HEALTHCHECKER_PACKAGE_NAME" \ "${ROOT_DIR}/build/$DOCKER_IMAGE_LABELER_PACKAGE_NAME" \ "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \ "${ROOT_DIR}/build/$LAMBDA_BUILDER_PACKAGE_NAME" \ diff --git a/tests/unit/checks.bats b/tests/unit/checks.bats index d91bb21a9..5a3886c1c 100644 --- a/tests/unit/checks.bats +++ b/tests/unit/checks.bats @@ -265,6 +265,6 @@ teardown() { run deploy_app nodejs-express dokku@$DOKKU_DOMAIN:$TEST_APP template_checks_file echo "output: $output" echo "status: $status" - assert_output_contains "/healthcheck" 2 + assert_output_contains "/healthcheck" assert_success }