Merge pull request #1791 from dokku/mh-pare-down-test-suite

don't run app deploy tests and spread out unit tests to 4 containers

tests/ci/parallel_runner.sh

@@ -4,6 +4,7 @@ set -eo pipefail
 
 MODE="$1"; MODE=${MODE:="testing"}
 
+# shellcheck disable=SC2120
 setup_circle() {
   echo "=====> setup_circle on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX"
   sudo -E CI=true make -e sshcommand
@@ -21,36 +22,39 @@ setup_circle() {
 }
 
 if [[ -n "$CIRCLE_NODE_INDEX" ]] && [[ "$MODE" == "setup" ]]; then
-  case "$CIRCLE_NODE_INDEX" in
-      3)
-        setup_circle buildstack
-        exit $?
-        ;;
-      *)
-        setup_circle
-        exit $?
-        ;;
-  esac
+  # shellcheck disable=SC2119
+  setup_circle
+  exit $?
+  # case "$CIRCLE_NODE_INDEX" in
+  #     3)
+  #       setup_circle buildstack
+  #       exit $?
+  #       ;;
+  #     *)
+  #       setup_circle
+  #       exit $?
+  #       ;;
+  # esac
 fi
 
 case "$CIRCLE_NODE_INDEX" in
   0)
-    echo "=====> make unit-tests (1/2) on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX"
+    echo "=====> make unit-tests (1/4) on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX"
     sudo -E UNIT_TEST_BATCH=1 make -e unit-tests
     ;;
 
   1)
-    echo "=====> make unit-tests (2/2) on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX"
+    echo "=====> make unit-tests (2/4) on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX"
     sudo -E UNIT_TEST_BATCH=2 make -e unit-tests
     ;;
 
   2)
-    echo "=====> make deploy-tests (herokuish release) on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX"
-    sudo -E make -e deploy-tests
+    echo "=====> make unit-tests (3/4) on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX"
+    sudo -E UNIT_TEST_BATCH=3 make -e unit-tests
     ;;
 
   3)
-    echo "=====> make deploy-tests (herokuish master) on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX"
-    sudo -E make -e deploy-tests
+    echo "=====> make unit-tests (4/4) on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX"
+    sudo -E UNIT_TEST_BATCH=4 make -e unit-tests
     ;;
 esac

tests/ci/unit_test_runner.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 is_number() {
-  local NUMBER=$1; local NUM_RE='^[0-9]+$'
+  local NUMBER=$1; local NUM_RE='^[1-4]+$'
   if [[ $NUMBER =~ $NUM_RE ]]; then
     return 0
   else
@@ -10,28 +10,13 @@ is_number() {
 }
 
 usage() {
-  echo "usage: $0 1|2"
+  echo "usage: $0 1|2|3|4"
   exit 0
 }
 
 BATCH_NUM="$1"
 is_number $BATCH_NUM || usage
 
-TESTS=($(find "$(dirname $0)"/../unit -maxdepth 1 -name "*.bats"))
-HALF_TESTS=$(( ${#TESTS[@]} / 2 ))
-FIRST_HALF=("${TESTS[@]:0:${HALF_TESTS}}")
-LAST_HALF=("${TESTS[@]:${HALF_TESTS}:${#TESTS[@]}}")
-
-case "$BATCH_NUM" in
-  1)
-    bats "${FIRST_HALF[@]}"
-  ;;
-
-  2)
-    bats "${LAST_HALF[@]}"
-  ;;
-
-  *)
-    usage
-  ;;
-esac
+TESTS=$(find "$(dirname $0)"/../unit -maxdepth 1 -name "${BATCH_NUM}0*.bats" | sort -n | xargs)
+echo "running the following tests $TESTS"
+bats $TESTS

tests/unit/10_core_1.bats

@@ -0,0 +1,70 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+setup() {
+  create_app
+}
+
+teardown() {
+  rm -rf /home/dokku/$TEST_APP/tls /home/dokku/tls
+  destroy_app
+  disable_tls_wildcard
+}
+
+assert_urls() {
+  urls=$@
+  run dokku urls $TEST_APP
+  echo "output: "$output
+  echo "status: "$status
+  assert_output < <(tr ' ' '\n' <<< "${urls}")
+}
+
+build_nginx_config() {
+  # simulate nginx post-deploy
+  dokku domains:setup $TEST_APP
+  dokku nginx:build-config $TEST_APP
+}
+
+@test "(core) remove exited containers" {
+  deploy_app
+  # make sure we have many exited containers of the same 'type'
+  run bash -c "for cnt in 1 2 3; do dokku run $TEST_APP hostname; done"
+  echo "output: "$output
+  echo "status: "$status
+  assert_success
+  run bash -c "docker ps -a -f 'status=exited' --no-trunc=false | grep '/exec hostname'"
+  echo "output: "$output
+  echo "status: "$status
+  assert_success
+  run dokku cleanup
+  echo "output: "$output
+  echo "status: "$status
+  assert_success
+  sleep 5  # wait for dokku cleanup to happen in the background
+  run bash -c "docker ps -a -f 'status=exited' --no-trunc=false | grep '/exec hostname'"
+  echo "output: "$output
+  echo "status: "$status
+  assert_failure
+  run bash -c "docker ps -a -f 'status=exited' -q --no-trunc=false"
+  echo "output: "$output
+  echo "status: "$status
+  assert_output ""
+}
+
+@test "(core) run (with tty)" {
+  deploy_app
+  run /bin/bash -c "dokku run $TEST_APP ls /app/package.json"
+  echo "output: "$output
+  echo "status: "$status
+  assert_success
+}
+
+@test "(core) run (without tty)" {
+  deploy_app
+  run /bin/bash -c ": |dokku run $TEST_APP ls /app/package.json"
+  echo "output: "$output
+  echo "status: "$status
+  assert_success
+}
+

tests/unit/20_config.bats

@@ -3,14 +3,17 @@
 load test_helper
 
 setup() {
-  [[ -f $DOKKU_ROOT/ENV ]] && mv -f $DOKKU_ROOT/ENV $DOKKU_ROOT/ENV.bak
-  sudo -H -u dokku /bin/bash -c "echo 'export global_test=true' > $DOKKU_ROOT/ENV"
+  [[ -f ${DOKKU_ROOT}/ENV ]] && mv -f ${DOKKU_ROOT}/ENV ${DOKKU_ROOT}/ENV.bak
+  sudo -H -u dokku /bin/bash -c "echo 'export global_test=true' > ${DOKKU_ROOT}/ENV"
   create_app
 }
 
 teardown() {
   destroy_app
-  [[ -f $DOKKU_ROOT/ENV.bak ]] && mv -f $DOKKU_ROOT/ENV.bak $DOKKU_ROOT/ENV
+  ls -la ${DOKKU_ROOT}
+  if [[ -f ${DOKKU_ROOT}/ENV.bak ]];then
+    mv -f ${DOKKU_ROOT}/ENV.bak ${DOKKU_ROOT}/ENV
+  fi
 }
 
 @test "(config) config:set --global" {

tests/unit/20_core_1.bats

@@ -0,0 +1,91 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+setup() {
+  [[ -f "$DOKKU_ROOT/VHOST" ]] && cp -f "$DOKKU_ROOT/VHOST" "$DOKKU_ROOT/VHOST.bak"
+  [[ -f "$DOKKU_ROOT/HOSTNAME" ]] && cp -f "$DOKKU_ROOT/HOSTNAME" "$DOKKU_ROOT/HOSTNAME.bak"
+  DOCKERFILE="$BATS_TMPDIR/Dockerfile"
+}
+
+teardown() {
+  destroy_app
+  [[ -f "$DOKKU_ROOT/VHOST.bak" ]] && mv "$DOKKU_ROOT/VHOST.bak" "$DOKKU_ROOT/VHOST"
+  [[ -f "$DOKKU_ROOT/HOSTNAME.bak" ]] && mv "$DOKKU_ROOT/HOSTNAME.bak" "$DOKKU_ROOT/HOSTNAME"
+}
+
+
+check_urls() {
+  local PATTERN="$1"
+  run bash -c "dokku --quiet urls $TEST_APP | egrep \"${1}\""
+  echo "output: "$output
+  echo "status: "$status
+  assert_success
+}
+
+@test "(core) port exposure (with global VHOST)" {
+  echo "dokku.me" > "$DOKKU_ROOT/VHOST"
+  deploy_app
+  CONTAINER_ID=$(< $DOKKU_ROOT/$TEST_APP/CONTAINER.web.1)
+  run bash -c "docker port $CONTAINER_ID | sed 's/[0-9.]*://' | egrep -q '[0-9]*'"
+  echo "output: "$output
+  echo "status: "$status
+  assert_failure
+
+  check_urls http://${TEST_APP}.dokku.me
+}
+
+@test "(core) port exposure (without global VHOST and real HOSTNAME)" {
+  rm "$DOKKU_ROOT/VHOST"
+  echo "${TEST_APP}.dokku.me" > "$DOKKU_ROOT/HOSTNAME"
+  deploy_app
+  CONTAINER_ID=$(< $DOKKU_ROOT/$TEST_APP/CONTAINER.web.1)
+  run bash -c "docker port $CONTAINER_ID | sed 's/[0-9.]*://' | egrep -q '[0-9]*'"
+  echo "output: "$output
+  echo "status: "$status
+  assert_success
+
+  HOSTNAME=$(< "$DOKKU_ROOT/HOSTNAME")
+  check_urls http://${HOSTNAME}:[0-9]+
+}
+
+@test "(core) port exposure (with NO_VHOST set)" {
+  deploy_app
+  dokku config:set $TEST_APP NO_VHOST=1
+  CONTAINER_ID=$(< $DOKKU_ROOT/$TEST_APP/CONTAINER.web.1)
+  run bash -c "docker port $CONTAINER_ID | sed 's/[0-9.]*://' | egrep -q '[0-9]*'"
+  echo "output: "$output
+  echo "status: "$status
+  assert_success
+
+  HOSTNAME=$(< "$DOKKU_ROOT/HOSTNAME")
+  check_urls http://${HOSTNAME}:[0-9]+
+}
+
+@test "(core) port exposure (without global VHOST and IPv4 address as HOSTNAME)" {
+  rm "$DOKKU_ROOT/VHOST"
+  echo "127.0.0.1" > "$DOKKU_ROOT/HOSTNAME"
+  deploy_app
+  CONTAINER_ID=$(< $DOKKU_ROOT/$TEST_APP/CONTAINER.web.1)
+  run bash -c "docker port $CONTAINER_ID | sed 's/[0-9.]*://' | egrep -q '[0-9]*'"
+  echo "output: "$output
+  echo "status: "$status
+  assert_success
+
+  HOSTNAME=$(< "$DOKKU_ROOT/HOSTNAME")
+  check_urls http://${HOSTNAME}:[0-9]+
+}
+
+@test "(core) port exposure (without global VHOST and IPv6 address as HOSTNAME)" {
+  rm "$DOKKU_ROOT/VHOST"
+  echo "fda5:c7db:a520:bb6d::aabb:ccdd:eeff" > "$DOKKU_ROOT/HOSTNAME"
+  deploy_app
+  CONTAINER_ID=$(< $DOKKU_ROOT/$TEST_APP/CONTAINER.web.1)
+  run bash -c "docker port $CONTAINER_ID | sed 's/[0-9.]*://' | egrep -q '[0-9]*'"
+  echo "output: "$output
+  echo "status: "$status
+  assert_success
+
+  HOSTNAME=$(< "$DOKKU_ROOT/HOSTNAME")
+  check_urls http://${HOSTNAME}:[0-9]+
+}

tests/unit/30_core_ports_2.bats

@@ -23,73 +23,6 @@ check_urls() {
   assert_success
 }
 
-@test "(core) port exposure (with global VHOST)" {
-  echo "dokku.me" > "$DOKKU_ROOT/VHOST"
-  deploy_app
-  CONTAINER_ID=$(< $DOKKU_ROOT/$TEST_APP/CONTAINER.web.1)
-  run bash -c "docker port $CONTAINER_ID | sed 's/[0-9.]*://' | egrep -q '[0-9]*'"
-  echo "output: "$output
-  echo "status: "$status
-  assert_failure
-
-  check_urls http://${TEST_APP}.dokku.me
-}
-
-@test "(core) port exposure (without global VHOST and real HOSTNAME)" {
-  rm "$DOKKU_ROOT/VHOST"
-  echo "${TEST_APP}.dokku.me" > "$DOKKU_ROOT/HOSTNAME"
-  deploy_app
-  CONTAINER_ID=$(< $DOKKU_ROOT/$TEST_APP/CONTAINER.web.1)
-  run bash -c "docker port $CONTAINER_ID | sed 's/[0-9.]*://' | egrep -q '[0-9]*'"
-  echo "output: "$output
-  echo "status: "$status
-  assert_success
-
-  HOSTNAME=$(< "$DOKKU_ROOT/HOSTNAME")
-  check_urls http://${HOSTNAME}:[0-9]+
-}
-
-@test "(core) port exposure (with NO_VHOST set)" {
-  deploy_app
-  dokku config:set $TEST_APP NO_VHOST=1
-  CONTAINER_ID=$(< $DOKKU_ROOT/$TEST_APP/CONTAINER.web.1)
-  run bash -c "docker port $CONTAINER_ID | sed 's/[0-9.]*://' | egrep -q '[0-9]*'"
-  echo "output: "$output
-  echo "status: "$status
-  assert_success
-
-  HOSTNAME=$(< "$DOKKU_ROOT/HOSTNAME")
-  check_urls http://${HOSTNAME}:[0-9]+
-}
-
-@test "(core) port exposure (without global VHOST and IPv4 address as HOSTNAME)" {
-  rm "$DOKKU_ROOT/VHOST"
-  echo "127.0.0.1" > "$DOKKU_ROOT/HOSTNAME"
-  deploy_app
-  CONTAINER_ID=$(< $DOKKU_ROOT/$TEST_APP/CONTAINER.web.1)
-  run bash -c "docker port $CONTAINER_ID | sed 's/[0-9.]*://' | egrep -q '[0-9]*'"
-  echo "output: "$output
-  echo "status: "$status
-  assert_success
-
-  HOSTNAME=$(< "$DOKKU_ROOT/HOSTNAME")
-  check_urls http://${HOSTNAME}:[0-9]+
-}
-
-@test "(core) port exposure (without global VHOST and IPv6 address as HOSTNAME)" {
-  rm "$DOKKU_ROOT/VHOST"
-  echo "fda5:c7db:a520:bb6d::aabb:ccdd:eeff" > "$DOKKU_ROOT/HOSTNAME"
-  deploy_app
-  CONTAINER_ID=$(< $DOKKU_ROOT/$TEST_APP/CONTAINER.web.1)
-  run bash -c "docker port $CONTAINER_ID | sed 's/[0-9.]*://' | egrep -q '[0-9]*'"
-  echo "output: "$output
-  echo "status: "$status
-  assert_success
-
-  HOSTNAME=$(< "$DOKKU_ROOT/HOSTNAME")
-  check_urls http://${HOSTNAME}:[0-9]+
-}
-
 @test "(core) port exposure (pre-deploy domains:add)" {
   create_app
   run dokku domains:add $TEST_APP www.test.app.dokku.me

tests/unit/40_core_2.bats

@@ -26,48 +26,6 @@ build_nginx_config() {
   dokku nginx:build-config $TEST_APP
 }
 
-@test "(core) remove exited containers" {
-  deploy_app
-  # make sure we have many exited containers of the same 'type'
-  run bash -c "for cnt in 1 2 3; do dokku run $TEST_APP hostname; done"
-  echo "output: "$output
-  echo "status: "$status
-  assert_success
-  run bash -c "docker ps -a -f 'status=exited' --no-trunc=false | grep '/exec hostname'"
-  echo "output: "$output
-  echo "status: "$status
-  assert_success
-  run dokku cleanup
-  echo "output: "$output
-  echo "status: "$status
-  assert_success
-  sleep 5  # wait for dokku cleanup to happen in the background
-  run bash -c "docker ps -a -f 'status=exited' --no-trunc=false | grep '/exec hostname'"
-  echo "output: "$output
-  echo "status: "$status
-  assert_failure
-  run bash -c "docker ps -a -f 'status=exited' -q --no-trunc=false"
-  echo "output: "$output
-  echo "status: "$status
-  assert_output ""
-}
-
-@test "(core) run (with tty)" {
-  deploy_app
-  run /bin/bash -c "dokku run $TEST_APP ls /app/package.json"
-  echo "output: "$output
-  echo "status: "$status
-  assert_success
-}
-
-@test "(core) run (without tty)" {
-  deploy_app
-  run /bin/bash -c ": |dokku run $TEST_APP ls /app/package.json"
-  echo "output: "$output
-  echo "status: "$status
-  assert_success
-}
-
 @test "(core) run (with --options)" {
   deploy_app
   run /bin/bash -c "dokku --force --quiet run $TEST_APP node --version"