diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu
index 1588bf5e..a1a60d63 100644
--- a/docker/Dockerfile.ubuntu
+++ b/docker/Dockerfile.ubuntu
@@ -79,6 +79,9 @@ if [ "$INSTALL_MEGATRON_DEPS" = "True" ]; then \
     SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") && echo $SITE_PACKAGES && \
     CUDNN_PATH=$SITE_PACKAGES/nvidia/cudnn CPLUS_INCLUDE_PATH=$SITE_PACKAGES/nvidia/cudnn/include \
     pip install git+https://github.com/NVIDIA/TransformerEngine.git@stable; \
+    cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/NVIDIA/apex && \
+    cd apex && pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ && \
+    cd / && rm -fr /tmp/apex && pip cache purge; \
 fi
 
 ENV SETUPTOOLS_USE_DISTUTILS=stdlib