From e1e273fb64a115ee28dad5a51864e9ca6daac676 Mon Sep 17 00:00:00 2001 From: Hans Kratz Date: Thu, 21 Oct 2021 11:02:47 +0200 Subject: [PATCH 1/3] CI: make cache download attempt time out after 10 minutes --- src/ci/docker/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index e42b4748fdc..ab81a9d1de2 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -70,7 +70,8 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then echo "Attempting to download $url" rm -f /tmp/rustci_docker_cache set +e - retry curl -y 30 -Y 10 --connect-timeout 30 -f -L -C - -o /tmp/rustci_docker_cache "$url" + retry curl --max-time 600 -y 30 -Y 10 --connect-timeout 30 -f -L -C - \ + -o /tmp/rustci_docker_cache "$url" loaded_images=$(docker load -i /tmp/rustci_docker_cache | sed 's/.* sha/sha/') set -e echo "Downloaded containers:\n$loaded_images" From 838e673d3b767c32dd54a3d8fee3c9caa65caa68 Mon Sep 17 00:00:00 2001 From: Hans Kratz Date: Thu, 21 Oct 2021 11:11:20 +0200 Subject: [PATCH 2/3] Debug output before loading docker images as that might hang. --- src/ci/docker/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index ab81a9d1de2..b9b6add99f8 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -72,6 +72,7 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then set +e retry curl --max-time 600 -y 30 -Y 10 --connect-timeout 30 -f -L -C - \ -o /tmp/rustci_docker_cache "$url" + echo "Loading images into docker" loaded_images=$(docker load -i /tmp/rustci_docker_cache | sed 's/.* sha/sha/') set -e echo "Downloaded containers:\n$loaded_images" From 3e6a69534f52b08057d18643b62d0f926b80909f Mon Sep 17 00:00:00 2001 From: Hans Kratz Date: Thu, 21 Oct 2021 22:52:00 +0200 Subject: [PATCH 3/3] Time out docker load after 10 minutes, kill after 12 due to CI hangs. --- src/ci/docker/run.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index b9b6add99f8..b1dacf79d26 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -73,7 +73,10 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then retry curl --max-time 600 -y 30 -Y 10 --connect-timeout 30 -f -L -C - \ -o /tmp/rustci_docker_cache "$url" echo "Loading images into docker" - loaded_images=$(docker load -i /tmp/rustci_docker_cache | sed 's/.* sha/sha/') + # docker load sometimes hangs in the CI, so time out after 10 minutes with TERM, + # KILL after 12 minutes + loaded_images=$(/usr/bin/timeout -k 720 600 docker load -i /tmp/rustci_docker_cache \ + | sed 's/.* sha/sha/') set -e echo "Downloaded containers:\n$loaded_images" fi