fetchipfs: support api/v0/get

pkgs/build-support/fetchipfs/builder.sh

@@ -7,15 +7,17 @@ source $stdenv/setup
 
 set -o noglob
 
+# we handle retries manually, since curl does also retry on http 500
+[ -z "$retryCount" ] && retryCount=2
+
 curl="curl            \
  --location           \
  --max-redirs 20      \
- --retry 2            \
+ --retry 0            \
  --disable-epsv       \
  --cookie-jar cookies \
  --insecure           \
  --speed-time 5       \
- -#                   \
  --fail               \
  $curlOpts            \
  $NIX_CURL_FLAGS"
@@ -28,9 +30,9 @@ finish() {
 
 ipfs_add() {
     if curl --retry 0 --head --silent "localhost:5001" > /dev/null; then
-        echo "=IPFS= add $ipfs"
+        echo "fetchipfs: add $ipfs"
         tar --owner=root --group=root -cWf "source.tar" $(echo *)
-        res=$(curl -# -F "file=@source.tar" "localhost:5001/api/v0/tar/add" | sed 's/.*"Hash":"\(.*\)".*/\1/')
+        res=$(curl -X POST -# -F "file=@source.tar" "localhost:5001/api/v0/tar/add" | sed 's/.*"Hash":"\(.*\)".*/\1/')
         if [ $ipfs != $res ]; then
             echo "\`ipfs tar add' results in $res when $ipfs is expected"
             exit 1
@@ -44,27 +46,63 @@ echo
 mkdir download
 cd download
 
-if curl --retry 0 --head --silent "localhost:5001" > /dev/null; then
+if curl -X POST --retry 0 --head --silent "localhost:5001" > /dev/null; then
     curlexit=18;
-    echo "=IPFS= get $ipfs"
-    # if we get error code 18, resume partial download
-    while [ $curlexit -eq 18 ]; do
+    echo "fetchipfs: get $ipfs"
+    api_path_tar="tar/cat?"
+    api_path_get="get?archive=true&compress=false&"
+    api_path="$api_path_tar"
+
+    retry_step=0
+    for (( loop_step = 1; ; loop_step++ )); do
+        if (( $loop_step % 50 == 0 )); then
+            # debug infinite loop: show progress every 50 steps
+            echo "fetchipfs: loop step $loop_step"
+        fi
+
         # keep this inside an if statement, since on failure it doesn't abort the script
-        if $curl -C - "http://localhost:5001/api/v0/tar/cat?arg=$ipfs" --output "$ipfs.tar"; then
+        if http_code=$($curl -s -w "%{http_code}" -X POST -C - "http://localhost:5001/api/v0/${api_path}arg=$ipfs" --output "$ipfs.tar")
+        then
+            sleep 0.5 # WORKAROUND tar: time stamp is 0.2 s in the future https://github.com/ipfs/go-ipfs/issues/8406
             unpackFile "$ipfs.tar"
             rm "$ipfs.tar"
-            set +o noglob
-            mv $(echo *) "$out"
+            if [[ "$api_path" == "$api_path_tar" ]]; then
+                mkdir -p "$out"
+                set +o noglob # enable glob
+                set -o dotglob # also glob hidden files
+                mv * "$out"
+            else
+                mv "$ipfs" "$out"
+            fi
             finish
         else
-            curlexit=$?;
+            curlexit=$?
+
+            if [[ $curlexit == 18 ]]; then
+                # continue partial download (curl -C -)
+                continue
+            fi
+
+            if [[ "$http_code" == 500 && "$api_path" == "$api_path_tar" ]]; then
+                # change api method
+                api_path="$api_path_get"
+                continue
+            fi
+
+            retry_step=$(($retry_step + 1))
+            if (( $retry_step > $retryCount )); then
+                echo "fetchipfs: retry end"
+                break
+            fi
+
+            echo "fetchipfs: retry $retry_step"
         fi
     done
 fi
 
 if test -n "$url"; then
     curlexit=18;
-    echo "Downloading $url"
+    echo "fetchipfs: download $url"
     while [ $curlexit -eq 18 ]; do
         # keep this inside an if statement, since on failure it doesn't abort the script
         if $curl "$url" -O; then