feat!: dag import - don't pin roots by default

Fixes: #9765

core/commands/dag/dag.go

@@ -176,13 +176,16 @@ var DagImportCmd = &cmds.Command{
 		Tagline: "Import the contents of .car files",
 		ShortDescription: `
 'ipfs dag import' imports all blocks present in supplied .car
-( Content Address aRchive ) files, recursively pinning any roots
-specified in the CAR file headers, unless --pin-roots is set to false.
+( Content Address aRchive ) files, optionally recursively pinning any
+roots specified in the CAR file headers if --pin-roots is set.
 
 Note:
   This command will import all blocks in the CAR file, not just those
-  reachable from the specified roots. However, these other blocks will
-  not be pinned and may be garbage collected later.
+  reachable from the specified roots. However, when using --pin-roots,
+  these other blocks will not be pinned and may be garbage collected
+  later. When not using --pin-roots, all blocks imported may be garbage
+  collected if no other pin operation is performed on them, or a root
+  that references them.
 
   The pinning of the roots happens after all car files are processed,
   permitting import of DAGs spanning multiple files.
@@ -200,7 +203,7 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/
 		cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(),
 	},
 	Options: []cmds.Option{
-		cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
+		cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing."),
 		cmds.BoolOption(silentOptionName, "No output."),
 		cmds.BoolOption(statsOptionName, "Output stats."),
 		cmdutils.AllowBigBlockOption,

docs/changelogs/v0.21.md

@@ -4,17 +4,20 @@
 
 ## v0.21.0
 
-- [Overview](#overview)
-- [🔦 Highlights](#-highlights)
-  - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping)
-  - [Gateway: `DeserializedResponses` config flag](#gateway-deserializedresponses-config-flag)
-  - [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client)
-  - [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages)
-  - [Gateway: subdomain redirects are now `text/html`](#gateway-subdomain-redirects-are-now-texthtml)
-  - [`ipfs dag stat` deduping statistics](#ipfs-dag-stat-deduping-statistics)
-  - [Accelerated DHT Client is no longer experimental](#--empty-repo-is-now-the-default)
-- [📝 Changelog](#-changelog)
-- [👨‍👩‍👧‍👦 Contributors](#-contributors)
+* [v0.21.0](#v0210)
+  * [Overview](#overview)
+  * [🔦 Highlights](#-highlights)
+    * [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping)
+    * [Gateway: `DeserializedResponses` config flag](#gateway-deserializedresponses-config-flag)
+    * [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client)
+    * [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages)
+    * [Gateway: subdomain redirects are now `text/html`](#gateway-subdomain-redirects-are-now-texthtml)
+    * [Gateway: support for CAR parameters of IPIP-402](#gateway-support-for-car-parameters-of-ipip-402)
+    * [`ipfs dag import` no longer pins by default](#ipfs-dag-import-no-longer-pins-by-default)
+    * [`ipfs dag stat` deduping statistics](#ipfs-dag-stat-deduping-statistics)
+    * [Accelerated DHT Client is no longer experimental](#accelerated-dht-client-is-no-longer-experimental)
+  * [📝 Changelog](#-changelog)
+  * [👨‍👩‍👧‍👦 Contributors](#-contributors)
 
 ### Overview
 
@@ -108,6 +111,10 @@ Rationale can be found in [kubo#9913](https://github.com/ipfs/kubo/pull/9913).
 
 The gateway now supports partial CAR export parameters as indicated in [IPIP-402](https://github.com/ipfs/specs/pull/402).
 
+#### `ipfs dag import` no longer pins by default
+
+With the gateway now supporting partial CAR exports, and incomplete DAG CARs becoming increasingly common. The pinning mode when using `ipfs dag import` has changed such that pinning the DAG or part of a DAG within an imported CAR is optional. To attempt to pin the DAG referenced by any roots in the CAR, opt in with the `--pin-roots` option.
+
 #### `ipfs dag stat` deduping statistics
 
 `ipfs dat stat` now accept multiple CIDs and will dump advanced statistics

test/sharness/t0054-dag-car-import-export.sh

@@ -41,7 +41,7 @@ do_import() {
       while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done &
       while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done &
 
-      ipfsi "$node" dag import "$@" 2>&1 && ipfsi "$node" repo verify &>/dev/null
+      ipfsi "$node" dag import --pin-roots "$@" 2>&1 && ipfsi "$node" repo verify &>/dev/null
       result=$?
 
       rm -f spin.gc &>/dev/null
@@ -117,7 +117,7 @@ EOE
   '
 
   test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" '
-    ipfsi 1 dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
+    ipfsi 1 dag import --stats --enc=json --pin-roots ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
       > naked_import_result_json_actual
   '
 
@@ -197,14 +197,14 @@ EOE
 head -3 multiroot_import_json_stats_expected > multiroot_import_json_expected
 
 test_expect_success "multiroot import works (--enc=json)" '
-  ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
+  ipfs dag import --enc=json --pin-roots ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
 '
 test_expect_success "multiroot import expected output" '
   test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual
 '
 
 test_expect_success "multiroot import works with --stats" '
-  ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
+  ipfs dag import --stats --enc=json --pin-roots ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
 '
 test_expect_success "multiroot import expected output" '
   test_cmp_sorted multiroot_import_json_stats_expected multiroot_import_json_actual
@@ -215,18 +215,18 @@ cat >pin_import_expected << EOE
 {"Stats":{"BlockCount":1198,"BlockBytesCount":468513}}
 EOE
 test_expect_success "pin-less import works" '
-  ipfs dag import --stats --enc=json --pin-roots=false \
+  ipfs dag import --stats --enc=json \
   ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \
   ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \
     > no-pin_import_actual
 '
-test_expect_success "expected no pins on --pin-roots=false" '
+test_expect_success "expected no pins on" '
   test_cmp pin_import_expected no-pin_import_actual
 '
 
 
 test_expect_success "naked root import works" '
-  ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
+  ipfs dag import --stats --enc=json --pin-roots ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
     > naked_root_import_json_actual
 '
 test_expect_success "naked root import expected output" '
@@ -253,7 +253,7 @@ cat > version_2_import_expected << EOE
 EOE
 
 test_expect_success "version 2 import" '
-  ipfs dag import --stats --enc=json \
+  ipfs dag import --stats --enc=json --pin-roots \
     ../t0054-dag-car-import-export-data/lotus_testnet_export_128_v2.car \
     ../t0054-dag-car-import-export-data/lotus_devnet_genesis_v2.car \
   > version_2_import_actual

test/sharness/t0109-gateway-web-_redirects.sh

@@ -14,7 +14,7 @@ test_launch_ipfs_daemon
 # Import test case
 # Run `ipfs cat /ipfs/$REDIRECTS_DIR_CID/_redirects` to see sample _redirects file
 test_expect_success "Add the _redirects file test directory" '
-  ipfs dag import ../t0109-gateway-web-_redirects-data/redirects.car
+  ipfs dag import --pin-roots ../t0109-gateway-web-_redirects-data/redirects.car
 '
 CAR_ROOT_CID=QmQyqMY5vUBSbSxyitJqthgwZunCQjDVtNd8ggVCxzuPQ4
 

test/sharness/t0113-gateway-symlink.sh

@@ -12,7 +12,7 @@ test_launch_ipfs_daemon
 # Import test case
 # See the static fixtures in ./t0113-gateway-symlink/
 test_expect_success "Add the test directory with symlinks" '
-  ipfs dag import ../t0113-gateway-symlink/testfiles.car
+  ipfs dag import --pin-roots ../t0113-gateway-symlink/testfiles.car
 '
 ROOT_DIR_CID=QmWvY6FaqFMS89YAQ9NAPjVP4WZKA1qbHbicc9HeSKQTgt # ./testfiles/
 

test/sharness/t0114-gateway-subdomains.sh

@@ -113,7 +113,7 @@ IPNS_ED25519_B58MH=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d
 IPNS_ED25519_B36CID=k51qzi5uqu5dk3v4rmjber23h16xnr23bsggmqqil9z2gduiis5se8dht36dam
 
 test_expect_success "Add the test fixtures" '
-  ipfs dag import ../t0114-gateway-subdomains/fixtures.car &&
+  ipfs dag import --pin-roots ../t0114-gateway-subdomains/fixtures.car &&
   ipfs routing put --allow-offline /ipns/${RSA_KEY} ../t0114-gateway-subdomains/${RSA_KEY}.ipns-record &&
   ipfs routing put --allow-offline /ipns/${ED25519_KEY} ../t0114-gateway-subdomains/${ED25519_KEY}.ipns-record
 '

test/sharness/t0115-gateway-dir-listing.sh

@@ -21,7 +21,7 @@ test_launch_ipfs_daemon_without_network
 # Import test case
 # See the static fixtures in ./t0115-gateway-dir-listing/
 test_expect_success "Add the test directory" '
-  ipfs dag import ../t0115-gateway-dir-listing/fixtures.car
+  ipfs dag import --pin-roots ../t0115-gateway-dir-listing/fixtures.car
 '
 DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir/
 FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt

test/sharness/t0116-gateway-cache.sh

@@ -35,7 +35,7 @@ TEST_IPNS_ID=k51qzi5uqu5dlxdsdu5fpuu7h69wu4ohp32iwm9pdt9nq3y5rpn3ln9j12zfhe
 # Import test case
 # See the static fixtures in ./t0116-gateway-cache/
 test_expect_success "Add the test directory" '
-  ipfs dag import ../t0116-gateway-cache/fixtures.car
+  ipfs dag import --pin-roots ../t0116-gateway-cache/fixtures.car
   ipfs routing put --allow-offline /ipns/${TEST_IPNS_ID} ../t0116-gateway-cache/${TEST_IPNS_ID}.ipns-record
 '
 

test/sharness/t0117-gateway-block.sh

@@ -10,7 +10,7 @@ test_launch_ipfs_daemon_without_network
 # Import test case
 # See the static fixtures in ./t0117-gateway-block/
 test_expect_success "Add the dir test directory" '
-  ipfs dag import ../t0117-gateway-block/fixtures.car
+  ipfs dag import --pin-roots ../t0117-gateway-block/fixtures.car
 '
 ROOT_DIR_CID=bafybeie72edlprgtlwwctzljf6gkn2wnlrddqjbkxo3jomh4n7omwblxly # ./
 FILE_CID=bafkreihhpc5y2pqvl5rbe5uuyhqjouybfs3rvlmisccgzue2kkt5zq6upq # ./dir/ascii.txt

test/sharness/t0122-gateway-tar.sh

@@ -13,7 +13,7 @@ INSIDE_ROOT_CID="bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y"
 # Import test case
 # See the static fixtures in ./t0122-gateway-tar/
 test_expect_success "Add the test directory" '
-  ipfs dag import ../t0122-gateway-tar/fixtures.car
+  ipfs dag import --pin-roots ../t0122-gateway-tar/fixtures.car
 '
 DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir
 FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt
@@ -63,9 +63,9 @@ test_expect_success "GET TAR with explicit ?filename= succeeds with modified Con
 "
 
 test_expect_success "Add CARs with relative paths to test with" '
-  ipfs dag import ../t0122-gateway-tar/outside-root.car > import_output &&
+  ipfs dag import --pin-roots ../t0122-gateway-tar/outside-root.car > import_output &&
   test_should_contain $OUTSIDE_ROOT_CID import_output &&
-  ipfs dag import ../t0122-gateway-tar/inside-root.car > import_output &&
+  ipfs dag import --pin-roots ../t0122-gateway-tar/inside-root.car > import_output &&
   test_should_contain $INSIDE_ROOT_CID import_output
 '
 

test/sharness/t0123-gateway-json-cbor.sh

@@ -10,7 +10,7 @@ test_launch_ipfs_daemon_without_network
 # Import test case
 # See the static fixtures in ./t0123-gateway-json-cbor/
 test_expect_success "Add the test directory" '
-  ipfs dag import ../t0123-gateway-json-cbor/fixtures.car
+  ipfs dag import --pin-roots ../t0123-gateway-json-cbor/fixtures.car
 '
 DIR_CID=bafybeiafyvqlazbbbtjnn6how5d6h6l6rxbqc4qgpbmteaiskjrffmyy4a # ./rootDir
 FILE_JSON_CID=bafkreibrppizs3g7axs2jdlnjua6vgpmltv7k72l7v7sa6mmht6mne3qqe # ./rootDir/ą/ę/t.json
@@ -155,11 +155,11 @@ DAG_JSON_TRAVERSAL_CID="baguqeeram5ujjqrwheyaty3w5gdsmoz6vittchvhk723jjqxk7hakxk
 DAG_PB_CID="bafybeiegxwlgmoh2cny7qlolykdf7aq7g6dlommarldrbm7c4hbckhfcke"
 
 test_expect_success "Add CARs for path traversal and DAG-PB representation tests" '
-  ipfs dag import ../t0123-gateway-json-cbor/dag-cbor-traversal.car > import_output &&
+  ipfs dag import --pin-roots ../t0123-gateway-json-cbor/dag-cbor-traversal.car > import_output &&
   test_should_contain $DAG_CBOR_TRAVERSAL_CID import_output &&
-  ipfs dag import ../t0123-gateway-json-cbor/dag-json-traversal.car > import_output &&
+  ipfs dag import --pin-roots ../t0123-gateway-json-cbor/dag-json-traversal.car > import_output &&
   test_should_contain $DAG_JSON_TRAVERSAL_CID import_output &&
-  ipfs dag import ../t0123-gateway-json-cbor/dag-pb.car > import_output &&
+  ipfs dag import --pin-roots ../t0123-gateway-json-cbor/dag-pb.car > import_output &&
   test_should_contain $DAG_PB_CID import_output
 '
 

test/sharness/t0124-gateway-ipns-record.sh

@@ -12,7 +12,7 @@ test_launch_ipfs_daemon
 IPNS_KEY=k51qzi5uqu5dh71qgwangrt6r0nd4094i88nsady6qgd1dhjcyfsaqmpp143ab
 FILE_CID=bafkreidfdrlkeq4m4xnxuyx6iae76fdm4wgl5d4xzsb77ixhyqwumhz244 # A file containing Hello IPFS
 test_expect_success "Add the test directory & IPNS records" '
-  ipfs dag import ../t0124-gateway-ipns-record/fixtures.car &&
+  ipfs dag import --pin-roots ../t0124-gateway-ipns-record/fixtures.car &&
   ipfs routing put /ipns/${IPNS_KEY} ../t0124-gateway-ipns-record/${IPNS_KEY}.ipns-record
 '
 

test/sharness/t0400-api-no-gateway.sh

@@ -13,7 +13,7 @@ test_init_ipfs
 # Import test case
 # See the static fixtures in ./t0400-api-no-gateway/
 test_expect_success "Add the test directory" '
-  ipfs dag import ../t0400-api-no-gateway/fixtures.car
+  ipfs dag import --pin-roots ../t0400-api-no-gateway/fixtures.car
 '
 HASH=QmNYERzV2LfD2kkfahtfv44ocHzEFK1sLBaE7zdcYT2GAZ # a file containing the string "testing"
 

testplans/bitswap/main.go

@@ -12,14 +12,14 @@ import (
 	"github.com/testground/sdk-go/runtime"
 	"github.com/testground/sdk-go/sync"
 
-	bitswap "github.com/ipfs/go-libipfs/bitswap"
-	bsnet "github.com/ipfs/go-libipfs/bitswap/network"
-	block "github.com/ipfs/go-libipfs/blocks"
 	"github.com/ipfs/go-cid"
 	datastore "github.com/ipfs/go-datastore"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	exchange "github.com/ipfs/go-ipfs-exchange-interface"
 	bstats "github.com/ipfs/go-ipfs-regression/bitswap"
+	bitswap "github.com/ipfs/go-libipfs/bitswap"
+	bsnet "github.com/ipfs/go-libipfs/bitswap/network"
+	block "github.com/ipfs/go-libipfs/blocks"
 	"github.com/libp2p/go-libp2p"
 	dht "github.com/libp2p/go-libp2p-kad-dht"
 	"github.com/libp2p/go-libp2p/core/host"