From a9a51306364e450a9167dcb3c7a746c95a227332 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 9 Jun 2023 15:43:31 +1000 Subject: [PATCH] feat!: dag import - don't pin roots by default Fixes: https://github.com/ipfs/kubo/issues/9765 --- core/commands/dag/dag.go | 13 +++++---- docs/changelogs/v0.21.md | 29 ++++++++++++-------- test/sharness/t0054-dag-car-import-export.sh | 10 +++---- test/sharness/t0122-gateway-tar.sh | 6 ++-- testplans/bitswap/main.go | 6 ++-- 5 files changed, 37 insertions(+), 27 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 7d21eb0710f5..783b36ac1512 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -176,13 +176,16 @@ var DagImportCmd = &cmds.Command{ Tagline: "Import the contents of .car files", ShortDescription: ` 'ipfs dag import' imports all blocks present in supplied .car -( Content Address aRchive ) files, recursively pinning any roots -specified in the CAR file headers, unless --pin-roots is set to false. +( Content Address aRchive ) files, optionally recursively pinning any +roots specified in the CAR file headers if --pin-roots is set. Note: This command will import all blocks in the CAR file, not just those - reachable from the specified roots. However, these other blocks will - not be pinned and may be garbage collected later. + reachable from the specified roots. However, when using --pin-roots, + these other blocks will not be pinned and may be garbage collected + later. When not using --pin-roots, all blocks imported may be garbage + collected if no other pin operation is performed on them, or a root + that references them. The pinning of the roots happens after all car files are processed, permitting import of DAGs spanning multiple files. @@ -200,7 +203,7 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/ cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(), }, Options: []cmds.Option{ - cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), + cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing."), cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(statsOptionName, "Output stats."), cmdutils.AllowBigBlockOption, diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md index 03420174814b..409e138baacf 100644 --- a/docs/changelogs/v0.21.md +++ b/docs/changelogs/v0.21.md @@ -4,17 +4,20 @@ ## v0.21.0 -- [Overview](#overview) -- [πŸ”¦ Highlights](#-highlights) - - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping) - - [Gateway: `DeserializedResponses` config flag](#gateway-deserializedresponses-config-flag) - - [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client) - - [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages) - - [Gateway: subdomain redirects are now `text/html`](#gateway-subdomain-redirects-are-now-texthtml) - - [`ipfs dag stat` deduping statistics](#ipfs-dag-stat-deduping-statistics) - - [Accelerated DHT Client is no longer experimental](#--empty-repo-is-now-the-default) -- [πŸ“ Changelog](#-changelog) -- [πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦ Contributors](#-contributors) +* [v0.21.0](#v0210) + * [Overview](#overview) + * [πŸ”¦ Highlights](#-highlights) + * [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping) + * [Gateway: `DeserializedResponses` config flag](#gateway-deserializedresponses-config-flag) + * [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client) + * [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages) + * [Gateway: subdomain redirects are now `text/html`](#gateway-subdomain-redirects-are-now-texthtml) + * [Gateway: support for CAR parameters of IPIP-402](#gateway-support-for-car-parameters-of-ipip-402) + * [`ipfs dag import` no longer pins by default](#ipfs-dag-import-no-longer-pins-by-default) + * [`ipfs dag stat` deduping statistics](#ipfs-dag-stat-deduping-statistics) + * [Accelerated DHT Client is no longer experimental](#accelerated-dht-client-is-no-longer-experimental) + * [πŸ“ Changelog](#-changelog) + * [πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦ Contributors](#-contributors) ### Overview @@ -108,6 +111,10 @@ Rationale can be found in [kubo#9913](https://github.com/ipfs/kubo/pull/9913). The gateway now supports partial CAR export parameters as indicated in [IPIP-402](https://github.com/ipfs/specs/pull/402). +#### `ipfs dag import` no longer pins by default + +With the gateway now supporting partial CAR exports, and incomplete DAG CARs becoming increasingly common. The pinning mode when using `ipfs dag import` has changed such that pinning the DAG or part of a DAG within an imported CAR is optional. To attempt to pin the DAG referenced by any roots in the CAR, opt in with the `--pin-roots` option. + #### `ipfs dag stat` deduping statistics `ipfs dat stat` now accept multiple CIDs and will dump advanced statistics diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index fe6f11476912..0ea3a53395f4 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -117,7 +117,7 @@ EOE ' test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" ' - ipfsi 1 dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ + ipfsi 1 dag import --stats --enc=json --pin-roots ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ > naked_import_result_json_actual ' @@ -197,7 +197,7 @@ EOE head -3 multiroot_import_json_stats_expected > multiroot_import_json_expected test_expect_success "multiroot import works (--enc=json)" ' - ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual + ipfs dag import --enc=json --pin-roots ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual ' test_expect_success "multiroot import expected output" ' test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual @@ -215,12 +215,12 @@ cat >pin_import_expected << EOE {"Stats":{"BlockCount":1198,"BlockBytesCount":468513}} EOE test_expect_success "pin-less import works" ' - ipfs dag import --stats --enc=json --pin-roots=false \ + ipfs dag import --stats --enc=json \ ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \ ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \ > no-pin_import_actual ' -test_expect_success "expected no pins on --pin-roots=false" ' +test_expect_success "expected no pins on" ' test_cmp pin_import_expected no-pin_import_actual ' @@ -253,7 +253,7 @@ cat > version_2_import_expected << EOE EOE test_expect_success "version 2 import" ' - ipfs dag import --stats --enc=json \ + ipfs dag import --stats --enc=json --pin-roots \ ../t0054-dag-car-import-export-data/lotus_testnet_export_128_v2.car \ ../t0054-dag-car-import-export-data/lotus_devnet_genesis_v2.car \ > version_2_import_actual diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh index 20cc1bf4cdc9..435623547a30 100755 --- a/test/sharness/t0122-gateway-tar.sh +++ b/test/sharness/t0122-gateway-tar.sh @@ -13,7 +13,7 @@ INSIDE_ROOT_CID="bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y" # Import test case # See the static fixtures in ./t0122-gateway-tar/ test_expect_success "Add the test directory" ' - ipfs dag import ../t0122-gateway-tar/fixtures.car + ipfs dag import --pin-roots ../t0122-gateway-tar/fixtures.car ' DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/Δ…/Δ™/file-ΕΊΕ‚.txt @@ -63,9 +63,9 @@ test_expect_success "GET TAR with explicit ?filename= succeeds with modified Con " test_expect_success "Add CARs with relative paths to test with" ' - ipfs dag import ../t0122-gateway-tar/outside-root.car > import_output && + ipfs dag import --pin-roots ../t0122-gateway-tar/outside-root.car > import_output && test_should_contain $OUTSIDE_ROOT_CID import_output && - ipfs dag import ../t0122-gateway-tar/inside-root.car > import_output && + ipfs dag import --pin-roots ../t0122-gateway-tar/inside-root.car > import_output && test_should_contain $INSIDE_ROOT_CID import_output ' diff --git a/testplans/bitswap/main.go b/testplans/bitswap/main.go index aa98d4f31aed..0af2ad95c6b3 100644 --- a/testplans/bitswap/main.go +++ b/testplans/bitswap/main.go @@ -12,14 +12,14 @@ import ( "github.com/testground/sdk-go/runtime" "github.com/testground/sdk-go/sync" - bitswap "github.com/ipfs/go-libipfs/bitswap" - bsnet "github.com/ipfs/go-libipfs/bitswap/network" - block "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/go-cid" datastore "github.com/ipfs/go-datastore" blockstore "github.com/ipfs/go-ipfs-blockstore" exchange "github.com/ipfs/go-ipfs-exchange-interface" bstats "github.com/ipfs/go-ipfs-regression/bitswap" + bitswap "github.com/ipfs/go-libipfs/bitswap" + bsnet "github.com/ipfs/go-libipfs/bitswap/network" + block "github.com/ipfs/go-libipfs/blocks" "github.com/libp2p/go-libp2p" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p/core/host"