From b30f46b7ac0df81d81077fe71435c7015b4a2d2e Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 9 Jun 2023 15:43:31 +1000 Subject: [PATCH 1/3] feat!: dag import - don't pin roots by default Fixes: https://github.com/ipfs/kubo/issues/9765 --- core/commands/dag/dag.go | 13 +++++---- docs/changelogs/v0.21.md | 29 ++++++++++++------- test/sharness/t0054-dag-car-import-export.sh | 16 +++++----- test/sharness/t0109-gateway-web-_redirects.sh | 2 +- test/sharness/t0113-gateway-symlink.sh | 2 +- test/sharness/t0114-gateway-subdomains.sh | 2 +- test/sharness/t0115-gateway-dir-listing.sh | 2 +- test/sharness/t0116-gateway-cache.sh | 2 +- test/sharness/t0117-gateway-block.sh | 2 +- test/sharness/t0122-gateway-tar.sh | 6 ++-- test/sharness/t0123-gateway-json-cbor.sh | 8 ++--- test/sharness/t0124-gateway-ipns-record.sh | 2 +- test/sharness/t0400-api-no-gateway.sh | 2 +- testplans/bitswap/main.go | 6 ++-- 14 files changed, 52 insertions(+), 42 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 7d21eb0710f..783b36ac151 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -176,13 +176,16 @@ var DagImportCmd = &cmds.Command{ Tagline: "Import the contents of .car files", ShortDescription: ` 'ipfs dag import' imports all blocks present in supplied .car -( Content Address aRchive ) files, recursively pinning any roots -specified in the CAR file headers, unless --pin-roots is set to false. +( Content Address aRchive ) files, optionally recursively pinning any +roots specified in the CAR file headers if --pin-roots is set. Note: This command will import all blocks in the CAR file, not just those - reachable from the specified roots. However, these other blocks will - not be pinned and may be garbage collected later. + reachable from the specified roots. However, when using --pin-roots, + these other blocks will not be pinned and may be garbage collected + later. When not using --pin-roots, all blocks imported may be garbage + collected if no other pin operation is performed on them, or a root + that references them. The pinning of the roots happens after all car files are processed, permitting import of DAGs spanning multiple files. @@ -200,7 +203,7 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/ cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(), }, Options: []cmds.Option{ - cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), + cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing."), cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(statsOptionName, "Output stats."), cmdutils.AllowBigBlockOption, diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md index 03420174814..409e138baac 100644 --- a/docs/changelogs/v0.21.md +++ b/docs/changelogs/v0.21.md @@ -4,17 +4,20 @@ ## v0.21.0 -- [Overview](#overview) -- [πŸ”¦ Highlights](#-highlights) - - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping) - - [Gateway: `DeserializedResponses` config flag](#gateway-deserializedresponses-config-flag) - - [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client) - - [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages) - - [Gateway: subdomain redirects are now `text/html`](#gateway-subdomain-redirects-are-now-texthtml) - - [`ipfs dag stat` deduping statistics](#ipfs-dag-stat-deduping-statistics) - - [Accelerated DHT Client is no longer experimental](#--empty-repo-is-now-the-default) -- [πŸ“ Changelog](#-changelog) -- [πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦ Contributors](#-contributors) +* [v0.21.0](#v0210) + * [Overview](#overview) + * [πŸ”¦ Highlights](#-highlights) + * [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping) + * [Gateway: `DeserializedResponses` config flag](#gateway-deserializedresponses-config-flag) + * [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client) + * [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages) + * [Gateway: subdomain redirects are now `text/html`](#gateway-subdomain-redirects-are-now-texthtml) + * [Gateway: support for CAR parameters of IPIP-402](#gateway-support-for-car-parameters-of-ipip-402) + * [`ipfs dag import` no longer pins by default](#ipfs-dag-import-no-longer-pins-by-default) + * [`ipfs dag stat` deduping statistics](#ipfs-dag-stat-deduping-statistics) + * [Accelerated DHT Client is no longer experimental](#accelerated-dht-client-is-no-longer-experimental) + * [πŸ“ Changelog](#-changelog) + * [πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦ Contributors](#-contributors) ### Overview @@ -108,6 +111,10 @@ Rationale can be found in [kubo#9913](https://github.com/ipfs/kubo/pull/9913). The gateway now supports partial CAR export parameters as indicated in [IPIP-402](https://github.com/ipfs/specs/pull/402). +#### `ipfs dag import` no longer pins by default + +With the gateway now supporting partial CAR exports, and incomplete DAG CARs becoming increasingly common. The pinning mode when using `ipfs dag import` has changed such that pinning the DAG or part of a DAG within an imported CAR is optional. To attempt to pin the DAG referenced by any roots in the CAR, opt in with the `--pin-roots` option. + #### `ipfs dag stat` deduping statistics `ipfs dat stat` now accept multiple CIDs and will dump advanced statistics diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index fe6f1147691..f67cf13e2d2 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -41,7 +41,7 @@ do_import() { while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done & while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done & - ipfsi "$node" dag import "$@" 2>&1 && ipfsi "$node" repo verify &>/dev/null + ipfsi "$node" dag import --pin-roots "$@" 2>&1 && ipfsi "$node" repo verify &>/dev/null result=$? rm -f spin.gc &>/dev/null @@ -117,7 +117,7 @@ EOE ' test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" ' - ipfsi 1 dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ + ipfsi 1 dag import --stats --enc=json --pin-roots ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ > naked_import_result_json_actual ' @@ -197,14 +197,14 @@ EOE head -3 multiroot_import_json_stats_expected > multiroot_import_json_expected test_expect_success "multiroot import works (--enc=json)" ' - ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual + ipfs dag import --enc=json --pin-roots ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual ' test_expect_success "multiroot import expected output" ' test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual ' test_expect_success "multiroot import works with --stats" ' - ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual + ipfs dag import --stats --enc=json --pin-roots ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual ' test_expect_success "multiroot import expected output" ' test_cmp_sorted multiroot_import_json_stats_expected multiroot_import_json_actual @@ -215,18 +215,18 @@ cat >pin_import_expected << EOE {"Stats":{"BlockCount":1198,"BlockBytesCount":468513}} EOE test_expect_success "pin-less import works" ' - ipfs dag import --stats --enc=json --pin-roots=false \ + ipfs dag import --stats --enc=json \ ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \ ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \ > no-pin_import_actual ' -test_expect_success "expected no pins on --pin-roots=false" ' +test_expect_success "expected no pins on" ' test_cmp pin_import_expected no-pin_import_actual ' test_expect_success "naked root import works" ' - ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ + ipfs dag import --stats --enc=json --pin-roots ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ > naked_root_import_json_actual ' test_expect_success "naked root import expected output" ' @@ -253,7 +253,7 @@ cat > version_2_import_expected << EOE EOE test_expect_success "version 2 import" ' - ipfs dag import --stats --enc=json \ + ipfs dag import --stats --enc=json --pin-roots \ ../t0054-dag-car-import-export-data/lotus_testnet_export_128_v2.car \ ../t0054-dag-car-import-export-data/lotus_devnet_genesis_v2.car \ > version_2_import_actual diff --git a/test/sharness/t0109-gateway-web-_redirects.sh b/test/sharness/t0109-gateway-web-_redirects.sh index 9ebfb911e35..0bc2a23b600 100755 --- a/test/sharness/t0109-gateway-web-_redirects.sh +++ b/test/sharness/t0109-gateway-web-_redirects.sh @@ -14,7 +14,7 @@ test_launch_ipfs_daemon # Import test case # Run `ipfs cat /ipfs/$REDIRECTS_DIR_CID/_redirects` to see sample _redirects file test_expect_success "Add the _redirects file test directory" ' - ipfs dag import ../t0109-gateway-web-_redirects-data/redirects.car + ipfs dag import --pin-roots ../t0109-gateway-web-_redirects-data/redirects.car ' CAR_ROOT_CID=QmQyqMY5vUBSbSxyitJqthgwZunCQjDVtNd8ggVCxzuPQ4 diff --git a/test/sharness/t0113-gateway-symlink.sh b/test/sharness/t0113-gateway-symlink.sh index 29e5b960de3..2cb4f319537 100755 --- a/test/sharness/t0113-gateway-symlink.sh +++ b/test/sharness/t0113-gateway-symlink.sh @@ -12,7 +12,7 @@ test_launch_ipfs_daemon # Import test case # See the static fixtures in ./t0113-gateway-symlink/ test_expect_success "Add the test directory with symlinks" ' - ipfs dag import ../t0113-gateway-symlink/testfiles.car + ipfs dag import --pin-roots ../t0113-gateway-symlink/testfiles.car ' ROOT_DIR_CID=QmWvY6FaqFMS89YAQ9NAPjVP4WZKA1qbHbicc9HeSKQTgt # ./testfiles/ diff --git a/test/sharness/t0114-gateway-subdomains.sh b/test/sharness/t0114-gateway-subdomains.sh index 1d19a657069..8be102c4f41 100755 --- a/test/sharness/t0114-gateway-subdomains.sh +++ b/test/sharness/t0114-gateway-subdomains.sh @@ -113,7 +113,7 @@ IPNS_ED25519_B58MH=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d IPNS_ED25519_B36CID=k51qzi5uqu5dk3v4rmjber23h16xnr23bsggmqqil9z2gduiis5se8dht36dam test_expect_success "Add the test fixtures" ' - ipfs dag import ../t0114-gateway-subdomains/fixtures.car && + ipfs dag import --pin-roots ../t0114-gateway-subdomains/fixtures.car && ipfs routing put --allow-offline /ipns/${RSA_KEY} ../t0114-gateway-subdomains/${RSA_KEY}.ipns-record && ipfs routing put --allow-offline /ipns/${ED25519_KEY} ../t0114-gateway-subdomains/${ED25519_KEY}.ipns-record ' diff --git a/test/sharness/t0115-gateway-dir-listing.sh b/test/sharness/t0115-gateway-dir-listing.sh index cf95bf4b05e..1ce0861b2eb 100755 --- a/test/sharness/t0115-gateway-dir-listing.sh +++ b/test/sharness/t0115-gateway-dir-listing.sh @@ -21,7 +21,7 @@ test_launch_ipfs_daemon_without_network # Import test case # See the static fixtures in ./t0115-gateway-dir-listing/ test_expect_success "Add the test directory" ' - ipfs dag import ../t0115-gateway-dir-listing/fixtures.car + ipfs dag import --pin-roots ../t0115-gateway-dir-listing/fixtures.car ' DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir/ FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/Δ…/Δ™/file-ΕΊΕ‚.txt diff --git a/test/sharness/t0116-gateway-cache.sh b/test/sharness/t0116-gateway-cache.sh index b2e6b3af38b..2c0f2f83301 100755 --- a/test/sharness/t0116-gateway-cache.sh +++ b/test/sharness/t0116-gateway-cache.sh @@ -35,7 +35,7 @@ TEST_IPNS_ID=k51qzi5uqu5dlxdsdu5fpuu7h69wu4ohp32iwm9pdt9nq3y5rpn3ln9j12zfhe # Import test case # See the static fixtures in ./t0116-gateway-cache/ test_expect_success "Add the test directory" ' - ipfs dag import ../t0116-gateway-cache/fixtures.car + ipfs dag import --pin-roots ../t0116-gateway-cache/fixtures.car ipfs routing put --allow-offline /ipns/${TEST_IPNS_ID} ../t0116-gateway-cache/${TEST_IPNS_ID}.ipns-record ' diff --git a/test/sharness/t0117-gateway-block.sh b/test/sharness/t0117-gateway-block.sh index b21425bc42c..a4a661bd194 100755 --- a/test/sharness/t0117-gateway-block.sh +++ b/test/sharness/t0117-gateway-block.sh @@ -10,7 +10,7 @@ test_launch_ipfs_daemon_without_network # Import test case # See the static fixtures in ./t0117-gateway-block/ test_expect_success "Add the dir test directory" ' - ipfs dag import ../t0117-gateway-block/fixtures.car + ipfs dag import --pin-roots ../t0117-gateway-block/fixtures.car ' ROOT_DIR_CID=bafybeie72edlprgtlwwctzljf6gkn2wnlrddqjbkxo3jomh4n7omwblxly # ./ FILE_CID=bafkreihhpc5y2pqvl5rbe5uuyhqjouybfs3rvlmisccgzue2kkt5zq6upq # ./dir/ascii.txt diff --git a/test/sharness/t0122-gateway-tar.sh b/test/sharness/t0122-gateway-tar.sh index 20cc1bf4cdc..435623547a3 100755 --- a/test/sharness/t0122-gateway-tar.sh +++ b/test/sharness/t0122-gateway-tar.sh @@ -13,7 +13,7 @@ INSIDE_ROOT_CID="bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y" # Import test case # See the static fixtures in ./t0122-gateway-tar/ test_expect_success "Add the test directory" ' - ipfs dag import ../t0122-gateway-tar/fixtures.car + ipfs dag import --pin-roots ../t0122-gateway-tar/fixtures.car ' DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/Δ…/Δ™/file-ΕΊΕ‚.txt @@ -63,9 +63,9 @@ test_expect_success "GET TAR with explicit ?filename= succeeds with modified Con " test_expect_success "Add CARs with relative paths to test with" ' - ipfs dag import ../t0122-gateway-tar/outside-root.car > import_output && + ipfs dag import --pin-roots ../t0122-gateway-tar/outside-root.car > import_output && test_should_contain $OUTSIDE_ROOT_CID import_output && - ipfs dag import ../t0122-gateway-tar/inside-root.car > import_output && + ipfs dag import --pin-roots ../t0122-gateway-tar/inside-root.car > import_output && test_should_contain $INSIDE_ROOT_CID import_output ' diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index b4b0446ffb8..323d23d584f 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -10,7 +10,7 @@ test_launch_ipfs_daemon_without_network # Import test case # See the static fixtures in ./t0123-gateway-json-cbor/ test_expect_success "Add the test directory" ' - ipfs dag import ../t0123-gateway-json-cbor/fixtures.car + ipfs dag import --pin-roots ../t0123-gateway-json-cbor/fixtures.car ' DIR_CID=bafybeiafyvqlazbbbtjnn6how5d6h6l6rxbqc4qgpbmteaiskjrffmyy4a # ./rootDir FILE_JSON_CID=bafkreibrppizs3g7axs2jdlnjua6vgpmltv7k72l7v7sa6mmht6mne3qqe # ./rootDir/Δ…/Δ™/t.json @@ -155,11 +155,11 @@ DAG_JSON_TRAVERSAL_CID="baguqeeram5ujjqrwheyaty3w5gdsmoz6vittchvhk723jjqxk7hakxk DAG_PB_CID="bafybeiegxwlgmoh2cny7qlolykdf7aq7g6dlommarldrbm7c4hbckhfcke" test_expect_success "Add CARs for path traversal and DAG-PB representation tests" ' - ipfs dag import ../t0123-gateway-json-cbor/dag-cbor-traversal.car > import_output && + ipfs dag import --pin-roots ../t0123-gateway-json-cbor/dag-cbor-traversal.car > import_output && test_should_contain $DAG_CBOR_TRAVERSAL_CID import_output && - ipfs dag import ../t0123-gateway-json-cbor/dag-json-traversal.car > import_output && + ipfs dag import --pin-roots ../t0123-gateway-json-cbor/dag-json-traversal.car > import_output && test_should_contain $DAG_JSON_TRAVERSAL_CID import_output && - ipfs dag import ../t0123-gateway-json-cbor/dag-pb.car > import_output && + ipfs dag import --pin-roots ../t0123-gateway-json-cbor/dag-pb.car > import_output && test_should_contain $DAG_PB_CID import_output ' diff --git a/test/sharness/t0124-gateway-ipns-record.sh b/test/sharness/t0124-gateway-ipns-record.sh index 1d05b56985a..605252c0f5e 100755 --- a/test/sharness/t0124-gateway-ipns-record.sh +++ b/test/sharness/t0124-gateway-ipns-record.sh @@ -12,7 +12,7 @@ test_launch_ipfs_daemon IPNS_KEY=k51qzi5uqu5dh71qgwangrt6r0nd4094i88nsady6qgd1dhjcyfsaqmpp143ab FILE_CID=bafkreidfdrlkeq4m4xnxuyx6iae76fdm4wgl5d4xzsb77ixhyqwumhz244 # A file containing Hello IPFS test_expect_success "Add the test directory & IPNS records" ' - ipfs dag import ../t0124-gateway-ipns-record/fixtures.car && + ipfs dag import --pin-roots ../t0124-gateway-ipns-record/fixtures.car && ipfs routing put /ipns/${IPNS_KEY} ../t0124-gateway-ipns-record/${IPNS_KEY}.ipns-record ' diff --git a/test/sharness/t0400-api-no-gateway.sh b/test/sharness/t0400-api-no-gateway.sh index d0daeece3da..7518db42766 100755 --- a/test/sharness/t0400-api-no-gateway.sh +++ b/test/sharness/t0400-api-no-gateway.sh @@ -13,7 +13,7 @@ test_init_ipfs # Import test case # See the static fixtures in ./t0400-api-no-gateway/ test_expect_success "Add the test directory" ' - ipfs dag import ../t0400-api-no-gateway/fixtures.car + ipfs dag import --pin-roots ../t0400-api-no-gateway/fixtures.car ' HASH=QmNYERzV2LfD2kkfahtfv44ocHzEFK1sLBaE7zdcYT2GAZ # a file containing the string "testing" diff --git a/testplans/bitswap/main.go b/testplans/bitswap/main.go index aa98d4f31ae..0af2ad95c6b 100644 --- a/testplans/bitswap/main.go +++ b/testplans/bitswap/main.go @@ -12,14 +12,14 @@ import ( "github.com/testground/sdk-go/runtime" "github.com/testground/sdk-go/sync" - bitswap "github.com/ipfs/go-libipfs/bitswap" - bsnet "github.com/ipfs/go-libipfs/bitswap/network" - block "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/go-cid" datastore "github.com/ipfs/go-datastore" blockstore "github.com/ipfs/go-ipfs-blockstore" exchange "github.com/ipfs/go-ipfs-exchange-interface" bstats "github.com/ipfs/go-ipfs-regression/bitswap" + bitswap "github.com/ipfs/go-libipfs/bitswap" + bsnet "github.com/ipfs/go-libipfs/bitswap/network" + block "github.com/ipfs/go-libipfs/blocks" "github.com/libp2p/go-libp2p" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p/core/host" From 20b7e18c085b3f7b2d3169b4e553ca2a666fec69 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 14 Jun 2023 22:01:43 +0200 Subject: [PATCH 2/3] test(ipip-402): dag import this adds basic regression test that guards behavior around partial cars with or without pinning --- .../README.md | 4 ++++ .../partial-dag-scope-entity.car | Bin 0 -> 1711 bytes test/sharness/t0054-dag-car-import-export.sh | 22 ++++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 test/sharness/t0054-dag-car-import-export-data/partial-dag-scope-entity.car diff --git a/test/sharness/t0054-dag-car-import-export-data/README.md b/test/sharness/t0054-dag-car-import-export-data/README.md index 033f837a092..786f9ade0e2 100644 --- a/test/sharness/t0054-dag-car-import-export-data/README.md +++ b/test/sharness/t0054-dag-car-import-export-data/README.md @@ -26,3 +26,7 @@ - lotus_devnet_genesis_v2.car - generated with `car index lotus_testnet_export_128.car > lotus_testnet_export_128_v2.car` - install `go-car` CLI from https://github.com/ipld/go-car + +- partial-dag-scope-entity.car + - unixfs directory entity exported from gateway via `?format=car&dag-scope=entity` ([IPIP-402](https://github.com/ipfs/specs/pull/402)) + - CAR roots includes directory CID, but only the root block is included in the CAR, making the DAG incomplete diff --git a/test/sharness/t0054-dag-car-import-export-data/partial-dag-scope-entity.car b/test/sharness/t0054-dag-car-import-export-data/partial-dag-scope-entity.car new file mode 100644 index 0000000000000000000000000000000000000000..b149a18df7ceb2921d2c53dd40b919e8ce25cdd1 GIT binary patch literal 1711 zcmai#dpJ~i7{?7(86*5m+eu8rw2PQQlT>vBb@4xeT-|z4Hd*APwpN}Zz zasuDhF?%&f53m_;rG~=4INA}rju?Y9X{4%*!|E}$tKg@X|Hj|&dO)0jC~;Jd(Z(46 z|1nrhw*&0>*#*V=+N6fI__`2bNd(g(+ub&OnzXT{u6@*0U>5_dU`(i#qe8AgqLj-J zuG@k+8uIWIm>6oBLKPYG9C-!%pBgZ~Qs(xej2i^JhG>Na%c_3-l$}!%nw;~WB6Hm5 zX~^@f$Pp&+)+tGqjIW*<+~Hpd*H?vBPB@;_uN6(z+4ACU=#=Ar~rosUBE&e zkQ)_i(*J7E_kBbDmh*NXu0$-riYzX;FF*Qtx=)U)1X#o=l-_KVYKOBZBXmEhLw-DgUO^q0iq!7sD0uKxH`tvuB$Vr zAet{J!WkF}<)jzKjVUi&pMMG*x6!hg0bcO6wFB!M zB#lq4t43krJ*V>4U>hE=5Wxw$SK_+O)3{MjebIno7x|TW2RN|~ExteeXTr1EerE5@ zlh2-q?zK%>W!|H0mi^1}=V{2~(x8hW3JfN*wKu0klqTxvha*!JK(WpuP>391biQOJw zX4zMI9BRnCARbJCUPKvs&uHr%V~SaK!6X#J30NwH>RJJC*IoaG9V2;@Q(o^|mz^bN zGgf*HIY!>9Og5>lvQewC@tQS&fntPGE>sB!GCrUZ9yPP7h?f3J*9A#$l8`oK_1Aj# z^U@DcW0TM!v7YS~w%Gufq6!hto$ET6l9(Azagn(vi)Pu(ys@+pO5A>V@};^ZcT=~p zmV4;&JPk4qV(q|G24koQ!3o|5ob2~bA9(6y_B|P#s4XZB>k7+yJo~2X^IkzD_Q&y~ zsZK?Cxe*1Oy8w{k1WIOro4c!qJl_H!;|IzXy^5L-5i?)MBWAvhsk<#(tY@wrim#b! za=v*W>x1uS8knR|$~Pba0&31Pru<*O;+@REzb3Y1;pTy3OY}}C_l>cl z``oSp`bHHhh$qcd5>UZ5BWfeXo`Gf-jsA$Esmx+i%p;c9lG>oW3=8owj`2j$bhZ{4 zD-jW*M92k$PSm?o8#!dKa{OnGSGJ>vC=@iqZ%_0lKhV%l9kwhD#yk2IazIzgj5qeAe;s{199=ZEcvpi_jcxAaa5H8R8<^lIfyiLR zx1efIqLy>F`FPp&+q1$}gg#7Jey1(1UD7||XgoTaneG|h7;$R?n5z*fnK=kIPK8Q^ P partial_nopin_import_expected << EOE +{"Stats":{"BlockCount":1,"BlockBytesCount":1618}} +EOE +test_expect_success "'ipfs dag import' without pinning works fine with incomplete DAG (unixfs dir exported as dag-scope=entity from IPIP-402)" ' + ipfs dag import --stats --enc=json --pin-roots=false ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car >partial_nopin_import_out 2>&1 && + test_cmp partial_nopin_import_expected partial_nopin_import_out +' +test_expect_success "'ipfs dag import' with no params in CLI mode produces exit code 0 (unixfs dir exported as dag-scope=entity from IPIP-402)" ' + test_expect_code 0 ipfs dag import ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car +' + +test_expect_success "'ipfs dag import' with pinning errors due to incomplete DAG (unixfs dir exported as dag-scope=entity from IPIP-402)" ' + ipfs dag import --stats --enc=json --pin-roots=true ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car >partial_pin_import_out 2>&1 && + test_should_contain "\"PinErrorMsg\":\"block was not found locally" partial_pin_import_out +' + +test_expect_success "'ipfs dag import' pin error in default CLI mode produces exit code 1 (unixfs dir exported as dag-scope=entity from IPIP-402)" ' + test_expect_code 1 ipfs dag import --pin-roots ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car >partial_pin_import_out 2>&1 && + test_should_contain "Error: pinning root \"QmPDC11yLAbVw3dX5jMeEuSdk4BiVjSd9X87zaYRdVjzW3\" FAILED: block was not found locally" partial_pin_import_out +' + test_done From e2ac0d4446125d111889c1c62c1209a6c22c8c83 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 14 Jun 2023 22:36:22 +0200 Subject: [PATCH 3/3] docs(ipip-402): ipip and dag import changelog --- docs/changelogs/v0.21.md | 56 ++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md index b2001f3cec5..0f5ea1cf609 100644 --- a/docs/changelogs/v0.21.md +++ b/docs/changelogs/v0.21.md @@ -4,20 +4,19 @@ ## v0.21.0 -* [v0.21.0](#v0210) - * [Overview](#overview) - * [πŸ”¦ Highlights](#-highlights) - * [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping) - * [Gateway: `DeserializedResponses` config flag](#gateway-deserializedresponses-config-flag) - * [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client) - * [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages) - * [Gateway: subdomain redirects are now `text/html`](#gateway-subdomain-redirects-are-now-texthtml) - * [Gateway: support for CAR parameters of IPIP-402](#gateway-support-for-car-parameters-of-ipip-402) - * [`ipfs dag import` no longer pins by default](#ipfs-dag-import-no-longer-pins-by-default) - * [`ipfs dag stat` deduping statistics](#ipfs-dag-stat-deduping-statistics) - * [Accelerated DHT Client is no longer experimental](#accelerated-dht-client-is-no-longer-experimental) - * [πŸ“ Changelog](#-changelog) - * [πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦ Contributors](#-contributors) +- [Overview](#overview) +- [πŸ”¦ Highlights](#-highlights) + - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping) + - [Gateway: `DeserializedResponses` config flag](#gateway-deserializedresponses-config-flag) + - [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client) + - [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages) + - [Gateway: subdomain redirects are now `text/html`](#gateway-subdomain-redirects-are-now-texthtml) + - [Gateway: support for partial CAR export parameters (IPIP-402)](#gateway-support-for-partial-car-export-parameters-ipip-402) + - [`ipfs dag import` no longer pins by default](#ipfs-dag-import-no-longer-pins-by-default) + - [`ipfs dag stat` deduping statistics](#ipfs-dag-stat-deduping-statistics) + - [Accelerated DHT Client is no longer experimental](#accelerated-dht-client-is-no-longer-experimental) +- [πŸ“ Changelog](#-changelog) +- [πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦ Contributors](#-contributors) ### Overview @@ -107,13 +106,36 @@ $ curl "https://subdomain-gw.example.net/ipfs/${cid}/" Rationale can be found in [kubo#9913](https://github.com/ipfs/kubo/pull/9913). -#### Gateway: support for CAR parameters of IPIP-402 +#### Gateway: support for partial CAR export parameters (IPIP-402) -The gateway now supports partial CAR export parameters as indicated in [IPIP-402](https://github.com/ipfs/specs/pull/402). +The gateway now supports optional CAR export parameters +`dag-scope=block|entity|all` and `entity-bytes=from:to` as specified in +[IPIP-402](https://github.com/ipfs/specs/pull/402). + +Batch block retrieval minimizes round trips, catering to the requirements of +light HTTP clients for directory enumeration, range requests, and content path +resolution. #### `ipfs dag import` no longer pins by default -With the gateway now supporting partial CAR exports, and incomplete DAG CARs becoming increasingly common. The pinning mode when using `ipfs dag import` has changed such that pinning the DAG or part of a DAG within an imported CAR is optional. To attempt to pin the DAG referenced by any roots in the CAR, opt in with the `--pin-roots` option. +With the gateway now capable of handling partial CAR exports +([IPIP-402](https://github.com/ipfs/specs/pull/402)) and incomplete DAG CARs +becoming more prevalent, there have been changes to the pinning mode when using +`ipfs dag import`. + +Recursive pinning of the entire DAG within an imported CAR is now optional. To +explicitly attempt pinning the DAG referenced by any roots present in the CAR, +you can opt in by using the `--pin-roots` option. + +Pinning incomplete DAG will produce an error: + +```console +$ curl 'http://127.0.0.1:8080/ipns/docs.ipfs.tech?format=car&dag-scope=entity' > ./partial-entity.car # Kubo 0.21.0 with IPIP-402 (only root block of unixfs dir) +$ ipfs dag import --stats --pin-roots=true ./partial-entity.car +Error pinning QmPDC11yLAbVw3dX5jMeEuSdk4BiVjSd9X87zaYRdVjzW3 FAILED: block was not found locally (offline): ipld: could not find QmPDvrDAz2aHeLjPVQ4uh1neyknUmDpf1GsBzAbpFhS8ro +Imported 1 blocks (1618 bytes) +[exit code 1] +``` #### `ipfs dag stat` deduping statistics