Skip to content

feat(dag): add --local-only to dag export and import #11229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ChayanDass wants to merge 10 commits into
base: master
Choose a base branch
from
Open

feat(dag): add --local-only to dag export and import #11229

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
9ad0c3b
feat(dag): add --local-only to dag export and import
ChayanDass Mar 7, 2026
fd7ee67
Merge branch 'master' into feat/dag-export-import-local-only
ChayanDass Mar 7, 2026
b402796
chore(deps): update go-car/v2 to latest master
lidel Mar 8, 2026
7b99db6
fix(dag): avoid CID round-trip in export and fix ci failure
ChayanDass Mar 10, 2026
3b27eed
dag: add validation and tests for --local-only flag
ChayanDass Mar 11, 2026
6a726f2
Merge branch 'master' into feat/dag-export-import-local-only
ChayanDass Mar 11, 2026
d24cde4
Merge branch 'master' into feat/dag-export-import-local-only
ChayanDass Mar 14, 2026
56a284a
Merge branch 'master' into feat/dag-export-import-local-only
ChayanDass Mar 16, 2026
6adbadf
Merge branch 'master' into feat/dag-export-import-local-only
ChayanDass Mar 18, 2026
76bc4e8
Merge branch 'master' into feat/dag-export-import-local-only
ChayanDass Mar 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions core/commands/dag/dag.go
View file
Open in desktop
Comment thread
ChayanDass marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ const (
statsOptionName = "stats"
fastProvideRootOptionName = "fast-provide-root"
fastProvideWaitOptionName = "fast-provide-wait"
localOnlyOptionName = "local-only"
)

// DagCmd provides a subset of commands for interacting with ipld dag objects
Expand Down Expand Up @@ -192,6 +193,9 @@ Note:
currently present in the blockstore does not represent a complete DAG,
pinning of that individual root will fail.

Use --local-only with --pin-roots=false when importing a partial CAR (e.g. from
'dag export --local-only').

Comment thread
ChayanDass marked this conversation as resolved.
Outdated
FAST PROVIDE OPTIMIZATION:

Root CIDs from CAR headers are immediately provided to the DHT in addition
Expand All @@ -213,6 +217,7 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/
},
Options: []cmds.Option{
cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
cmds.BoolOption(localOnlyOptionName, "Import a partial CAR without pinning roots (for CARs from dag export --local-only)."),
cmds.BoolOption(silentOptionName, "No output."),
cmds.BoolOption(statsOptionName, "Output stats."),
cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CIDs to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"),
Expand Down Expand Up @@ -285,6 +290,7 @@ CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/
},
Options: []cmds.Option{
cmds.BoolOption(progressOptionName, "p", "Display progress on CLI. Defaults to true when STDERR is a TTY."),
cmds.BoolOption(localOnlyOptionName, "If set, only blocks present locally are exported; missing blocks are skipped (partial CAR). Use with --offline for a local-only DAG walk."),
},
Run: dagExport,
PostRun: cmds.PostRunMap{
Expand Down
36 changes: 29 additions & 7 deletions core/commands/dag/export.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package dagcmd

import (
"bytes"
"context"
"errors"
"fmt"
Expand All @@ -16,7 +17,10 @@ import (
"github.com/ipfs/kubo/core/commands/cmdutils"
iface "github.com/ipfs/kubo/core/coreiface"
gocar "github.com/ipld/go-car/v2"
"github.com/ipld/go-ipld-prime/datamodel"
"github.com/ipld/go-ipld-prime/linking"
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
"github.com/ipld/go-ipld-prime/traversal"
selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
)

Expand All @@ -27,6 +31,7 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
return err
}

localOnly, _ := req.Options[localOnlyOptionName].(bool)
api, err := cmdenv.GetApi(env, req)
if err != nil {
return err
Expand All @@ -51,7 +56,25 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
}()

lsys := cidlink.DefaultLinkSystem()
lsys.SetReadStorage(&dagStore{dag: api.Dag(), ctx: req.Context})
ds := &dagStore{dag: api.Dag(), ctx: req.Context}
if localOnly {
lsys.StorageReadOpener = func(lctx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) {
cl, ok := lnk.(cidlink.Link)
if !ok {
return nil, fmt.Errorf("unsupported link type: %T", lnk)
}
data, err := ds.Get(lctx.Ctx, cl.Cid.String())
Comment thread
ChayanDass marked this conversation as resolved.
Outdated
if err != nil {
if ipld.IsNotFound(err) {
return nil, traversal.SkipMe{}
}
return nil, fmt.Errorf("local block read failed: %w", err)
}
return bytes.NewReader(data), nil
}
} else {
lsys.SetReadStorage(ds)
}

// Uncomment the following to support CARv2 output.
/*
Expand Down Expand Up @@ -189,12 +212,11 @@ func (ds *dagStore) Has(ctx context.Context, key string) (bool, error) {
}

func cidFromBinString(key string) (cid.Cid, error) {
l, k, err := cid.CidFromBytes([]byte(key))
if err != nil {
return cid.Undef, fmt.Errorf("dagStore: key was not a cid: %w", err)
if l, k, err := cid.CidFromBytes([]byte(key)); err == nil && l == len(key) {
return k, nil
}
if l != len(key) {
return cid.Undef, fmt.Errorf("dagSore: key was not a cid: had %d bytes leftover", len(key)-l)
if c, decodeErr := cid.Decode(key); decodeErr == nil {
return c, nil
}
return k, nil
return cid.Undef, fmt.Errorf("dagStore: key was not a cid (binary or base58)")
}
4 changes: 4 additions & 0 deletions core/commands/dag/import.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,11 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
}

doPinRoots, _ := req.Options[pinRootsOptionName].(bool)
localOnly, _ := req.Options[localOnlyOptionName].(bool)

if doPinRoots && localOnly {
return fmt.Errorf("cannot pass both --%s and --%s", pinRootsOptionName, localOnlyOptionName)
}
fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool)
fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool)

Expand Down
2 changes: 1 addition & 1 deletion docs/examples/kubo-as-a-library/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ require (
github.com/ipfs/go-peertaskqueue v0.8.3 // indirect
github.com/ipfs/go-test v0.2.3 // indirect
github.com/ipfs/go-unixfsnode v1.10.3 // indirect
github.com/ipld/go-car/v2 v2.16.0 // indirect
github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 // indirect
github.com/ipld/go-codec-dagpb v1.7.0 // indirect
github.com/ipld/go-ipld-prime v0.22.0 // indirect
github.com/ipshipyard/p2p-forge v0.7.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions docs/examples/kubo-as-a-library/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,8 +416,8 @@ github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc=
github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o=
github.com/ipfs/go-unixfsnode v1.10.3 h1:c8sJjuGNkxXAQH75P+f5ngPda/9T+DrboVA0TcDGvGI=
github.com/ipfs/go-unixfsnode v1.10.3/go.mod h1:2Jlc7DoEwr12W+7l8Hr6C7XF4NHST3gIkqSArLhGSxU=
github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco=
github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34=
github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 h1:aUJhNVqQZV3PhxQX/BPAkurYnsxBmMLx2a7YPsgd5NU=
github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070/go.mod h1:eN3661vGFqfxzIfPLrCQIdKcY9iIOCgZON1CedVtvwk=
github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0=
github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM=
github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8=
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ require (
github.com/ipfs/go-metrics-prometheus v0.1.0
github.com/ipfs/go-test v0.2.3
github.com/ipfs/go-unixfsnode v1.10.3
github.com/ipld/go-car/v2 v2.16.0
github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070
github.com/ipld/go-codec-dagpb v1.7.0
github.com/ipld/go-ipld-prime v0.22.0
github.com/ipshipyard/p2p-forge v0.7.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,8 +458,8 @@ github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc=
github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o=
github.com/ipfs/go-unixfsnode v1.10.3 h1:c8sJjuGNkxXAQH75P+f5ngPda/9T+DrboVA0TcDGvGI=
github.com/ipfs/go-unixfsnode v1.10.3/go.mod h1:2Jlc7DoEwr12W+7l8Hr6C7XF4NHST3gIkqSArLhGSxU=
github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco=
github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34=
github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 h1:aUJhNVqQZV3PhxQX/BPAkurYnsxBmMLx2a7YPsgd5NU=
github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070/go.mod h1:eN3661vGFqfxzIfPLrCQIdKcY9iIOCgZON1CedVtvwk=
github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0=
github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM=
github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8=
Expand Down
2 changes: 1 addition & 1 deletion test/dependencies/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ require (
github.com/ipfs/go-metrics-interface v0.3.0 // indirect
github.com/ipfs/go-unixfsnode v1.10.3 // indirect
github.com/ipfs/kubo v0.31.0 // indirect
github.com/ipld/go-car/v2 v2.16.0 // indirect
github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 // indirect
github.com/ipld/go-codec-dagpb v1.7.0 // indirect
github.com/ipld/go-ipld-prime v0.22.0 // indirect
github.com/ipshipyard/p2p-forge v0.7.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions test/dependencies/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -496,8 +496,8 @@ github.com/ipfs/iptb v1.4.1 h1:faXd3TKGPswbHyZecqqg6UfbES7RDjTKQb+6VFPKDUo=
github.com/ipfs/iptb v1.4.1/go.mod h1:nTsBMtVYFEu0FjC5DgrErnABm3OG9ruXkFXGJoTV5OA=
github.com/ipfs/iptb-plugins v0.5.1 h1:11PNTNEt2+SFxjUcO5qpyCTXqDj6T8Tx9pU/G4ytCIQ=
github.com/ipfs/iptb-plugins v0.5.1/go.mod h1:mscJAjRnu4g16QK6oUBn9RGpcp8ueJmLfmPxIG/At78=
github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco=
github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34=
github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 h1:aUJhNVqQZV3PhxQX/BPAkurYnsxBmMLx2a7YPsgd5NU=
github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070/go.mod h1:eN3661vGFqfxzIfPLrCQIdKcY9iIOCgZON1CedVtvwk=
github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0=
github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM=
github.com/ipld/go-ipld-prime v0.22.0 h1:YJhDhjEOvOYaqshd3b4atIWUoRg/rKrgmwCyUHwlbuY=
Expand Down
Loading