Skip to content

Commit e517b65

Browse files
jbenetjbenet
committedJul 29, 2015
Merge pull request #1506 from ipfs/feat/patch-create
allow patch to optionally create intermediate dirs
·
v0.34.0-rc1floodsub-0
2 parents 681da0a + aa7d946 commit e517b65

File tree

8 files changed

+488
-81
lines changed

8 files changed

+488
-81
lines changed
 

‎core/commands/object.go

+27-65
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
cmds "github.com/ipfs/go-ipfs/commands"
1919
core "github.com/ipfs/go-ipfs/core"
2020
dag "github.com/ipfs/go-ipfs/merkledag"
21+
dagutils "github.com/ipfs/go-ipfs/merkledag/utils"
2122
path "github.com/ipfs/go-ipfs/path"
2223
ft "github.com/ipfs/go-ipfs/unixfs"
2324
u "github.com/ipfs/go-ipfs/util"
@@ -453,7 +454,9 @@ This removes the link named foo from the hash in $FOO_BAR and returns the
453454
resulting object hash.
454455
`,
455456
},
456-
Options: []cmds.Option{},
457+
Options: []cmds.Option{
458+
cmds.BoolOption("create", "p", "create intermediate directories on add-link"),
459+
},
457460
Arguments: []cmds.Argument{
458461
cmds.StringArg("root", true, false, "the hash of the node to modify"),
459462
cmds.StringArg("command", true, false, "the operation to perform"),
@@ -467,9 +470,13 @@ resulting object hash.
467470
return
468471
}
469472

470-
rhash := key.B58KeyDecode(req.Arguments()[0])
473+
rootarg := req.Arguments()[0]
474+
if strings.HasPrefix(rootarg, "/ipfs/") {
475+
rootarg = rootarg[6:]
476+
}
477+
rhash := key.B58KeyDecode(rootarg)
471478
if rhash == "" {
472-
res.SetError(fmt.Errorf("incorrectly formatted root hash"), cmds.ErrNormal)
479+
res.SetError(fmt.Errorf("incorrectly formatted root hash: %s", req.Arguments()[0]), cmds.ErrNormal)
473480
return
474481
}
475482

@@ -580,19 +587,18 @@ func rmLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
580587
return "", err
581588
}
582589

583-
name := req.Arguments()[2]
590+
path := req.Arguments()[2]
584591

585-
err = root.RemoveNodeLink(name)
586-
if err != nil {
587-
return "", err
588-
}
592+
e := dagutils.NewDagEditor(nd.DAG, root)
589593

590-
newkey, err := nd.DAG.Add(root)
594+
err = e.RmLink(req.Context(), path)
591595
if err != nil {
592596
return "", err
593597
}
594598

595-
return newkey, nil
599+
nnode := e.GetNode()
600+
601+
return nnode.Key()
596602
}
597603

598604
func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
@@ -608,72 +614,28 @@ func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
608614
path := req.Arguments()[2]
609615
childk := key.B58KeyDecode(req.Arguments()[3])
610616

611-
parts := strings.Split(path, "/")
612-
613-
nnode, err := insertNodeAtPath(req.Context(), nd.DAG, root, parts, childk)
617+
create, _, err := req.Option("create").Bool()
614618
if err != nil {
615619
return "", err
616620
}
617-
return nnode.Key()
618-
}
619-
620-
func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childk key.Key) (*dag.Node, error) {
621-
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
622-
childnd, err := ds.Get(ctx, childk)
623-
if err != nil {
624-
cancel()
625-
return nil, err
626-
}
627-
cancel()
628-
629-
err = root.AddNodeLinkClean(childname, childnd)
630-
if err != nil {
631-
return nil, err
632-
}
633-
634-
_, err = ds.Add(root)
635-
if err != nil {
636-
return nil, err
637-
}
638-
return root, nil
639-
}
640-
641-
func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert key.Key) (*dag.Node, error) {
642-
if len(path) == 1 {
643-
return addLink(ctx, ds, root, path[0], toinsert)
644-
}
645-
646-
child, err := root.GetNodeLink(path[0])
647-
if err != nil {
648-
return nil, err
649-
}
650-
651-
nd, err := child.GetNode(ctx, ds)
652-
if err != nil {
653-
return nil, err
654-
}
655621

656-
ndprime, err := insertNodeAtPath(ctx, ds, nd, path[1:], toinsert)
657-
if err != nil {
658-
return nil, err
622+
var createfunc func() *dag.Node
623+
if create {
624+
createfunc = func() *dag.Node {
625+
return &dag.Node{Data: ft.FolderPBData()}
626+
}
659627
}
660628

661-
err = root.RemoveNodeLink(path[0])
662-
if err != nil {
663-
return nil, err
664-
}
629+
e := dagutils.NewDagEditor(nd.DAG, root)
665630

666-
err = root.AddNodeLinkClean(path[0], ndprime)
631+
err = e.InsertNodeAtPath(req.Context(), path, childk, createfunc)
667632
if err != nil {
668-
return nil, err
633+
return "", err
669634
}
670635

671-
_, err = ds.Add(root)
672-
if err != nil {
673-
return nil, err
674-
}
636+
nnode := e.GetNode()
675637

676-
return root, nil
638+
return nnode.Key()
677639
}
678640

679641
func nodeFromTemplate(template string) (*dag.Node, error) {

‎merkledag/coding.go

+1-11
Original file line numberDiff line numberDiff line change
@@ -37,16 +37,6 @@ func (n *Node) Unmarshal(encoded []byte) error {
3737
return nil
3838
}
3939

40-
// MarshalTo encodes a *Node instance into a given byte slice.
41-
// The conversion uses an intermediate PBNode.
42-
func (n *Node) MarshalTo(encoded []byte) error {
43-
pbn := n.getPBNode()
44-
if _, err := pbn.MarshalTo(encoded); err != nil {
45-
return fmt.Errorf("Marshal failed. %v", err)
46-
}
47-
return nil
48-
}
49-
5040
// Marshal encodes a *Node instance into a new byte slice.
5141
// The conversion uses an intermediate PBNode.
5242
func (n *Node) Marshal() ([]byte, error) {
@@ -82,7 +72,7 @@ func (n *Node) Encoded(force bool) ([]byte, error) {
8272
var err error
8373
n.encoded, err = n.Marshal()
8474
if err != nil {
85-
return []byte{}, err
75+
return nil, err
8676
}
8777
n.cached = u.Hash(n.encoded)
8878
}

‎merkledag/merkledag_test.go

+68
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"io"
77
"io/ioutil"
8+
"strings"
89
"sync"
910
"testing"
1011

@@ -221,3 +222,70 @@ func runBatchFetchTest(t *testing.T, read io.Reader) {
221222

222223
wg.Wait()
223224
}
225+
226+
func TestRecursiveAdd(t *testing.T) {
227+
a := &Node{Data: []byte("A")}
228+
b := &Node{Data: []byte("B")}
229+
c := &Node{Data: []byte("C")}
230+
d := &Node{Data: []byte("D")}
231+
e := &Node{Data: []byte("E")}
232+
233+
err := a.AddNodeLink("blah", b)
234+
if err != nil {
235+
t.Fatal(err)
236+
}
237+
238+
err = b.AddNodeLink("foo", c)
239+
if err != nil {
240+
t.Fatal(err)
241+
}
242+
243+
err = b.AddNodeLink("bar", d)
244+
if err != nil {
245+
t.Fatal(err)
246+
}
247+
248+
err = d.AddNodeLink("baz", e)
249+
if err != nil {
250+
t.Fatal(err)
251+
}
252+
253+
dsp := getDagservAndPinner(t)
254+
err = dsp.ds.AddRecursive(a)
255+
if err != nil {
256+
t.Fatal(err)
257+
}
258+
259+
assertCanGet(t, dsp.ds, a)
260+
assertCanGet(t, dsp.ds, b)
261+
assertCanGet(t, dsp.ds, c)
262+
assertCanGet(t, dsp.ds, d)
263+
assertCanGet(t, dsp.ds, e)
264+
}
265+
266+
func assertCanGet(t *testing.T, ds DAGService, n *Node) {
267+
k, err := n.Key()
268+
if err != nil {
269+
t.Fatal(err)
270+
}
271+
272+
_, err = ds.Get(context.TODO(), k)
273+
if err != nil {
274+
t.Fatal(err)
275+
}
276+
}
277+
278+
func TestCantGet(t *testing.T) {
279+
dsp := getDagservAndPinner(t)
280+
a := &Node{Data: []byte("A")}
281+
282+
k, err := a.Key()
283+
if err != nil {
284+
t.Fatal(err)
285+
}
286+
287+
_, err = dsp.ds.Get(context.TODO(), k)
288+
if !strings.Contains(err.Error(), "not found") {
289+
t.Fatal("expected err not found, got: ", err)
290+
}
291+
}

‎merkledag/node.go

+24-5
Original file line numberDiff line numberDiff line change
@@ -129,13 +129,23 @@ func (n *Node) AddRawLink(name string, l *Link) error {
129129
// Remove a link on this node by the given name
130130
func (n *Node) RemoveNodeLink(name string) error {
131131
n.encoded = nil
132-
for i, l := range n.Links {
133-
if l.Name == name {
134-
n.Links = append(n.Links[:i], n.Links[i+1:]...)
135-
return nil
132+
good := make([]*Link, 0, len(n.Links))
133+
var found bool
134+
135+
for _, l := range n.Links {
136+
if l.Name != name {
137+
good = append(good, l)
138+
} else {
139+
found = true
136140
}
137141
}
138-
return ErrNotFound
142+
n.Links = good
143+
144+
if !found {
145+
return ErrNotFound
146+
}
147+
148+
return nil
139149
}
140150

141151
// Return a copy of the link with given name
@@ -153,6 +163,15 @@ func (n *Node) GetNodeLink(name string) (*Link, error) {
153163
return nil, ErrNotFound
154164
}
155165

166+
func (n *Node) GetLinkedNode(ctx context.Context, ds DAGService, name string) (*Node, error) {
167+
lnk, err := n.GetNodeLink(name)
168+
if err != nil {
169+
return nil, err
170+
}
171+
172+
return lnk.GetNode(ctx, ds)
173+
}
174+
156175
// Copy returns a copy of the node.
157176
// NOTE: does not make copies of Node objects in the links.
158177
func (n *Node) Copy() *Node {

‎merkledag/node_test.go

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package merkledag
2+
3+
import (
4+
"testing"
5+
)
6+
7+
func TestRemoveLink(t *testing.T) {
8+
nd := &Node{
9+
Links: []*Link{
10+
&Link{Name: "a"},
11+
&Link{Name: "b"},
12+
&Link{Name: "a"},
13+
&Link{Name: "a"},
14+
&Link{Name: "c"},
15+
&Link{Name: "a"},
16+
},
17+
}
18+
19+
err := nd.RemoveNodeLink("a")
20+
if err != nil {
21+
t.Fatal(err)
22+
}
23+
24+
if len(nd.Links) != 2 {
25+
t.Fatal("number of links incorrect")
26+
}
27+
28+
if nd.Links[0].Name != "b" {
29+
t.Fatal("link order wrong")
30+
}
31+
32+
if nd.Links[1].Name != "c" {
33+
t.Fatal("link order wrong")
34+
}
35+
36+
// should fail
37+
err = nd.RemoveNodeLink("a")
38+
if err != ErrNotFound {
39+
t.Fatal("should have failed to remove link")
40+
}
41+
42+
// ensure nothing else got touched
43+
if len(nd.Links) != 2 {
44+
t.Fatal("number of links incorrect")
45+
}
46+
47+
if nd.Links[0].Name != "b" {
48+
t.Fatal("link order wrong")
49+
}
50+
51+
if nd.Links[1].Name != "c" {
52+
t.Fatal("link order wrong")
53+
}
54+
}

‎merkledag/utils/utils.go

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
package dagutils
2+
3+
import (
4+
"errors"
5+
"strings"
6+
7+
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
8+
9+
key "github.com/ipfs/go-ipfs/blocks/key"
10+
dag "github.com/ipfs/go-ipfs/merkledag"
11+
)
12+
13+
type Editor struct {
14+
root *dag.Node
15+
ds dag.DAGService
16+
}
17+
18+
func NewDagEditor(ds dag.DAGService, root *dag.Node) *Editor {
19+
return &Editor{
20+
root: root,
21+
ds: ds,
22+
}
23+
}
24+
25+
func (e *Editor) GetNode() *dag.Node {
26+
return e.root.Copy()
27+
}
28+
29+
func (e *Editor) AddLink(ctx context.Context, childname string, childk key.Key) error {
30+
nd, err := addLink(ctx, e.ds, e.root, childname, childk)
31+
if err != nil {
32+
return err
33+
}
34+
e.root = nd
35+
return nil
36+
}
37+
38+
func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childk key.Key) (*dag.Node, error) {
39+
if childname == "" {
40+
return nil, errors.New("cannot create link with no name!")
41+
}
42+
43+
childnd, err := ds.Get(ctx, childk)
44+
if err != nil {
45+
return nil, err
46+
}
47+
48+
// ensure no link with that name already exists
49+
_ = root.RemoveNodeLink(childname) // ignore error, only option is ErrNotFound
50+
51+
err = root.AddNodeLinkClean(childname, childnd)
52+
if err != nil {
53+
return nil, err
54+
}
55+
56+
_, err = ds.Add(root)
57+
if err != nil {
58+
return nil, err
59+
}
60+
return root, nil
61+
}
62+
63+
func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert key.Key, create func() *dag.Node) error {
64+
splpath := strings.Split(path, "/")
65+
nd, err := insertNodeAtPath(ctx, e.ds, e.root, splpath, toinsert, create)
66+
if err != nil {
67+
return err
68+
}
69+
e.root = nd
70+
return nil
71+
}
72+
73+
func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert key.Key, create func() *dag.Node) (*dag.Node, error) {
74+
if len(path) == 1 {
75+
return addLink(ctx, ds, root, path[0], toinsert)
76+
}
77+
78+
nd, err := root.GetLinkedNode(ctx, ds, path[0])
79+
if err != nil {
80+
// if 'create' is true, we create directories on the way down as needed
81+
if err == dag.ErrNotFound && create != nil {
82+
nd = create()
83+
} else {
84+
return nil, err
85+
}
86+
}
87+
88+
ndprime, err := insertNodeAtPath(ctx, ds, nd, path[1:], toinsert, create)
89+
if err != nil {
90+
return nil, err
91+
}
92+
93+
_ = root.RemoveNodeLink(path[0])
94+
err = root.AddNodeLinkClean(path[0], ndprime)
95+
if err != nil {
96+
return nil, err
97+
}
98+
99+
_, err = ds.Add(root)
100+
if err != nil {
101+
return nil, err
102+
}
103+
104+
return root, nil
105+
}
106+
107+
func (e *Editor) RmLink(ctx context.Context, path string) error {
108+
splpath := strings.Split(path, "/")
109+
nd, err := rmLink(ctx, e.ds, e.root, splpath)
110+
if err != nil {
111+
return err
112+
}
113+
e.root = nd
114+
return nil
115+
}
116+
117+
func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string) (*dag.Node, error) {
118+
if len(path) == 1 {
119+
// base case, remove node in question
120+
err := root.RemoveNodeLink(path[0])
121+
if err != nil {
122+
return nil, err
123+
}
124+
125+
_, err = ds.Add(root)
126+
if err != nil {
127+
return nil, err
128+
}
129+
130+
return root, nil
131+
}
132+
133+
nd, err := root.GetLinkedNode(ctx, ds, path[0])
134+
if err != nil {
135+
return nil, err
136+
}
137+
138+
nnode, err := rmLink(ctx, ds, nd, path[1:])
139+
if err != nil {
140+
return nil, err
141+
}
142+
143+
_ = root.RemoveNodeLink(path[0])
144+
err = root.AddNodeLinkClean(path[0], nnode)
145+
if err != nil {
146+
return nil, err
147+
}
148+
149+
_, err = ds.Add(root)
150+
if err != nil {
151+
return nil, err
152+
}
153+
154+
return root, nil
155+
}

‎merkledag/utils/utils_test.go

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
package dagutils
2+
3+
import (
4+
"strings"
5+
"testing"
6+
7+
key "github.com/ipfs/go-ipfs/blocks/key"
8+
dag "github.com/ipfs/go-ipfs/merkledag"
9+
mdtest "github.com/ipfs/go-ipfs/merkledag/test"
10+
11+
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
12+
)
13+
14+
func TestAddLink(t *testing.T) {
15+
ds := mdtest.Mock(t)
16+
fishnode := &dag.Node{
17+
Data: []byte("fishcakes!"),
18+
}
19+
20+
fk, err := ds.Add(fishnode)
21+
if err != nil {
22+
t.Fatal(err)
23+
}
24+
25+
nd := new(dag.Node)
26+
nnode, err := addLink(context.Background(), ds, nd, "fish", fk)
27+
if err != nil {
28+
t.Fatal(err)
29+
}
30+
31+
fnprime, err := nnode.GetLinkedNode(context.Background(), ds, "fish")
32+
if err != nil {
33+
t.Fatal(err)
34+
}
35+
36+
fnpkey, err := fnprime.Key()
37+
if err != nil {
38+
t.Fatal(err)
39+
}
40+
41+
if fnpkey != fk {
42+
t.Fatal("wrong child node found!")
43+
}
44+
}
45+
46+
func assertNodeAtPath(t *testing.T, ds dag.DAGService, root *dag.Node, path string, exp key.Key) {
47+
parts := strings.Split(path, "/")
48+
cur := root
49+
for _, e := range parts {
50+
nxt, err := cur.GetLinkedNode(context.Background(), ds, e)
51+
if err != nil {
52+
t.Fatal(err)
53+
}
54+
55+
cur = nxt
56+
}
57+
58+
curk, err := cur.Key()
59+
if err != nil {
60+
t.Fatal(err)
61+
}
62+
63+
if curk != exp {
64+
t.Fatal("node not as expected at end of path")
65+
}
66+
}
67+
68+
func TestInsertNode(t *testing.T) {
69+
ds := mdtest.Mock(t)
70+
root := new(dag.Node)
71+
e := NewDagEditor(ds, root)
72+
73+
testInsert(t, e, "a", "anodefortesting", false, "")
74+
testInsert(t, e, "a/b", "data", false, "")
75+
testInsert(t, e, "a/b/c/d/e", "blah", false, "merkledag: not found")
76+
testInsert(t, e, "a/b/c/d/e", "foo", true, "")
77+
testInsert(t, e, "a/b/c/d/f", "baz", true, "")
78+
testInsert(t, e, "a/b/c/d/f", "bar", true, "")
79+
80+
testInsert(t, e, "", "bar", true, "cannot create link with no name!")
81+
testInsert(t, e, "////", "slashes", true, "cannot create link with no name!")
82+
83+
k, err := e.GetNode().Key()
84+
if err != nil {
85+
t.Fatal(err)
86+
}
87+
88+
if k.B58String() != "QmThorWojP6YzLJwDukxiYCoKQSwyrMCvdt4WZ6rPm221t" {
89+
t.Fatal("output was different than expected")
90+
}
91+
}
92+
93+
func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) {
94+
child := &dag.Node{Data: []byte(data)}
95+
ck, err := e.ds.Add(child)
96+
if err != nil {
97+
t.Fatal(err)
98+
}
99+
100+
var c func() *dag.Node
101+
if create {
102+
c = func() *dag.Node {
103+
return &dag.Node{}
104+
}
105+
}
106+
107+
err = e.InsertNodeAtPath(context.TODO(), path, ck, c)
108+
if experr != "" {
109+
var got string
110+
if err != nil {
111+
got = err.Error()
112+
}
113+
if got != experr {
114+
t.Fatalf("expected '%s' but got '%s'", experr, got)
115+
}
116+
return
117+
}
118+
119+
if err != nil {
120+
t.Fatal(err)
121+
}
122+
123+
assertNodeAtPath(t, e.ds, e.root, path, ck)
124+
}

‎test/sharness/t0051-object.sh

+35
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,22 @@ test_description="Test object command"
1010

1111
test_init_ipfs
1212

13+
test_patch_create_path() {
14+
root=$1
15+
name=$2
16+
target=$3
17+
18+
test_expect_success "object patch --create works" '
19+
PCOUT=$(ipfs object patch --create $root add-link $name $target)
20+
'
21+
22+
test_expect_success "output looks good" '
23+
ipfs cat $PCOUT/$name > tpcp_out &&
24+
ipfs cat $target > tpcp_exp &&
25+
test_cmp tpcp_out tpcp_exp
26+
'
27+
}
28+
1329
test_object_cmd() {
1430

1531
test_expect_success "'ipfs add testData' succeeds" '
@@ -145,6 +161,25 @@ test_object_cmd() {
145161
echo QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn > rmlink_exp &&
146162
test_cmp rmlink_exp rmlink_output
147163
'
164+
165+
test_expect_success "multilayer rm-link should work" '
166+
ipfs object patch $(cat multi_patch) rm-link a/b/c > multi_link_rm_out
167+
'
168+
169+
test_expect_success "output looks good" '
170+
echo "QmZD3r9cZjzU8huNY2JS9TC6n8daDfT8TmE8zBSqG31Wvq" > multi_link_rm_exp &&
171+
test_cmp multi_link_rm_out multi_link_rm_exp
172+
'
173+
174+
test_patch_create_path $EMPTY a/b/c $FILE
175+
176+
test_patch_create_path $EMPTY a $FILE
177+
178+
test_patch_create_path $EMPTY a/b/b/b/b $FILE
179+
180+
test_expect_success "create bad path fails" '
181+
test_must_fail ipfs object patch --create $EMPTY add-link / $FILE
182+
'
148183
}
149184

150185
# should work offline

0 commit comments

Comments
 (0)
Please sign in to comment.