Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 609a335

Browse files
whyrusleepingwhyrusleeping
committedJul 29, 2015
WIP: separate private and public blocks
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
1 parent 25fd6b9 commit 609a335

File tree

10 files changed

+69
-34
lines changed

10 files changed

+69
-34
lines changed
 

‎blocks/blockstore/blockstore.go

+7-4
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ var log = eventlog.Logger("blockstore")
2121
// BlockPrefix namespaces blockstore datastores
2222
var BlockPrefix = ds.NewKey("blocks")
2323

24+
var PrivateBlockPrefix = ds.NewKey("private")
25+
2426
var ValueTypeMismatch = errors.New("The retrieved value is not a Block")
2527

2628
var ErrNotFound = errors.New("blockstore: block not found")
@@ -52,11 +54,12 @@ type GCBlockstore interface {
5254
}
5355

5456
func NewBlockstore(d ds.Batching) *blockstore {
55-
var dsb ds.Batching
56-
dd := dsns.Wrap(d, BlockPrefix)
57-
dsb = dd
57+
return NewBlockstoreWithPrefix(d, BlockPrefix)
58+
}
59+
60+
func NewBlockstoreWithPrefix(d ds.Batching, prefix ds.Key) *blockstore {
5861
return &blockstore{
59-
datastore: dsb,
62+
datastore: dsns.Wrap(d, prefix),
6063
}
6164
}
6265

‎blockservice/blockservice.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ package blockservice
55

66
import (
77
"errors"
8-
"fmt"
98

109
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
1110
blocks "github.com/ipfs/go-ipfs/blocks"
@@ -50,7 +49,7 @@ type BlockService struct {
5049
// NewBlockService creates a BlockService with given datastore instance.
5150
func New(bs blockstore.Blockstore, rem exchange.Interface) (*BlockService, error) {
5251
if bs == nil {
53-
return nil, fmt.Errorf("BlockService requires valid blockstore")
52+
panic("BlockService requires valid blockstore")
5453
}
5554
if rem == nil {
5655
log.Warning("blockservice running in local (offline) mode.")

‎core/core.go

+26-9
Original file line numberDiff line numberDiff line change
@@ -87,13 +87,16 @@ type IpfsNode struct {
8787
PrivateKey ic.PrivKey // the local node's private Key
8888

8989
// Services
90-
Peerstore peer.Peerstore // storage for other Peer instances
91-
Blockstore bstore.GCBlockstore // the block store (lower level)
92-
Blocks *bserv.BlockService // the block service, get/add blocks.
93-
DAG merkledag.DAGService // the merkle dag service, get/add objects.
94-
Resolver *path.Resolver // the path resolution system
95-
Reporter metrics.Reporter
96-
Discovery discovery.Service
90+
Peerstore peer.Peerstore // storage for other Peer instances
91+
Blockstore bstore.GCBlockstore // the block store (lower level)
92+
PrivBlocks bstore.GCBlockstore // the private blockstore
93+
Blocks *bserv.BlockService // the block service, get/add blocks.
94+
95+
DAG merkledag.DAGService // the merkle dag service, get/add objects.
96+
PrivDAG merkledag.DAGService // the private merkledag service for local objects
97+
Resolver *path.Resolver // the path resolution system
98+
Reporter metrics.Reporter
99+
Discovery discovery.Service
97100

98101
// Online
99102
PeerHost p2phost.Host // the network host (server+client)
@@ -151,9 +154,16 @@ func NewIPFSNode(ctx context.Context, option ConfigOption) (*IpfsNode, error) {
151154
node.Peerstore = peer.NewPeerstore()
152155
}
153156
node.DAG = merkledag.NewDAGService(node.Blocks)
154-
node.Pinning, err = pin.LoadPinner(node.Repo.Datastore(), node.DAG)
157+
158+
privbserv, err := bserv.New(node.PrivBlocks, offline.Exchange(node.PrivBlocks))
159+
if err != nil {
160+
return nil, err
161+
}
162+
node.PrivDAG = merkledag.NewDAGService(privbserv)
163+
164+
node.Pinning, err = pin.LoadPinner(node.Repo.Datastore(), node.DAG, node.PrivDAG)
155165
if err != nil {
156-
node.Pinning = pin.NewPinner(node.Repo.Datastore(), node.DAG)
166+
node.Pinning = pin.NewPinner(node.Repo.Datastore(), node.DAG, node.PrivDAG)
157167
}
158168
node.Resolver = &path.Resolver{DAG: node.DAG}
159169

@@ -229,6 +239,13 @@ func standardWithRouting(r repo.Repo, online bool, routingOption RoutingOption,
229239
return nil, err
230240
}
231241

242+
n.PrivBlocks, err = bstore.WriteCached(
243+
bstore.NewBlockstoreWithPrefix(n.Repo.Datastore(), bstore.PrivateBlockPrefix),
244+
kSizeBlockstoreWriteCache)
245+
if err != nil {
246+
return nil, err
247+
}
248+
232249
if online {
233250
do := setupDiscoveryOption(n.Repo.Config().Discovery)
234251
if err := n.startOnlineServices(ctx, routingOption, hostOption, do); err != nil {

‎core/mock/mock.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ func NewMockNode() (*core.IpfsNode, error) {
7272

7373
nd.DAG = mdag.NewDAGService(bserv)
7474

75-
nd.Pinning = pin.NewPinner(nd.Repo.Datastore(), nd.DAG)
75+
nd.Pinning = pin.NewPinner(nd.Repo.Datastore(), nd.DAG, nd.PrivDAG)
7676

7777
// Namespace resolver
7878
nd.Namesys = nsys.NewNameSystem(nd.Routing)

‎merkledag/merkledag_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func getDagservAndPinner(t *testing.T) dagservAndPinner {
3737
t.Fatal(err)
3838
}
3939
dserv := NewDAGService(blockserv)
40-
mpin := pin.NewPinner(db, dserv)
40+
mpin := pin.NewPinner(db, dserv, dserv)
4141
return dagservAndPinner{
4242
ds: dserv,
4343
mp: mpin,

‎pin/pin.go

+13-10
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,12 @@ type pinner struct {
6464
// not delete them.
6565
internalPin map[key.Key]struct{}
6666
dserv mdag.DAGService
67+
privdag mdag.DAGService
6768
dstore ds.Datastore
6869
}
6970

7071
// NewPinner creates a new pinner using the given datastore as a backend
71-
func NewPinner(dstore ds.Datastore, serv mdag.DAGService) Pinner {
72+
func NewPinner(dstore ds.Datastore, pub mdag.DAGService, priv mdag.DAGService) Pinner {
7273

7374
// Load set from given datastore...
7475
rcset := set.NewSimpleBlockSet()
@@ -78,8 +79,9 @@ func NewPinner(dstore ds.Datastore, serv mdag.DAGService) Pinner {
7879
return &pinner{
7980
recursePin: rcset,
8081
directPin: dirset,
81-
dserv: serv,
82+
dserv: pub,
8283
dstore: dstore,
84+
privdag: priv,
8385
}
8486
}
8587

@@ -208,7 +210,7 @@ func (p *pinner) RemovePinWithMode(key key.Key, mode PinMode) {
208210
}
209211

210212
// LoadPinner loads a pinner and its keysets from the given datastore
211-
func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) {
213+
func LoadPinner(d ds.Datastore, dserv mdag.DAGService, priv mdag.DAGService) (Pinner, error) {
212214
p := new(pinner)
213215

214216
rootKeyI, err := d.Get(pinDatastoreKey)
@@ -225,7 +227,7 @@ func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) {
225227
ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5)
226228
defer cancel()
227229

228-
root, err := dserv.Get(ctx, rootKey)
230+
root, err := priv.Get(ctx, rootKey)
229231
if err != nil {
230232
return nil, fmt.Errorf("cannot find pinning root object: %v", err)
231233
}
@@ -238,15 +240,15 @@ func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) {
238240
}
239241

240242
{ // load recursive set
241-
recurseKeys, err := loadSet(ctx, dserv, root, linkRecursive, recordInternal)
243+
recurseKeys, err := loadSet(ctx, priv, root, linkRecursive, recordInternal)
242244
if err != nil {
243245
return nil, fmt.Errorf("cannot load recursive pins: %v", err)
244246
}
245247
p.recursePin = set.SimpleSetFromKeys(recurseKeys)
246248
}
247249

248250
{ // load direct set
249-
directKeys, err := loadSet(ctx, dserv, root, linkDirect, recordInternal)
251+
directKeys, err := loadSet(ctx, priv, root, linkDirect, recordInternal)
250252
if err != nil {
251253
return nil, fmt.Errorf("cannot load direct pins: %v", err)
252254
}
@@ -257,6 +259,7 @@ func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) {
257259

258260
// assign services
259261
p.dserv = dserv
262+
p.privdag = priv
260263
p.dstore = d
261264

262265
return p, nil
@@ -286,7 +289,7 @@ func (p *pinner) Flush() error {
286289

287290
root := &mdag.Node{}
288291
{
289-
n, err := storeSet(ctx, p.dserv, p.directPin.GetKeys(), recordInternal)
292+
n, err := storeSet(ctx, p.privdag, p.directPin.GetKeys(), recordInternal)
290293
if err != nil {
291294
return err
292295
}
@@ -296,7 +299,7 @@ func (p *pinner) Flush() error {
296299
}
297300

298301
{
299-
n, err := storeSet(ctx, p.dserv, p.recursePin.GetKeys(), recordInternal)
302+
n, err := storeSet(ctx, p.privdag, p.recursePin.GetKeys(), recordInternal)
300303
if err != nil {
301304
return err
302305
}
@@ -306,12 +309,12 @@ func (p *pinner) Flush() error {
306309
}
307310

308311
// add the empty node, its referenced by the pin sets but never created
309-
_, err := p.dserv.Add(new(mdag.Node))
312+
_, err := p.privdag.Add(new(mdag.Node))
310313
if err != nil {
311314
return err
312315
}
313316

314-
k, err := p.dserv.Add(root)
317+
k, err := p.privdag.Add(root)
315318
if err != nil {
316319
return err
317320
}

‎pin/pin_test.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ func TestPinnerBasic(t *testing.T) {
4848
dserv := mdag.NewDAGService(bserv)
4949

5050
// TODO does pinner need to share datastore with blockservice?
51-
p := NewPinner(dstore, dserv)
51+
p := NewPinner(dstore, dserv, dserv)
5252

5353
a, ak := randNode()
5454
_, err = dserv.Add(a)
@@ -132,7 +132,7 @@ func TestPinnerBasic(t *testing.T) {
132132
t.Fatal(err)
133133
}
134134

135-
np, err := LoadPinner(dstore, dserv)
135+
np, err := LoadPinner(dstore, dserv, dserv)
136136
if err != nil {
137137
t.Fatal(err)
138138
}
@@ -156,7 +156,7 @@ func TestDuplicateSemantics(t *testing.T) {
156156
dserv := mdag.NewDAGService(bserv)
157157

158158
// TODO does pinner need to share datastore with blockservice?
159-
p := NewPinner(dstore, dserv)
159+
p := NewPinner(dstore, dserv, dserv)
160160

161161
a, _ := randNode()
162162
_, err = dserv.Add(a)
@@ -192,7 +192,7 @@ func TestFlush(t *testing.T) {
192192
}
193193

194194
dserv := mdag.NewDAGService(bserv)
195-
p := NewPinner(dstore, dserv)
195+
p := NewPinner(dstore, dserv, dserv)
196196
_, k := randNode()
197197

198198
p.PinWithMode(k, Recursive)
@@ -213,7 +213,7 @@ func TestPinRecursiveFail(t *testing.T) {
213213

214214
dserv := mdag.NewDAGService(bserv)
215215

216-
p := NewPinner(dstore, dserv)
216+
p := NewPinner(dstore, dserv, dserv)
217217

218218
a, _ := randNode()
219219
b, _ := randNode()

‎repo/fsrepo/defaultds.go

+4
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,10 @@ func openDefaultDatastore(r *FSRepo) (repo.Datastore, error) {
6161
Prefix: ds.NewKey("/blocks"),
6262
Datastore: metricsBlocks,
6363
},
64+
{
65+
Prefix: ds.NewKey("/private"),
66+
Datastore: metricsBlocks,
67+
},
6468
{
6569
Prefix: ds.NewKey("/"),
6670
Datastore: metricsLevelDB,

‎test/integration/core.go

+9
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,16 @@ func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig, routin
4646
return nil, err
4747
}
4848
exch := bitswap.New(ctx, p, bsn, bstore, alwaysSendToPeer)
49+
50+
privbstore, err := blockstore.WriteCached(
51+
blockstore.NewBlockstoreWithPrefix(ds, blockstore.PrivateBlockPrefix),
52+
kWriteCacheElems)
53+
if err != nil {
54+
return nil, err
55+
}
56+
4957
n.Blockstore = bstore
58+
n.PrivBlocks = privbstore
5059
n.Exchange = exch
5160
n.Routing = dhtt
5261
return n, nil

‎unixfs/mod/dagmodifier_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ func getMockDagServ(t testing.TB) (mdag.DAGService, pin.Pinner) {
3636
t.Fatal(err)
3737
}
3838
dserv := mdag.NewDAGService(bserv)
39-
return dserv, pin.NewPinner(tsds, dserv)
39+
return dserv, pin.NewPinner(tsds, dserv, dserv)
4040
}
4141

4242
func getMockDagServAndBstore(t testing.TB) (mdag.DAGService, blockstore.GCBlockstore, pin.Pinner) {
@@ -48,7 +48,7 @@ func getMockDagServAndBstore(t testing.TB) (mdag.DAGService, blockstore.GCBlocks
4848
t.Fatal(err)
4949
}
5050
dserv := mdag.NewDAGService(bserv)
51-
return dserv, bstore, pin.NewPinner(tsds, dserv)
51+
return dserv, bstore, pin.NewPinner(tsds, dserv, dserv)
5252
}
5353

5454
func getNode(t testing.TB, dserv mdag.DAGService, size int64, pinner pin.Pinner) ([]byte, *mdag.Node) {

0 commit comments

Comments
 (0)
Please sign in to comment.