diff --git a/.circleci/main.yml b/.circleci/main.yml index bdfe72f07e7..27bc3ccd38a 100644 --- a/.circleci/main.yml +++ b/.circleci/main.yml @@ -216,7 +216,7 @@ jobs: command: | npm init -y npm install ipfs@^0.59.1 - npm install ipfs-interop@^7.0.3 + npm install ipfs/interop#fix/go-ipfs-automatic-sharding npm install mocha-circleci-reporter@0.0.3 working_directory: ~/ipfs/go-ipfs/interop - run: diff --git a/core/coreapi/test/path_test.go b/core/coreapi/test/path_test.go index cc7ba2efaef..20ba2227d46 100644 --- a/core/coreapi/test/path_test.go +++ b/core/coreapi/test/path_test.go @@ -14,7 +14,6 @@ import ( "github.com/ipld/go-ipld-prime" ) - func TestPathUnixFSHAMTPartial(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -27,16 +26,17 @@ func TestPathUnixFSHAMTPartial(t *testing.T) { a := apis[0] // Setting this after instantiating the swarm so that it's not clobbered by loading the go-ipfs config - prevVal := uio.UseHAMTSharding - uio.UseHAMTSharding = true + // TODO: Either create a sharded directory manually, or make it such that it is automatically sized as sharded + prevVal := uio.HAMTShardingSize + uio.HAMTShardingSize = 1 defer func() { - uio.UseHAMTSharding = prevVal + uio.HAMTShardingSize = prevVal }() // Create and add a sharded directory dir := make(map[string]files.Node) // Make sure we have at least two levels of sharding - for i := 0; i < uio.DefaultShardWidth + 1; i++ { + for i := 0; i < uio.DefaultShardWidth+1; i++ { dir[strconv.Itoa(i)] = files.NewBytesFile([]byte(strconv.Itoa(i))) } @@ -67,7 +67,7 @@ func TestPathUnixFSHAMTPartial(t *testing.T) { for k := range dir { // The node will go out to the (non-existent) network looking for the missing block. Make sure we're erroring // because we exceeded the timeout on our query - timeoutCtx, timeoutCancel := context.WithTimeout(ctx, time.Second * 1) + timeoutCtx, timeoutCancel := context.WithTimeout(ctx, time.Second*1) _, err := a.ResolveNode(timeoutCtx, path.Join(r, k)) if err != nil { if timeoutCtx.Err() == nil { diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 6146f807a71..50ced95fd15 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -218,14 +218,15 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...options.Unixf func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, settings *options.UnixfsLsSettings) coreiface.DirEntry { if linkres.Err != nil { - return coreiface.DirEntry{Err: linkres.Err} + return coreiface.DirEntry{ + Err: linkres.Err, + } } lnk := coreiface.DirEntry{ Name: linkres.Link.Name, Cid: linkres.Link.Cid, } - switch lnk.Cid.Type() { case cid.Raw: // No need to check with raw leaves diff --git a/core/node/groups.go b/core/node/groups.go index 930a2151bef..f307e259780 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -16,6 +16,7 @@ import ( "github.com/ipfs/go-ipfs/core/node/libp2p" "github.com/ipfs/go-ipfs/p2p" + "github.com/alecthomas/units" offline "github.com/ipfs/go-ipfs-exchange-offline" offroute "github.com/ipfs/go-ipfs-routing/offline" uio "github.com/ipfs/go-unixfs/io" @@ -317,7 +318,7 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option { } // TEMP: setting global sharding switch here - uio.UseHAMTSharding = cfg.Experimental.ShardingEnabled + uio.HAMTShardingSize = int(256 * units.KiB) return fx.Options( bcfgOpts, diff --git a/docs/examples/go-ipfs-as-a-library/go.sum b/docs/examples/go-ipfs-as-a-library/go.sum index b2cc10b62dc..3f9f801bbde 100644 --- a/docs/examples/go-ipfs-as-a-library/go.sum +++ b/docs/examples/go-ipfs-as-a-library/go.sum @@ -62,6 +62,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= @@ -467,7 +468,6 @@ github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFq github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew= github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0= github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= -github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6ZpusRg= github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU= github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo= @@ -517,7 +517,6 @@ github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Ax github.com/ipfs/go-log/v2 v2.3.0 h1:31Re/cPqFHpsRHgyVwjWADPoF0otB1WrjTy8ZFYwEZU= github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g= github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= -github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4= github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M= @@ -527,8 +526,8 @@ github.com/ipfs/go-merkledag v0.4.0/go.mod h1:XshXBkhyeS63YNGisLL1uDSfuTyrQIxVUO github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks= -github.com/ipfs/go-mfs v0.1.2 h1:DlelNSmH+yz/Riy0RjPKlooPg0KML4lXGdLw7uZkfAg= -github.com/ipfs/go-mfs v0.1.2/go.mod h1:T1QBiZPEpkPLzDqEJLNnbK55BVKVlNi2a+gVm4diFo0= +github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164 h1:0ATu9s5KktHhm8aYRSe1ysOJPik3dRwU/uag1Bcz+tg= +github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164/go.mod h1:A525zyeY2o078AoxhjJirOlDTXI1GnZxiYQnESGJ9WU= github.com/ipfs/go-namesys v0.3.1 h1:DqmeXlVODejOyECAqoqhSB5JGRv8aRFhtG0oPDmxsMc= github.com/ipfs/go-namesys v0.3.1/go.mod h1:/BL4xk8LP5Lq82AmaRKyxZv/eYRlumNiU9SZUe1Hlps= github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno= @@ -543,10 +542,10 @@ github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUn github.com/ipfs/go-peertaskqueue v0.4.0 h1:x1hFgA4JOUJ3ntPfqLRu6v4k6kKL0p07r3RSg9JNyHI= github.com/ipfs/go-peertaskqueue v0.4.0/go.mod h1:KL9F49hXJMoXCad8e5anivjN+kWdr+CyGcyh4K6doLc= github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4= -github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= -github.com/ipfs/go-unixfs v0.2.5 h1:irj/WzIcgTBay48mSMUYDbKlIzIocXWcuUUsi5qOMOE= -github.com/ipfs/go-unixfs v0.2.5/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= +github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0= +github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa h1:b2JM6ADtCk9+DgUmhwR15oxQSnyEucqNssy6GRfm+J4= +github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0= github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s= github.com/ipfs/go-unixfsnode v1.1.3 h1:IyqJBGIEvcHvll1wDDVIHOEVXnE+IH6tjzTWpZ6kGiI= github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLfRYdWY9veZ4= diff --git a/go.mod b/go.mod index 739d85be399..3cbfa9fee03 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/ipfs/go-ipfs require ( bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc contrib.go.opencensus.io/exporter/prometheus v0.3.0 + github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a github.com/blang/semver/v4 v4.0.0 github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-systemd/v22 v22.3.2 @@ -49,11 +50,11 @@ require ( github.com/ipfs/go-merkledag v0.4.0 github.com/ipfs/go-metrics-interface v0.0.1 github.com/ipfs/go-metrics-prometheus v0.0.2 - github.com/ipfs/go-mfs v0.1.2 + github.com/ipfs/go-mfs v0.1.3-0.20211112012225-23d6734eab23 github.com/ipfs/go-namesys v0.3.1 github.com/ipfs/go-path v0.1.2 github.com/ipfs/go-pinning-service-http-client v0.1.0 - github.com/ipfs/go-unixfs v0.2.5 + github.com/ipfs/go-unixfs v0.2.7-0.20211112011223-bd53b6a811b1 github.com/ipfs/go-unixfsnode v1.1.3 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.5.1 diff --git a/go.sum b/go.sum index 643d4206182..145cd25e17e 100644 --- a/go.sum +++ b/go.sum @@ -65,6 +65,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= +github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= @@ -385,6 +387,8 @@ github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/go-bitfield v1.0.0 h1:y/XHm2GEmD9wKngheWNNCNL0pzrWXZwCdQGv1ikXknQ= +github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= github.com/ipfs/go-bitswap v0.0.9/go.mod h1:kAPf5qgn2W2DrgAcscZ3HrM9qh4pH+X8Fkk3UPrwvis= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= @@ -529,7 +533,6 @@ github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Ax github.com/ipfs/go-log/v2 v2.3.0 h1:31Re/cPqFHpsRHgyVwjWADPoF0otB1WrjTy8ZFYwEZU= github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g= github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= -github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4= github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M= @@ -540,8 +543,8 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s= github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks= -github.com/ipfs/go-mfs v0.1.2 h1:DlelNSmH+yz/Riy0RjPKlooPg0KML4lXGdLw7uZkfAg= -github.com/ipfs/go-mfs v0.1.2/go.mod h1:T1QBiZPEpkPLzDqEJLNnbK55BVKVlNi2a+gVm4diFo0= +github.com/ipfs/go-mfs v0.1.3-0.20211112012225-23d6734eab23 h1:RP+F0BONWIXRDb1x5QXk9PV7AnEosaKZCzU4RInsNvw= +github.com/ipfs/go-mfs v0.1.3-0.20211112012225-23d6734eab23/go.mod h1:+NT2mLpzr3LKkRZbjYSFjUTUzHAqJjPbBfnDsLJPJjw= github.com/ipfs/go-namesys v0.3.1 h1:DqmeXlVODejOyECAqoqhSB5JGRv8aRFhtG0oPDmxsMc= github.com/ipfs/go-namesys v0.3.1/go.mod h1:/BL4xk8LP5Lq82AmaRKyxZv/eYRlumNiU9SZUe1Hlps= github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno= @@ -557,10 +560,9 @@ github.com/ipfs/go-peertaskqueue v0.4.0 h1:x1hFgA4JOUJ3ntPfqLRu6v4k6kKL0p07r3RSg github.com/ipfs/go-peertaskqueue v0.4.0/go.mod h1:KL9F49hXJMoXCad8e5anivjN+kWdr+CyGcyh4K6doLc= github.com/ipfs/go-pinning-service-http-client v0.1.0 h1:Au0P4NglL5JfzhNSZHlZ1qra+IcJyO3RWMd9EYCwqSY= github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4= -github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= -github.com/ipfs/go-unixfs v0.2.5 h1:irj/WzIcgTBay48mSMUYDbKlIzIocXWcuUUsi5qOMOE= -github.com/ipfs/go-unixfs v0.2.5/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= +github.com/ipfs/go-unixfs v0.2.7-0.20211112011223-bd53b6a811b1 h1:FFSVXA9ns5IqwQRZUgq4GFk7qN1C9LbAwY6CJwk0I4Q= +github.com/ipfs/go-unixfs v0.2.7-0.20211112011223-bd53b6a811b1/go.mod h1:t8BWCW4OvTjcxQsX4e+GFroSZ5fCUXB5ywIMbw9eH/Y= github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s= github.com/ipfs/go-unixfsnode v1.1.3 h1:IyqJBGIEvcHvll1wDDVIHOEVXnE+IH6tjzTWpZ6kGiI= github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLfRYdWY9veZ4= diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index 3276ff04d18..535630f97ea 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -45,21 +45,37 @@ verify_dir_contents() { ' } +# Same as `t0260-sharding.sh`, we need a big directory now to trigger sharding. +test_expect_success "set up test data" ' + mkdir big_dir + for i in `seq 6500` # just to be sure + do + echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars + done +' + test_sharding() { local EXTRA ARGS EXTRA=$1 ARGS=$2 # only applied to the initial directory test_expect_success "make a directory $EXTRA" ' - ipfs files mkdir $ARGS /foo + ipfs add $ARGS -r -q big_dir | tail -n1 > sharddir_cid && + ipfs files cp /ipfs/`cat sharddir_cid` /foo ' test_expect_success "can make 100 files in a directory $EXTRA" ' printf "" > list_exp_raw - for i in `seq 100 -1 1` + # Already present in the directory to trigger sharding + for i in `seq 6500` # just to be sure + do + echo `printf "file%06d" $i` >> list_exp_raw + done + + for i in `seq 10100 -1 10000` do - echo $i | ipfs files write --create /foo/file$i || return 1 - echo file$i >> list_exp_raw + echo $i | ipfs files write --create /foo/`printf "file%06d" $i` || return 1 + echo `printf "file%06d" $i` >> list_exp_raw done ' # Create the files in reverse (unsorted) order (`seq 100 -1 1`) @@ -80,13 +96,13 @@ test_sharding() { ' test_expect_success "can read a file from sharded directory $EXTRA" ' - ipfs files read /foo/file65 > file_out && + ipfs files read /foo/file000065 > file_out && echo "65" > file_exp && test_cmp file_out file_exp ' test_expect_success "can pin a file from sharded directory $EXTRA" ' - ipfs files stat --hash /foo/file42 > pin_file_hash && + ipfs files stat --hash /foo/file000042 > pin_file_hash && ipfs pin add < pin_file_hash > pin_hash ' @@ -817,10 +833,10 @@ test_expect_success "enable sharding in config" ' test_launch_ipfs_daemon_without_network -SHARD_HASH=QmPkwLJTYZRGPJ8Lazr9qPdrLmswPtUjaDbEpmR9jEh1se +SHARD_HASH=QmWSqxvnnHh6NL9ctzpwMiWg7iyCKTfcvLoQbCqC7qPQHw test_sharding "(cidv0)" -SHARD_HASH=bafybeib46tpawg2d2hhlmmn2jvgio33wqkhlehxrem7wbfvqqikure37rm +SHARD_HASH=bafybeid7vaa7ywcd3mgrkdohpyqm5wxtmkwovyvxftzio3wuie3vlx4tcq test_sharding "(cidv1 root)" "--cid-version=1" test_kill_ipfs_daemon diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh index 4270d688d06..1a1c046661f 100755 --- a/test/sharness/t0260-sharding.sh +++ b/test/sharness/t0260-sharding.sh @@ -8,60 +8,76 @@ test_description="Test directory sharding" . lib/test-lib.sh +# We shard based on size with a threshold of 256 KiB (see `HAMTShardingSize` +# in core/node/groups.go) above which directories are sharded. +# The directory size is estimated as the size of each link (roughly entry name, +# here of always 10 chars, and CID byte length, normally 34 bytes). So we need +# 256 KiB / (34 + 10) ~ 6000 entries in the directory to trigger sharding. +# We create then two directories: one above the threshold (big_dir) and one +# below (small_dir), and hard-code the CIDs of their sharded and unsharded +# codifications as IPFS directories. test_expect_success "set up test data" ' - mkdir testdata - for i in `seq 2000` + mkdir big_dir + for i in `seq 5960` # Just above the number of entries that trigger sharding for 256KiB do - echo $i > testdata/file$i + echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars done -' -test_add_large_dir() { + mkdir small_dir + for i in `seq 100` + do + echo $i > small_dir/`printf "file%06d" $i` + done +' +# CID of big_dir/ which will be sharded. +SHARDED="QmWfjnRWRvdvYezQWnfbvrvY7JjrpevsE9cato1x76UqGr" +# CID of big_dir/ once we remove half its entries and trigger a switch +# back to a basic directory (un-sharding). +UNSHAREDED="QmbVxi5zDdzytrjdufUejM92JsWj8wGVmukk6tiPce3p1m" +# CID of small_dir/ which will *not* be sharded. +NOT_SHARDED="QmdBXmm4HRpUhyzzctbFvi2tLai3XFL1YjmE1qfpJe61NX" + +test_add_dir_with_sharding_enabled() { exphash="$1" - test_expect_success "ipfs add on very large directory succeeds" ' - ipfs add -r -Q testdata > sharddir_out && + input_dir="$2" + test_expect_success "ipfs add directory (with sharding enabled) succeeds" ' + ipfs add -r -Q $input_dir > sharddir_out && echo "$exphash" > sharddir_exp && test_cmp sharddir_exp sharddir_out ' - test_expect_success "ipfs get on very large directory succeeds" ' - ipfs get -o testdata-out "$exphash" && - test_cmp testdata testdata-out + test_expect_success "ipfs add directory (with sharding enabled) succeeds matches expected hash" ' + ipfs get -o output_dir "$exphash" && + test_cmp $input_dir output_dir + rm output_dir -r ' } test_init_ipfs -UNSHARDED="QmavrTrQG4VhoJmantURAYuw3bowq3E2WcvP36NRQDAC1N" -test_add_large_dir "$UNSHARDED" +test_add_dir_with_sharding_enabled "$SHARDED" big_dir +test_add_dir_with_sharding_enabled "$NOT_SHARDED" small_dir test_launch_ipfs_daemon -test_add_large_dir "$UNSHARDED" +test_add_dir_with_sharding_enabled "$SHARDED" big_dir +test_add_dir_with_sharding_enabled "$NOT_SHARDED" small_dir test_kill_ipfs_daemon -test_expect_success "enable sharding" ' - ipfs config --json Experimental.ShardingEnabled true -' - -SHARDED="QmSCJD1KYLhVVHqBK3YyXuoEqHt7vggyJhzoFYbT8v1XYL" -test_add_large_dir "$SHARDED" - -test_launch_ipfs_daemon - -test_add_large_dir "$SHARDED" - -test_kill_ipfs_daemon - -test_expect_success "sharded and unsharded output look the same" ' - ipfs ls "$SHARDED" | sort > sharded_out && - ipfs ls "$UNSHARDED" | sort > unsharded_out && - test_cmp sharded_out unsharded_out +test_expect_success "remove a few entries from big_dir/ to trigger unsharding" ' + ipfs files cp /ipfs/$SHARDED /big_dir && + for i in `seq 5` + do + ipfs files rm /big_dir/`printf "file%06d" $i` + done && + ipfs files stat --hash /big_dir > unshard_dir_hash && + echo "$UNSHAREDED" > unshard_exp && + test_cmp unshard_exp unshard_dir_hash ' test_expect_success "ipfs cat error output the same" ' test_expect_code 1 ipfs cat "$SHARDED" 2> sharded_err && - test_expect_code 1 ipfs cat "$UNSHARDED" 2> unsharded_err && + test_expect_code 1 ipfs cat "$NOT_SHARDED" 2> unsharded_err && test_cmp sharded_err unsharded_err ' @@ -69,7 +85,7 @@ test_expect_success "'ipfs ls --resolve-type=false --size=false' admits missing ipfs ls "$SHARDED" | head -1 > first_file && ipfs ls --size=false "$SHARDED" | sort > sharded_out_nosize && read -r HASH _ NAME expected && - curl -sfo actual "http://127.0.0.1:$GWAY_PORT/ipfs/$SHARDED/file100" && + curl -sfo actual "http://127.0.0.1:$GWAY_PORT/ipfs/$SHARDED/file000100" && test_cmp expected actual ' test_expect_success "'ipfs resolve' can resolve sharded dirs" ' echo /ipfs/QmZ3RfWk1u5LEGYLHA633B5TNJy3Du27K6Fny9wcxpowGS > expected && - ipfs resolve "/ipfs/$SHARDED/file100" > actual && + ipfs resolve "/ipfs/$SHARDED/file000100" > actual && test_cmp expected actual ' test_kill_ipfs_daemon -test_add_large_dir_v1() { +test_add_dir_with_sharding_enabled_v1() { exphash="$1" + input_dir="$2" test_expect_success "ipfs add (CIDv1) on very large directory succeeds" ' - ipfs add -r -Q --cid-version=1 testdata > sharddir_out && + ipfs add -r -Q --cid-version=1 "$input_dir" > sharddir_out && echo "$exphash" > sharddir_exp && test_cmp sharddir_exp sharddir_out ' test_expect_success "can access a path under the dir" ' - ipfs cat "$exphash/file20" > file20_out && - test_cmp testdata/file20 file20_out + ipfs cat "$exphash/file003020" > file3020_out && + test_cmp "$input_dir/file003020" file3020_out ' } # this hash implies the directory is CIDv1 and leaf entries are CIDv1 and raw -SHARDEDV1="bafybeibiemewfzzdyhq2l74wrd6qj2oz42usjlktgnlqv4yfawgouaqn4u" -test_add_large_dir_v1 "$SHARDEDV1" +SHARDEDV1="bafybeiglv7wdwjqaqxjs4nk3uo4vvldhmmod5vkinvgwihyrv6bdzuzbbm" +test_add_dir_with_sharding_enabled_v1 "$SHARDEDV1" big_dir test_launch_ipfs_daemon -test_add_large_dir_v1 "$SHARDEDV1" +test_add_dir_with_sharding_enabled_v1 "$SHARDEDV1" big_dir test_kill_ipfs_daemon