blob: 2339743a317f243b4363745bff18be6fc1e41b95 [file] [log] [blame]
Manish R Jain9b4612b2017-03-14 11:30:041/*
2 * Copyright 2017 Dgraph Labs, Inc. and Contributors
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Manish R Jain22e69902017-04-24 02:02:0117package badger
Manish R Jain61b81192017-01-26 07:17:1818
19import (
Ibrahim Jarifd8e1fcf2019-07-19 05:07:0320 "bytes"
Manish R Jain61b81192017-01-26 07:17:1821 "fmt"
Ganesh Acharya080745a2017-04-04 05:15:4022 "io/ioutil"
Balaji Jinnahd981f472020-07-10 06:25:0223 "math"
Manish R Jain61b81192017-01-26 07:17:1824 "math/rand"
25 "os"
ashishf7bbdb82019-03-19 15:27:0826 "reflect"
Manish R Jaind22c0e82018-09-19 17:44:0127 "sync"
Manish R Jain61b81192017-01-26 07:17:1828 "testing"
Manish R Jain22378322019-05-12 07:24:2129 "time"
Ganesh Acharya080745a2017-04-04 05:15:4030
aman bansalb69163b2021-01-13 05:25:3731 "github.com/dgraph-io/badger/v3/y"
Manish R Jaind22c0e82018-09-19 17:44:0132 humanize "github.com/dustin/go-humanize"
Motakjuqff18eb02017-06-27 07:39:1533 "github.com/stretchr/testify/require"
Manish R Jain61b81192017-01-26 07:17:1834)
35
aman bansal5dbae102021-02-04 12:59:5136func TestDynamicValueThreshold(t *testing.T) {
Ibrahim Jarif6c35ad62021-02-09 20:11:1137 t.Skip()
aman bansal5dbae102021-02-04 12:59:5138 dir, err := ioutil.TempDir("", "badger-test")
39 y.Check(err)
40 defer removeDir(dir)
41 kv, _ := Open(getTestOptions(dir).WithValueThreshold(32).WithVLogPercentile(0.99))
42 defer kv.Close()
43 log := &kv.vlog
44 for vl := 32; vl <= 1024; vl = vl + 4 {
45 for i := 0; i < 1000; i++ {
46 val := make([]byte, vl)
47 y.Check2(rand.Read(val))
48 e1 := &Entry{
49 Key: []byte(fmt.Sprintf("samplekey_%d_%d", vl, i)),
50 Value: val,
51 meta: bitValuePointer,
52 }
53 b := new(request)
54 b.Entries = []*Entry{e1}
55 log.write([]*request{b})
56 }
57 t.Logf("value threshold is %d \n", log.db.valueThreshold())
58 }
59
60 for vl := 511; vl >= 31; vl = vl - 4 {
61 for i := 0; i < 5000; i++ {
62 val := make([]byte, vl)
63 y.Check2(rand.Read(val))
64 e1 := &Entry{
65 Key: []byte(fmt.Sprintf("samplekey_%d_%d", vl, i)),
66 Value: val,
67 meta: bitValuePointer,
68 }
69 b := new(request)
70 b.Entries = []*Entry{e1}
71 log.write([]*request{b})
72 }
73 t.Logf("value threshold is %d \n", log.db.valueThreshold())
74 }
75 require.Equal(t, log.db.valueThreshold(), int64(995))
76}
77
Ganesh Acharyabd270632017-04-27 16:31:2478func TestValueBasic(t *testing.T) {
Ashish Goswami8c85c272019-06-19 08:21:2279 dir, err := ioutil.TempDir("", "badger-test")
Ganesh Acharya080745a2017-04-04 05:15:4080 y.Check(err)
Ibrahim Jarif00039ea2019-10-17 13:02:3281 defer removeDir(dir)
Ganesh Acharya080745a2017-04-04 05:15:4082
Ibrahim Jarif675efcd2020-06-24 15:16:1283 kv, _ := Open(getTestOptions(dir).WithValueThreshold(32))
Manish R Jain4cab2332017-04-24 07:32:1584 defer kv.Close()
Manish R Jainfb0f0aa2017-05-03 01:17:0485 log := &kv.vlog
Manish R Jainf4542202017-04-14 01:56:4986
Sam Hughes8de624f2017-09-20 23:30:1687 // Use value big enough that the value log writes them even if SyncWrites is false.
88 const val1 = "sampleval012345678901234567890123"
89 const val2 = "samplevalb012345678901234567890123"
aman bansal5dbae102021-02-04 12:59:5190 require.True(t, int64(len(val1)) >= kv.vlog.db.valueThreshold())
Sam Hughes8de624f2017-09-20 23:30:1691
Ibrahim Jarifd8e1fcf2019-07-19 05:07:0392 e1 := &Entry{
Janardhan Reddyffaaa662017-10-02 06:47:4693 Key: []byte("samplekey"),
94 Value: []byte(val1),
Deepak Joisa5499e52017-11-02 02:03:1495 meta: bitValuePointer,
Manish R Jainb61131e2017-04-19 09:53:3596 }
Deepak Jois5bc2e162017-11-30 03:04:5797 e2 := &Entry{
Janardhan Reddyffaaa662017-10-02 06:47:4698 Key: []byte("samplekeyb"),
99 Value: []byte(val2),
Deepak Joisa5499e52017-11-02 02:03:14100 meta: bitValuePointer,
Ganesh Acharyaf9e14712017-04-30 20:39:14101 }
102
Manish R Jain9230dd02017-04-29 01:57:00103 b := new(request)
Ibrahim Jarifd8e1fcf2019-07-19 05:07:03104 b.Entries = []*Entry{e1, e2}
Manish R Jaindadcb4d2017-04-21 10:03:05105
Manish R Jain20849402017-05-17 08:36:32106 log.write([]*request{b})
Ganesh Acharyaf9e14712017-04-30 20:39:14107 require.Len(t, b.Ptrs, 2)
Manish R Jaina6860cb2017-10-02 07:32:11108 t.Logf("Pointer written: %+v %+v\n", b.Ptrs[0], b.Ptrs[1])
Manish R Jain526b6652017-04-13 08:22:46109
Naman Jaina35b08d2021-01-14 06:01:03110 buf1, lf1, err1 := log.readValueBytes(b.Ptrs[0])
111 buf2, lf2, err2 := log.readValueBytes(b.Ptrs[1])
Deepak Joisb9aae1b2017-08-31 05:06:49112 require.NoError(t, err1)
113 require.NoError(t, err2)
balajia425b0e2019-09-24 14:06:36114 defer runCallback(log.getUnlockCallback(lf1))
115 defer runCallback(log.getUnlockCallback(lf2))
116 e1, err = lf1.decodeEntry(buf1, b.Ptrs[0].Offset)
117 require.NoError(t, err)
118 e2, err = lf1.decodeEntry(buf2, b.Ptrs[1].Offset)
119 require.NoError(t, err)
120 readEntries := []Entry{*e1, *e2}
Deepak Jois5bc2e162017-11-30 03:04:57121 require.EqualValues(t, []Entry{
Ganesh Acharya080745a2017-04-04 05:15:40122 {
balajia425b0e2019-09-24 14:06:36123 Key: []byte("samplekey"),
124 Value: []byte(val1),
125 meta: bitValuePointer,
126 offset: b.Ptrs[0].Offset,
Ganesh Acharyaf9e14712017-04-30 20:39:14127 },
128 {
balajia425b0e2019-09-24 14:06:36129 Key: []byte("samplekeyb"),
130 Value: []byte(val2),
131 meta: bitValuePointer,
132 offset: b.Ptrs[1].Offset,
Ganesh Acharya080745a2017-04-04 05:15:40133 },
134 }, readEntries)
Manish R Jain83aa09d2017-10-04 07:20:27135
Ganesh Acharya080745a2017-04-04 05:15:40136}
137
Manish R Jaind22c0e82018-09-19 17:44:01138func TestValueGCManaged(t *testing.T) {
Manish R Jainb21f5912021-04-27 04:37:45139 t.Skipf("Value Log is not used in managed mode.")
140
Ashish Goswami8c85c272019-06-19 08:21:22141 dir, err := ioutil.TempDir("", "badger-test")
Manish R Jaind22c0e82018-09-19 17:44:01142 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:32143 defer removeDir(dir)
Manish R Jaind22c0e82018-09-19 17:44:01144
145 N := 10000
Ibrahim Jarif6c35ad62021-02-09 20:11:11146
Manish R Jaind22c0e82018-09-19 17:44:01147 opt := getTestOptions(dir)
148 opt.ValueLogMaxEntries = uint32(N / 10)
Manish R Jain970454a2018-10-02 22:05:04149 opt.managedTxns = true
Ibrahim Jarif6c35ad62021-02-09 20:11:11150 opt.BaseTableSize = 1 << 15
151 opt.ValueThreshold = 1 << 10
152 opt.MemTableSize = 1 << 15
153
Manish R Jaind22c0e82018-09-19 17:44:01154 db, err := Open(opt)
155 require.NoError(t, err)
156 defer db.Close()
157
158 var ts uint64
159 newTs := func() uint64 {
marigonzes28ef9bf2019-01-31 17:54:06160 ts++
Manish R Jaind22c0e82018-09-19 17:44:01161 return ts
162 }
163
164 sz := 64 << 10
165 var wg sync.WaitGroup
166 for i := 0; i < N; i++ {
167 v := make([]byte, sz)
168 rand.Read(v[:rand.Intn(sz)])
169
170 wg.Add(1)
171 txn := db.NewTransactionAt(newTs(), true)
Ashish Goswamie9447c92019-05-28 05:28:37172 require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v)))
Manish R Jaind22c0e82018-09-19 17:44:01173 require.NoError(t, txn.CommitAt(newTs(), func(err error) {
174 wg.Done()
175 require.NoError(t, err)
176 }))
177 }
178
179 for i := 0; i < N; i++ {
180 wg.Add(1)
181 txn := db.NewTransactionAt(newTs(), true)
182 require.NoError(t, txn.Delete([]byte(fmt.Sprintf("key%d", i))))
183 require.NoError(t, txn.CommitAt(newTs(), func(err error) {
184 wg.Done()
185 require.NoError(t, err)
186 }))
187 }
188 wg.Wait()
189 files, err := ioutil.ReadDir(dir)
190 require.NoError(t, err)
191 for _, fi := range files {
Naman Jain7770c6e2021-03-30 13:56:44192 t.Logf("File: %s. Size: %s\n", fi.Name(), humanize.IBytes(uint64(fi.Size())))
Manish R Jaind22c0e82018-09-19 17:44:01193 }
194
Manish R Jaine3a0d292020-10-07 01:41:41195 db.SetDiscardTs(math.MaxUint32)
196 db.Flatten(3)
197
Manish R Jaind22c0e82018-09-19 17:44:01198 for i := 0; i < 100; i++ {
199 // Try at max 100 times to GC even a single value log file.
200 if err := db.RunValueLogGC(0.0001); err == nil {
201 return // Done
202 }
203 }
204 require.Fail(t, "Unable to GC even a single value log file.")
205}
206
Ganesh Acharyabd270632017-04-27 16:31:24207func TestValueGC(t *testing.T) {
Ashish Goswami8c85c272019-06-19 08:21:22208 dir, err := ioutil.TempDir("", "badger-test")
Manish R Jain4cab2332017-04-24 07:32:15209 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:32210 defer removeDir(dir)
Szymon Matejczyk0fdb95e2017-05-08 14:55:14211 opt := getTestOptions(dir)
212 opt.ValueLogFileSize = 1 << 20
Ibrahim Jarif6c35ad62021-02-09 20:11:11213 opt.BaseTableSize = 1 << 15
214 opt.ValueThreshold = 1 << 10
Szymon Matejczyk0fdb95e2017-05-08 14:55:14215
Manish R Jainabaad902017-10-04 10:55:56216 kv, _ := Open(opt)
Ganesh Acharyaf9e14712017-04-30 20:39:14217 defer kv.Close()
Manish R Jain4cab2332017-04-24 07:32:15218
Szymon Matejczyk0fdb95e2017-05-08 14:55:14219 sz := 32 << 10
Manish R Jaina6860cb2017-10-02 07:32:11220 txn := kv.NewTransaction(true)
Manish R Jain4cab2332017-04-24 07:32:15221 for i := 0; i < 100; i++ {
Manish R Jain97cc4342017-04-25 05:03:15222 v := make([]byte, sz)
Peter Staceec0f11d2017-08-04 04:57:35223 rand.Read(v[:rand.Intn(sz)])
Ashish Goswamie9447c92019-05-28 05:28:37224 require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v)))
Janardhan Reddyffaaa662017-10-02 06:47:46225 if i%20 == 0 {
Manish R Jain6daccf92018-10-09 01:05:00226 require.NoError(t, txn.Commit())
Manish R Jaina6860cb2017-10-02 07:32:11227 txn = kv.NewTransaction(true)
Janardhan Reddyffaaa662017-10-02 06:47:46228 }
Manish R Jain4cab2332017-04-24 07:32:15229 }
Manish R Jain6daccf92018-10-09 01:05:00230 require.NoError(t, txn.Commit())
Manish R Jain4cab2332017-04-24 07:32:15231
232 for i := 0; i < 45; i++ {
Janardhan Reddyffaaa662017-10-02 06:47:46233 txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i)))
Manish R Jain4cab2332017-04-24 07:32:15234 }
235
Sam Hughesedbce1b2017-09-01 17:08:23236 kv.vlog.filesLock.RLock()
Sam Hughes2a6fc672017-09-03 22:08:47237 lf := kv.vlog.filesMap[kv.vlog.sortedFids()[0]]
Sam Hughesedbce1b2017-09-01 17:08:23238 kv.vlog.filesLock.RUnlock()
Manish R Jain4cab2332017-04-24 07:32:15239
Ganesh Acharyaf9e14712017-04-30 20:39:14240 // lf.iterate(0, func(e Entry) bool {
241 // e.print("lf")
242 // return true
243 // })
Manish R Jain4cab2332017-04-24 07:32:15244
Manish R Jaine3a0d292020-10-07 01:41:41245 kv.vlog.rewrite(lf)
Manish R Jain4cab2332017-04-24 07:32:15246 for i := 45; i < 100; i++ {
Manish R Jain55c350d2017-05-30 10:04:50247 key := []byte(fmt.Sprintf("key%d", i))
Manish R Jain83aa09d2017-10-04 07:20:27248
249 require.NoError(t, kv.View(func(txn *Txn) error {
250 item, err := txn.Get(key)
251 require.NoError(t, err)
Manish R Jaincfd203d2017-10-05 02:50:34252 val := getItemValue(t, item)
Manish R Jain83aa09d2017-10-04 07:20:27253 require.NotNil(t, val)
254 require.True(t, len(val) == sz, "Size found: %d", len(val))
255 return nil
256 }))
Manish R Jain4cab2332017-04-24 07:32:15257 }
258}
259
Janardhan Reddye0c226b2017-07-24 07:42:09260func TestValueGC2(t *testing.T) {
Ashish Goswami8c85c272019-06-19 08:21:22261 dir, err := ioutil.TempDir("", "badger-test")
Janardhan Reddye0c226b2017-07-24 07:42:09262 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:32263 defer removeDir(dir)
Janardhan Reddye0c226b2017-07-24 07:42:09264 opt := getTestOptions(dir)
265 opt.ValueLogFileSize = 1 << 20
Ibrahim Jarif6c35ad62021-02-09 20:11:11266 opt.BaseTableSize = 1 << 15
267 opt.ValueThreshold = 1 << 10
Janardhan Reddye0c226b2017-07-24 07:42:09268
Manish R Jainabaad902017-10-04 10:55:56269 kv, _ := Open(opt)
Janardhan Reddye0c226b2017-07-24 07:42:09270 defer kv.Close()
271
272 sz := 32 << 10
Manish R Jaina6860cb2017-10-02 07:32:11273 txn := kv.NewTransaction(true)
Janardhan Reddye0c226b2017-07-24 07:42:09274 for i := 0; i < 100; i++ {
275 v := make([]byte, sz)
Peter Staceec0f11d2017-08-04 04:57:35276 rand.Read(v[:rand.Intn(sz)])
Ashish Goswamie9447c92019-05-28 05:28:37277 require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v)))
Janardhan Reddyffaaa662017-10-02 06:47:46278 if i%20 == 0 {
Manish R Jain6daccf92018-10-09 01:05:00279 require.NoError(t, txn.Commit())
Manish R Jaina6860cb2017-10-02 07:32:11280 txn = kv.NewTransaction(true)
Janardhan Reddyf4076252017-09-05 01:28:57281 }
Janardhan Reddye0c226b2017-07-24 07:42:09282 }
Manish R Jain6daccf92018-10-09 01:05:00283 require.NoError(t, txn.Commit())
Janardhan Reddye0c226b2017-07-24 07:42:09284
285 for i := 0; i < 5; i++ {
Janardhan Reddyffaaa662017-10-02 06:47:46286 txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i)))
Janardhan Reddye0c226b2017-07-24 07:42:09287 }
288
Janardhan Reddye0c226b2017-07-24 07:42:09289 for i := 5; i < 10; i++ {
290 v := []byte(fmt.Sprintf("value%d", i))
Janardhan Reddyffaaa662017-10-02 06:47:46291 txnSet(t, kv, []byte(fmt.Sprintf("key%d", i)), v, 0)
Janardhan Reddye0c226b2017-07-24 07:42:09292 }
293
Sam Hughesedbce1b2017-09-01 17:08:23294 kv.vlog.filesLock.RLock()
Sam Hughes2a6fc672017-09-03 22:08:47295 lf := kv.vlog.filesMap[kv.vlog.sortedFids()[0]]
Sam Hughesedbce1b2017-09-01 17:08:23296 kv.vlog.filesLock.RUnlock()
Janardhan Reddye0c226b2017-07-24 07:42:09297
298 // lf.iterate(0, func(e Entry) bool {
299 // e.print("lf")
300 // return true
301 // })
302
Manish R Jaine3a0d292020-10-07 01:41:41303 kv.vlog.rewrite(lf)
Janardhan Reddye0c226b2017-07-24 07:42:09304 for i := 0; i < 5; i++ {
305 key := []byte(fmt.Sprintf("key%d", i))
Manish R Jain83aa09d2017-10-04 07:20:27306 require.NoError(t, kv.View(func(txn *Txn) error {
307 _, err := txn.Get(key)
Manish R Jainc3ac2ff2018-04-30 15:43:55308 require.Equal(t, ErrKeyNotFound, err)
Manish R Jain83aa09d2017-10-04 07:20:27309 return nil
310 }))
Janardhan Reddye0c226b2017-07-24 07:42:09311 }
312 for i := 5; i < 10; i++ {
313 key := []byte(fmt.Sprintf("key%d", i))
Manish R Jain83aa09d2017-10-04 07:20:27314 require.NoError(t, kv.View(func(txn *Txn) error {
315 item, err := txn.Get(key)
316 require.NoError(t, err)
Manish R Jaincfd203d2017-10-05 02:50:34317 val := getItemValue(t, item)
Manish R Jain83aa09d2017-10-04 07:20:27318 require.NotNil(t, val)
319 require.Equal(t, string(val), fmt.Sprintf("value%d", i))
320 return nil
321 }))
Janardhan Reddye0c226b2017-07-24 07:42:09322 }
Manish R Jaine3a0d292020-10-07 01:41:41323 // Moved entries.
Janardhan Reddye0c226b2017-07-24 07:42:09324 for i := 10; i < 100; i++ {
325 key := []byte(fmt.Sprintf("key%d", i))
Manish R Jain83aa09d2017-10-04 07:20:27326 require.NoError(t, kv.View(func(txn *Txn) error {
327 item, err := txn.Get(key)
328 require.NoError(t, err)
Manish R Jaincfd203d2017-10-05 02:50:34329 val := getItemValue(t, item)
Manish R Jain83aa09d2017-10-04 07:20:27330 require.NotNil(t, val)
331 require.True(t, len(val) == sz, "Size found: %d", len(val))
332 return nil
333 }))
Janardhan Reddye0c226b2017-07-24 07:42:09334 }
335}
336
Sam Hughes9ed12b92017-09-14 05:15:36337func TestValueGC3(t *testing.T) {
Ashish Goswami8c85c272019-06-19 08:21:22338 dir, err := ioutil.TempDir("", "badger-test")
Sam Hughes9ed12b92017-09-14 05:15:36339 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:32340 defer removeDir(dir)
Sam Hughes9ed12b92017-09-14 05:15:36341 opt := getTestOptions(dir)
342 opt.ValueLogFileSize = 1 << 20
Ibrahim Jarif6c35ad62021-02-09 20:11:11343 opt.BaseTableSize = 1 << 15
344 opt.ValueThreshold = 1 << 10
Sam Hughes9ed12b92017-09-14 05:15:36345
Manish R Jainabaad902017-10-04 10:55:56346 kv, err := Open(opt)
Sam Hughes9ed12b92017-09-14 05:15:36347 require.NoError(t, err)
348 defer kv.Close()
349
350 // We want to test whether an iterator can continue through a value log GC.
351
352 valueSize := 32 << 10
353
354 var value3 []byte
Manish R Jaina6860cb2017-10-02 07:32:11355 txn := kv.NewTransaction(true)
Sam Hughes9ed12b92017-09-14 05:15:36356 for i := 0; i < 100; i++ {
357 v := make([]byte, valueSize) // 32K * 100 will take >=3'276'800 B.
358 if i == 3 {
359 value3 = v
360 }
361 rand.Read(v[:])
362 // Keys key000, key001, key002, such that sorted order matches insertion order
Ashish Goswamie9447c92019-05-28 05:28:37363 require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%03d", i)), v)))
Janardhan Reddyffaaa662017-10-02 06:47:46364 if i%20 == 0 {
Manish R Jain6daccf92018-10-09 01:05:00365 require.NoError(t, txn.Commit())
Manish R Jaina6860cb2017-10-02 07:32:11366 txn = kv.NewTransaction(true)
Sam Hughes9ed12b92017-09-14 05:15:36367 }
Sam Hughes9ed12b92017-09-14 05:15:36368 }
Manish R Jain6daccf92018-10-09 01:05:00369 require.NoError(t, txn.Commit())
Sam Hughes9ed12b92017-09-14 05:15:36370
371 // Start an iterator to keys in the first value log file
372 itOpt := IteratorOptions{
373 PrefetchValues: false,
374 PrefetchSize: 0,
375 Reverse: false,
376 }
377
Manish R Jaina6860cb2017-10-02 07:32:11378 txn = kv.NewTransaction(true)
Janardhan Reddyffaaa662017-10-02 06:47:46379 it := txn.NewIterator(itOpt)
Sam Hughes9ed12b92017-09-14 05:15:36380 defer it.Close()
381 // Walk a few keys
382 it.Rewind()
383 require.True(t, it.Valid())
384 item := it.Item()
385 require.Equal(t, []byte("key000"), item.Key())
386 it.Next()
387 require.True(t, it.Valid())
388 item = it.Item()
389 require.Equal(t, []byte("key001"), item.Key())
390 it.Next()
391 require.True(t, it.Valid())
392 item = it.Item()
393 require.Equal(t, []byte("key002"), item.Key())
394
395 // Like other tests, we pull out a logFile to rewrite it directly
396
397 kv.vlog.filesLock.RLock()
398 logFile := kv.vlog.filesMap[kv.vlog.sortedFids()[0]]
399 kv.vlog.filesLock.RUnlock()
400
Manish R Jaine3a0d292020-10-07 01:41:41401 kv.vlog.rewrite(logFile)
Sam Hughes9ed12b92017-09-14 05:15:36402 it.Next()
403 require.True(t, it.Valid())
404 item = it.Item()
405 require.Equal(t, []byte("key003"), item.Key())
Manish R Jain83aa09d2017-10-04 07:20:27406
Manish R Jainc10276c2018-09-25 04:13:48407 v3, err := item.ValueCopy(nil)
Manish R Jain83aa09d2017-10-04 07:20:27408 require.NoError(t, err)
Sam Hughes9ed12b92017-09-14 05:15:36409 require.Equal(t, value3, v3)
410}
411
Janardhan Reddy9b31d1a2017-10-03 23:53:02412func TestValueGC4(t *testing.T) {
Ashish Goswami8c85c272019-06-19 08:21:22413 dir, err := ioutil.TempDir("", "badger-test")
Janardhan Reddy9b31d1a2017-10-03 23:53:02414 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:32415 defer removeDir(dir)
Janardhan Reddy9b31d1a2017-10-03 23:53:02416 opt := getTestOptions(dir)
417 opt.ValueLogFileSize = 1 << 20
Ibrahim Jarif6c35ad62021-02-09 20:11:11418 opt.BaseTableSize = 1 << 15
419 opt.ValueThreshold = 1 << 10
Janardhan Reddy9b31d1a2017-10-03 23:53:02420
Manish R Jain1d37a102018-10-25 22:11:04421 kv, err := Open(opt)
422 require.NoError(t, err)
Janardhan Reddy9b31d1a2017-10-03 23:53:02423
424 sz := 128 << 10 // 5 entries per value log file.
425 txn := kv.NewTransaction(true)
426 for i := 0; i < 24; i++ {
427 v := make([]byte, sz)
428 rand.Read(v[:rand.Intn(sz)])
Ashish Goswamie9447c92019-05-28 05:28:37429 require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v)))
Janardhan Reddy9b31d1a2017-10-03 23:53:02430 if i%3 == 0 {
Manish R Jain6daccf92018-10-09 01:05:00431 require.NoError(t, txn.Commit())
Janardhan Reddy9b31d1a2017-10-03 23:53:02432 txn = kv.NewTransaction(true)
433 }
434 }
Manish R Jain6daccf92018-10-09 01:05:00435 require.NoError(t, txn.Commit())
Janardhan Reddy9b31d1a2017-10-03 23:53:02436
437 for i := 0; i < 8; i++ {
438 txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i)))
439 }
440
441 for i := 8; i < 16; i++ {
442 v := []byte(fmt.Sprintf("value%d", i))
443 txnSet(t, kv, []byte(fmt.Sprintf("key%d", i)), v, 0)
444 }
445
446 kv.vlog.filesLock.RLock()
447 lf0 := kv.vlog.filesMap[kv.vlog.sortedFids()[0]]
448 lf1 := kv.vlog.filesMap[kv.vlog.sortedFids()[1]]
449 kv.vlog.filesLock.RUnlock()
450
451 // lf.iterate(0, func(e Entry) bool {
452 // e.print("lf")
453 // return true
454 // })
455
Manish R Jaine3a0d292020-10-07 01:41:41456 kv.vlog.rewrite(lf0)
457 kv.vlog.rewrite(lf1)
Janardhan Reddy9b31d1a2017-10-03 23:53:02458
Ibrahim Jarif509de732020-06-30 15:30:38459 require.NoError(t, kv.Close())
Martin Martinez Rivera1dd99e42019-01-03 18:06:36460
Ibrahim Jarif509de732020-06-30 15:30:38461 kv, err = Open(opt)
Manish R Jain1d37a102018-10-25 22:11:04462 require.NoError(t, err)
Janardhan Reddy9b31d1a2017-10-03 23:53:02463
464 for i := 0; i < 8; i++ {
465 key := []byte(fmt.Sprintf("key%d", i))
Manish R Jain83aa09d2017-10-04 07:20:27466 require.NoError(t, kv.View(func(txn *Txn) error {
467 _, err := txn.Get(key)
468 require.Equal(t, ErrKeyNotFound, err)
469 return nil
470 }))
Janardhan Reddy9b31d1a2017-10-03 23:53:02471 }
472 for i := 8; i < 16; i++ {
473 key := []byte(fmt.Sprintf("key%d", i))
Manish R Jain83aa09d2017-10-04 07:20:27474 require.NoError(t, kv.View(func(txn *Txn) error {
475 item, err := txn.Get(key)
476 require.NoError(t, err)
Manish R Jaincfd203d2017-10-05 02:50:34477 val := getItemValue(t, item)
Manish R Jain83aa09d2017-10-04 07:20:27478 require.NotNil(t, val)
479 require.Equal(t, string(val), fmt.Sprintf("value%d", i))
480 return nil
481 }))
Janardhan Reddy9b31d1a2017-10-03 23:53:02482 }
Ibrahim Jarif509de732020-06-30 15:30:38483 require.NoError(t, kv.Close())
Janardhan Reddy9b31d1a2017-10-03 23:53:02484}
485
ashishf7bbdb82019-03-19 15:27:08486func TestPersistLFDiscardStats(t *testing.T) {
Ashish Goswami8c85c272019-06-19 08:21:22487 dir, err := ioutil.TempDir("", "badger-test")
ashishf7bbdb82019-03-19 15:27:08488 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:32489 defer removeDir(dir)
ashishf7bbdb82019-03-19 15:27:08490 opt := getTestOptions(dir)
Ibrahim Jarif0506f782020-11-02 08:01:52491 // Force more compaction by reducing the number of L0 tables.
492 opt.NumLevelZeroTables = 1
ashishf7bbdb82019-03-19 15:27:08493 opt.ValueLogFileSize = 1 << 20
Ibrahim Jarif0506f782020-11-02 08:01:52494 // Avoid compaction on close so that the discard map remains the same.
Manish R Jain22378322019-05-12 07:24:21495 opt.CompactL0OnClose = false
Ibrahim Jarif6c35ad62021-02-09 20:11:11496 opt.MemTableSize = 1 << 15
497 opt.ValueThreshold = 1 << 10
ashishf7bbdb82019-03-19 15:27:08498
499 db, err := Open(opt)
500 require.NoError(t, err)
501
502 sz := 128 << 10 // 5 entries per value log file.
503 v := make([]byte, sz)
504 rand.Read(v[:rand.Intn(sz)])
505 txn := db.NewTransaction(true)
506 for i := 0; i < 500; i++ {
Ashish Goswamie9447c92019-05-28 05:28:37507 require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v)))
ashishf7bbdb82019-03-19 15:27:08508 if i%3 == 0 {
509 require.NoError(t, txn.Commit())
510 txn = db.NewTransaction(true)
511 }
512 }
Martin Martinez Riveraa712a3f2019-05-30 18:14:00513 require.NoError(t, txn.Commit(), "error while committing txn")
ashishf7bbdb82019-03-19 15:27:08514
515 for i := 0; i < 500; i++ {
Ashish Goswamicd5884e2019-06-04 13:15:39516 // use Entry.WithDiscard() to delete entries, because this causes data to be flushed on
517 // disk, creating SSTs. Simple Delete was having data in Memtables only.
Manish R Jain22378322019-05-12 07:24:21518 err = db.Update(func(txn *Txn) error {
Ashish Goswamie9447c92019-05-28 05:28:37519 return txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v).WithDiscard())
Manish R Jain22378322019-05-12 07:24:21520 })
521 require.NoError(t, err)
ashishf7bbdb82019-03-19 15:27:08522 }
523
Ibrahim Jarif675efcd2020-06-24 15:16:12524 time.Sleep(2 * time.Second) // wait for compaction to complete
ashishf7bbdb82019-03-19 15:27:08525
Manish R Jaine3a0d292020-10-07 01:41:41526 persistedMap := make(map[uint64]uint64)
527 db.vlog.discardStats.Lock()
528 require.True(t, db.vlog.discardStats.Len() > 1, "some discardStats should be generated")
Manish R Jaina78c0862021-02-05 02:38:20529 db.vlog.discardStats.Iterate(func(fid, val uint64) {
Manish R Jaine3a0d292020-10-07 01:41:41530 persistedMap[fid] = val
531 })
Ashish Goswamic1cf0d72019-10-16 10:38:10532
Manish R Jaine3a0d292020-10-07 01:41:41533 require.NoError(t, db.Close())
ashishf7bbdb82019-03-19 15:27:08534
Ibrahim Jarif0b8eb4c2020-08-19 13:12:06535 // Avoid running compactors on reopening badger.
536 opt.NumCompactors = 0
ashishf7bbdb82019-03-19 15:27:08537 db, err = Open(opt)
538 require.NoError(t, err)
539 defer db.Close()
Ashish Goswamic1cf0d72019-10-16 10:38:10540 time.Sleep(1 * time.Second) // Wait for discardStats to be populated by populateDiscardStats().
Manish R Jaine3a0d292020-10-07 01:41:41541 db.vlog.discardStats.Lock()
542 statsMap := make(map[uint64]uint64)
Manish R Jaina78c0862021-02-05 02:38:20543 db.vlog.discardStats.Iterate(func(fid, val uint64) {
Manish R Jaine3a0d292020-10-07 01:41:41544 statsMap[fid] = val
545 })
546 require.True(t, reflect.DeepEqual(persistedMap, statsMap), "Discard maps are not equal")
547 db.vlog.discardStats.Unlock()
ashishf7bbdb82019-03-19 15:27:08548}
549
Manish R Jaine3a0d292020-10-07 01:41:41550func TestValueChecksums(t *testing.T) {
Ashish Goswami8c85c272019-06-19 08:21:22551 dir, err := ioutil.TempDir("", "badger-test")
Peter Stace766bf272017-08-08 03:40:34552 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:32553 defer removeDir(dir)
Peter Stace766bf272017-08-08 03:40:34554
555 // Set up SST with K1=V1
Manish R Jain273f40c2017-09-21 05:44:36556 opts := getTestOptions(dir)
557 opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb
Manish R Jaine3a0d292020-10-07 01:41:41558 opts.VerifyValueChecksum = true
Manish R Jainabaad902017-10-04 10:55:56559 kv, err := Open(opts)
Peter Stace766bf272017-08-08 03:40:34560 require.NoError(t, err)
Janardhan Reddyffaaa662017-10-02 06:47:46561 require.NoError(t, kv.Close())
Sam Hughes8de624f2017-09-20 23:30:16562
563 var (
Janardhan Reddyffaaa662017-10-02 06:47:46564 k0 = []byte("k0")
Sam Hughes8de624f2017-09-20 23:30:16565 k1 = []byte("k1")
566 k2 = []byte("k2")
567 k3 = []byte("k3")
Manish R Jain7af00762018-05-08 23:42:00568 v0 = []byte("value0-012345678901234567890123012345678901234567890123")
569 v1 = []byte("value1-012345678901234567890123012345678901234567890123")
570 v2 = []byte("value2-012345678901234567890123012345678901234567890123")
571 v3 = []byte("value3-012345678901234567890123012345678901234567890123")
Sam Hughes8de624f2017-09-20 23:30:16572 )
Sam Hughes8de624f2017-09-20 23:30:16573
Janardhan Reddyffaaa662017-10-02 06:47:46574 // Use a vlog with K0=V0 and a (corrupted) second transaction(k1,k2)
Manish R Jaine3a0d292020-10-07 01:41:41575 buf, offset := createMemFile(t, []*Entry{
Janardhan Reddyffaaa662017-10-02 06:47:46576 {Key: k0, Value: v0},
Deepak Jois3e999872017-09-07 08:11:00577 {Key: k1, Value: v1},
578 {Key: k2, Value: v2},
Peter Stace766bf272017-08-08 03:40:34579 })
Manish R Jaine3a0d292020-10-07 01:41:41580 buf[offset-1]++ // Corrupt last byte
581 require.NoError(t, ioutil.WriteFile(kv.mtFilePath(1), buf, 0777))
Peter Stace766bf272017-08-08 03:40:34582
583 // K1 should exist, but K2 shouldn't.
Manish R Jainabaad902017-10-04 10:55:56584 kv, err = Open(opts)
Peter Stace766bf272017-08-08 03:40:34585 require.NoError(t, err)
Manish R Jain83aa09d2017-10-04 07:20:27586
587 require.NoError(t, kv.View(func(txn *Txn) error {
Manish R Jaine3a0d292020-10-07 01:41:41588 // Replay should have added K0.
Manish R Jain83aa09d2017-10-04 07:20:27589 item, err := txn.Get(k0)
590 require.NoError(t, err)
Manish R Jaincfd203d2017-10-05 02:50:34591 require.Equal(t, getItemValue(t, item), v0)
Manish R Jain83aa09d2017-10-04 07:20:27592
593 _, err = txn.Get(k1)
Manish R Jainc3ac2ff2018-04-30 15:43:55594 require.Equal(t, ErrKeyNotFound, err)
Manish R Jain83aa09d2017-10-04 07:20:27595
596 _, err = txn.Get(k2)
Manish R Jainc3ac2ff2018-04-30 15:43:55597 require.Equal(t, ErrKeyNotFound, err)
Manish R Jain83aa09d2017-10-04 07:20:27598 return nil
599 }))
600
Peter Stace766bf272017-08-08 03:40:34601 // Write K3 at the end of the vlog.
Janardhan Reddyffaaa662017-10-02 06:47:46602 txnSet(t, kv, k3, v3, 0)
Peter Stace766bf272017-08-08 03:40:34603 require.NoError(t, kv.Close())
604
Manish R Jaine3a0d292020-10-07 01:41:41605 // The DB should contain K0 and K3 (K1 and k2 was lost when Badger started up
Peter Stace766bf272017-08-08 03:40:34606 // last due to checksum failure).
Manish R Jainabaad902017-10-04 10:55:56607 kv, err = Open(opts)
Peter Stace766bf272017-08-08 03:40:34608 require.NoError(t, err)
Manish R Jain83aa09d2017-10-04 07:20:27609
610 {
611 txn := kv.NewTransaction(false)
612
613 iter := txn.NewIterator(DefaultIteratorOptions)
614 iter.Seek(k0)
615 require.True(t, iter.Valid())
616 it := iter.Item()
617 require.Equal(t, it.Key(), k0)
618 require.Equal(t, getItemValue(t, it), v0)
619 iter.Next()
620 require.True(t, iter.Valid())
621 it = iter.Item()
622 require.Equal(t, it.Key(), k3)
623 require.Equal(t, getItemValue(t, it), v3)
624
625 iter.Close()
626 txn.Discard()
627 }
628
Peter Stace766bf272017-08-08 03:40:34629 require.NoError(t, kv.Close())
630}
631
Manish R Jaine3a0d292020-10-07 01:41:41632// TODO: Do we need this test?
633func TestPartialAppendToWAL(t *testing.T) {
Ashish Goswami8c85c272019-06-19 08:21:22634 dir, err := ioutil.TempDir("", "badger-test")
Peter Stace766bf272017-08-08 03:40:34635 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:32636 defer removeDir(dir)
Peter Stace766bf272017-08-08 03:40:34637
638 // Create skeleton files.
Manish R Jain273f40c2017-09-21 05:44:36639 opts := getTestOptions(dir)
640 opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb
Ibrahim Jarif675efcd2020-06-24 15:16:12641 opts.ValueThreshold = 32
Manish R Jainabaad902017-10-04 10:55:56642 kv, err := Open(opts)
Peter Stace766bf272017-08-08 03:40:34643 require.NoError(t, err)
644 require.NoError(t, kv.Close())
645
Sam Hughes8de624f2017-09-20 23:30:16646 var (
Janardhan Reddyffaaa662017-10-02 06:47:46647 k0 = []byte("k0")
Sam Hughes8de624f2017-09-20 23:30:16648 k1 = []byte("k1")
649 k2 = []byte("k2")
650 k3 = []byte("k3")
Manish R Jain7af00762018-05-08 23:42:00651 v0 = []byte("value0-01234567890123456789012012345678901234567890123")
652 v1 = []byte("value1-01234567890123456789012012345678901234567890123")
653 v2 = []byte("value2-01234567890123456789012012345678901234567890123")
654 v3 = []byte("value3-01234567890123456789012012345678901234567890123")
Sam Hughes8de624f2017-09-20 23:30:16655 )
656 // Values need to be long enough to actually get written to value log.
aman bansal5dbae102021-02-04 12:59:51657 require.True(t, int64(len(v3)) >= kv.vlog.db.valueThreshold())
Sam Hughes8de624f2017-09-20 23:30:16658
Peter Stace766bf272017-08-08 03:40:34659 // Create truncated vlog to simulate a partial append.
Janardhan Reddyffaaa662017-10-02 06:47:46660 // k0 - single transaction, k1 and k2 in another transaction
Manish R Jaine3a0d292020-10-07 01:41:41661 buf, offset := createMemFile(t, []*Entry{
Janardhan Reddyffaaa662017-10-02 06:47:46662 {Key: k0, Value: v0},
Deepak Jois3e999872017-09-07 08:11:00663 {Key: k1, Value: v1},
664 {Key: k2, Value: v2},
Peter Stace766bf272017-08-08 03:40:34665 })
Manish R Jaine3a0d292020-10-07 01:41:41666 buf = buf[:offset-6]
667 require.NoError(t, ioutil.WriteFile(kv.mtFilePath(1), buf, 0777))
Peter Stace766bf272017-08-08 03:40:34668
Janardhan Reddyffaaa662017-10-02 06:47:46669 // Badger should now start up
Manish R Jainabaad902017-10-04 10:55:56670 kv, err = Open(opts)
Peter Stace766bf272017-08-08 03:40:34671 require.NoError(t, err)
Manish R Jain83aa09d2017-10-04 07:20:27672
673 require.NoError(t, kv.View(func(txn *Txn) error {
674 item, err := txn.Get(k0)
675 require.NoError(t, err)
Manish R Jaincfd203d2017-10-05 02:50:34676 require.Equal(t, v0, getItemValue(t, item))
Manish R Jain83aa09d2017-10-04 07:20:27677
678 _, err = txn.Get(k1)
Janardhan Reddy101a2dc2018-03-01 04:01:15679 require.Equal(t, ErrKeyNotFound, err)
Manish R Jain83aa09d2017-10-04 07:20:27680 _, err = txn.Get(k2)
Janardhan Reddy101a2dc2018-03-01 04:01:15681 require.Equal(t, ErrKeyNotFound, err)
Manish R Jain83aa09d2017-10-04 07:20:27682 return nil
683 }))
Peter Stace766bf272017-08-08 03:40:34684
685 // When K3 is set, it should be persisted after a restart.
Janardhan Reddyffaaa662017-10-02 06:47:46686 txnSet(t, kv, k3, v3, 0)
Peter Stace766bf272017-08-08 03:40:34687 require.NoError(t, kv.Close())
Martin Martinez Riveraf937f212019-01-10 19:23:42688 kv, err = Open(opts)
Peter Stace766bf272017-08-08 03:40:34689 require.NoError(t, err)
Janardhan Reddyffaaa662017-10-02 06:47:46690 checkKeys(t, kv, [][]byte{k3})
Janardhan Reddy101a2dc2018-03-01 04:01:15691 // Replay value log from beginning, badger head is past k2.
Martin Martinez Riveraf937f212019-01-10 19:23:42692 require.NoError(t, kv.vlog.Close())
Peter Stace766bf272017-08-08 03:40:34693}
694
Manish R Jaine3a0d292020-10-07 01:41:41695func TestReadOnlyOpenWithPartialAppendToWAL(t *testing.T) {
Ashish Goswami8c85c272019-06-19 08:21:22696 dir, err := ioutil.TempDir("", "badger-test")
Allen Lucedc0df252018-03-08 03:45:33697 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:32698 defer removeDir(dir)
Allen Lucedc0df252018-03-08 03:45:33699
700 // Create skeleton files.
701 opts := getTestOptions(dir)
702 opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb
703 kv, err := Open(opts)
704 require.NoError(t, err)
705 require.NoError(t, kv.Close())
706
707 var (
708 k0 = []byte("k0")
709 k1 = []byte("k1")
710 k2 = []byte("k2")
711 v0 = []byte("value0-012345678901234567890123")
712 v1 = []byte("value1-012345678901234567890123")
713 v2 = []byte("value2-012345678901234567890123")
714 )
715
716 // Create truncated vlog to simulate a partial append.
717 // k0 - single transaction, k1 and k2 in another transaction
Manish R Jaine3a0d292020-10-07 01:41:41718 buf, offset := createMemFile(t, []*Entry{
Allen Lucedc0df252018-03-08 03:45:33719 {Key: k0, Value: v0},
720 {Key: k1, Value: v1},
721 {Key: k2, Value: v2},
722 })
Manish R Jaine3a0d292020-10-07 01:41:41723 buf = buf[:offset-6]
Naman Jain16ec6332020-11-03 08:33:52724 require.NoError(t, ioutil.WriteFile(kv.mtFilePath(1), buf, 0777))
Allen Lucedc0df252018-03-08 03:45:33725
726 opts.ReadOnly = true
727 // Badger should fail a read-only open with values to replay
Martin Martinez Rivera50ddf212019-09-30 17:10:22728 _, err = Open(opts)
Allen Lucedc0df252018-03-08 03:45:33729 require.Error(t, err)
Manish R Jaine3a0d292020-10-07 01:41:41730 require.Regexp(t, "Log truncate required", err.Error())
Allen Lucedc0df252018-03-08 03:45:33731}
732
Manish R Jain935ef5d2017-09-15 03:20:15733func TestValueLogTrigger(t *testing.T) {
Deepak Jois37c2a902017-10-18 04:55:55734 t.Skip("Difficult to trigger compaction, so skipping. Re-enable after fixing #226")
Ashish Goswami8c85c272019-06-19 08:21:22735 dir, err := ioutil.TempDir("", "badger-test")
Manish R Jain935ef5d2017-09-15 03:20:15736 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:32737 defer removeDir(dir)
Manish R Jain935ef5d2017-09-15 03:20:15738
739 opt := getTestOptions(dir)
740 opt.ValueLogFileSize = 1 << 20
Manish R Jainabaad902017-10-04 10:55:56741 kv, err := Open(opt)
Manish R Jain935ef5d2017-09-15 03:20:15742 require.NoError(t, err)
743
744 // Write a lot of data, so it creates some work for valug log GC.
745 sz := 32 << 10
Manish R Jaina6860cb2017-10-02 07:32:11746 txn := kv.NewTransaction(true)
Manish R Jain935ef5d2017-09-15 03:20:15747 for i := 0; i < 100; i++ {
748 v := make([]byte, sz)
749 rand.Read(v[:rand.Intn(sz)])
Ashish Goswamie9447c92019-05-28 05:28:37750 require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v)))
Janardhan Reddyffaaa662017-10-02 06:47:46751 if i%20 == 0 {
Manish R Jain6daccf92018-10-09 01:05:00752 require.NoError(t, txn.Commit())
Manish R Jaina6860cb2017-10-02 07:32:11753 txn = kv.NewTransaction(true)
Janardhan Reddyffaaa662017-10-02 06:47:46754 }
Manish R Jain935ef5d2017-09-15 03:20:15755 }
Manish R Jain6daccf92018-10-09 01:05:00756 require.NoError(t, txn.Commit())
Manish R Jain935ef5d2017-09-15 03:20:15757
758 for i := 0; i < 45; i++ {
Janardhan Reddyffaaa662017-10-02 06:47:46759 txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i)))
Manish R Jain935ef5d2017-09-15 03:20:15760 }
761
Deepak Jois37c2a902017-10-18 04:55:55762 require.NoError(t, kv.RunValueLogGC(0.5))
763
Manish R Jain935ef5d2017-09-15 03:20:15764 require.NoError(t, kv.Close())
765
766 err = kv.RunValueLogGC(0.5)
Manish R Jainabaad902017-10-04 10:55:56767 require.Equal(t, ErrRejected, err, "Error should be returned after closing DB.")
Manish R Jain935ef5d2017-09-15 03:20:15768}
769
Manish R Jaine3a0d292020-10-07 01:41:41770// createMemFile creates a new memFile and returns the last valid offset.
771func createMemFile(t *testing.T, entries []*Entry) ([]byte, uint32) {
Ashish Goswami8c85c272019-06-19 08:21:22772 dir, err := ioutil.TempDir("", "badger-test")
Peter Stace766bf272017-08-08 03:40:34773 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:32774 defer removeDir(dir)
Peter Stace766bf272017-08-08 03:40:34775
Manish R Jain273f40c2017-09-21 05:44:36776 opts := getTestOptions(dir)
777 opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb
Manish R Jainabaad902017-10-04 10:55:56778 kv, err := Open(opts)
Peter Stace766bf272017-08-08 03:40:34779 require.NoError(t, err)
Manish R Jaine3a0d292020-10-07 01:41:41780 defer kv.Close()
781
Deepak Joisa5499e52017-11-02 02:03:14782 txnSet(t, kv, entries[0].Key, entries[0].Value, entries[0].meta)
Manish R Jaine3a0d292020-10-07 01:41:41783
Janardhan Reddyffaaa662017-10-02 06:47:46784 entries = entries[1:]
Manish R Jaina6860cb2017-10-02 07:32:11785 txn := kv.NewTransaction(true)
Janardhan Reddyffaaa662017-10-02 06:47:46786 for _, entry := range entries {
Ashish Goswamie9447c92019-05-28 05:28:37787 require.NoError(t, txn.SetEntry(NewEntry(entry.Key, entry.Value).WithMeta(entry.meta)))
Janardhan Reddyffaaa662017-10-02 06:47:46788 }
Manish R Jain6daccf92018-10-09 01:05:00789 require.NoError(t, txn.Commit())
Peter Stace766bf272017-08-08 03:40:34790
Manish R Jaine3a0d292020-10-07 01:41:41791 filename := kv.mtFilePath(1)
Peter Stace766bf272017-08-08 03:40:34792 buf, err := ioutil.ReadFile(filename)
793 require.NoError(t, err)
Manish R Jaine3a0d292020-10-07 01:41:41794 return buf, kv.mt.wal.writeAt
Peter Stace766bf272017-08-08 03:40:34795}
796
Manish R Jaine3a0d292020-10-07 01:41:41797// This test creates two mem files and corrupts the last bit of the first file.
798func TestPenultimateMemCorruption(t *testing.T) {
Ashish Goswami8c85c272019-06-19 08:21:22799 dir, err := ioutil.TempDir("", "badger-test")
Manish R Jain1d37a102018-10-25 22:11:04800 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:32801 defer removeDir(dir)
Manish R Jain1d37a102018-10-25 22:11:04802 opt := getTestOptions(dir)
Manish R Jain1d37a102018-10-25 22:11:04803
804 db0, err := Open(opt)
805 require.NoError(t, err)
Ibrahim Jarifb7628322020-06-09 10:46:19806 defer func() { require.NoError(t, db0.Close()) }()
Manish R Jain1d37a102018-10-25 22:11:04807
808 h := testHelper{db: db0, t: t}
Manish R Jaine3a0d292020-10-07 01:41:41809 h.writeRange(0, 2) // 00001.mem
810
811 // Move the current memtable to the db.imm and create a new memtable so
812 // that we can have more than one mem files.
813 require.Zero(t, len(db0.imm))
814 db0.imm = append(db0.imm, db0.mt)
815 db0.mt, err = db0.newMemTable()
816
817 h.writeRange(3, 7) // 00002.mem
818
819 // Verify we have all the data we wrote.
Manish R Jain1d37a102018-10-25 22:11:04820 h.readRange(0, 7)
821
Manish R Jaine3a0d292020-10-07 01:41:41822 for i := 2; i >= 1; i-- {
823 fpath := db0.mtFilePath(i)
Manish R Jain1d37a102018-10-25 22:11:04824 fi, err := os.Stat(fpath)
825 require.NoError(t, err)
826 require.True(t, fi.Size() > 0, "Empty file at log=%d", i)
Manish R Jaine3a0d292020-10-07 01:41:41827 if i == 1 {
828 // This should corrupt the last entry in the first memtable (that is entry number 2)
829 wal := db0.imm[0].wal
830 wal.Fd.WriteAt([]byte{0}, int64(wal.writeAt-1))
Manish R Jain1d37a102018-10-25 22:11:04831 require.NoError(t, err)
Manish R Jaine3a0d292020-10-07 01:41:41832 // We have corrupted the file. We can remove it. If we don't remove
833 // the imm here, the db.close in defer will crash since db0.mt !=
834 // db0.imm[0]
835 db0.imm = db0.imm[:0]
Manish R Jain1d37a102018-10-25 22:11:04836 }
837 }
838 // Simulate a crash by not closing db0, but releasing the locks.
839 if db0.dirLockGuard != nil {
840 require.NoError(t, db0.dirLockGuard.release())
Ibrahim Jarifb7628322020-06-09 10:46:19841 db0.dirLockGuard = nil
Manish R Jain1d37a102018-10-25 22:11:04842 }
843 if db0.valueDirGuard != nil {
844 require.NoError(t, db0.valueDirGuard.release())
Ibrahim Jarifb7628322020-06-09 10:46:19845 db0.valueDirGuard = nil
Manish R Jain1d37a102018-10-25 22:11:04846 }
847
Manish R Jain1d37a102018-10-25 22:11:04848 db1, err := Open(opt)
849 require.NoError(t, err)
850 h.db = db1
Manish R Jaine3a0d292020-10-07 01:41:41851 // Only 2 should be gone because it is at the end of 0001.mem (first memfile).
852 h.readRange(0, 1)
Manish R Jain1d37a102018-10-25 22:11:04853 h.readRange(3, 7)
854 err = db1.View(func(txn *Txn) error {
855 _, err := txn.Get(h.key(2)) // Verify that 2 is gone.
856 require.Equal(t, ErrKeyNotFound, err)
857 return nil
858 })
859 require.NoError(t, err)
860 require.NoError(t, db1.Close())
861}
862
Manish R Jainabaad902017-10-04 10:55:56863func checkKeys(t *testing.T, kv *DB, keys [][]byte) {
Peter Stace766bf272017-08-08 03:40:34864 i := 0
Manish R Jaina6860cb2017-10-02 07:32:11865 txn := kv.NewTransaction(false)
Ibrahim Jarif0f2e6292020-01-07 12:34:02866 defer txn.Discard()
Janardhan Reddyffaaa662017-10-02 06:47:46867 iter := txn.NewIterator(IteratorOptions{})
Ibrahim Jarif0f2e6292020-01-07 12:34:02868 defer iter.Close()
Peter Stace766bf272017-08-08 03:40:34869 for iter.Seek(keys[0]); iter.Valid(); iter.Next() {
870 require.Equal(t, iter.Item().Key(), keys[i])
871 i++
872 }
873 require.Equal(t, i, len(keys))
874}
875
Manish R Jain1d37a102018-10-25 22:11:04876type testHelper struct {
877 db *DB
878 t *testing.T
879 val []byte
880}
881
882func (th *testHelper) key(i int) []byte {
883 return []byte(fmt.Sprintf("%010d", i))
884}
885func (th *testHelper) value() []byte {
886 if len(th.val) > 0 {
887 return th.val
888 }
889 th.val = make([]byte, 100)
890 y.Check2(rand.Read(th.val))
891 return th.val
892}
893
894// writeRange [from, to].
895func (th *testHelper) writeRange(from, to int) {
896 for i := from; i <= to; i++ {
897 err := th.db.Update(func(txn *Txn) error {
Ashish Goswamie9447c92019-05-28 05:28:37898 return txn.SetEntry(NewEntry(th.key(i), th.value()))
Manish R Jain1d37a102018-10-25 22:11:04899 })
900 require.NoError(th.t, err)
901 }
902}
903
904func (th *testHelper) readRange(from, to int) {
905 for i := from; i <= to; i++ {
906 err := th.db.View(func(txn *Txn) error {
907 item, err := txn.Get(th.key(i))
908 if err != nil {
909 return err
910 }
Martin Martinez Riveraf7963032019-01-04 01:53:25911 return item.Value(func(val []byte) error {
Manish R Jain1d37a102018-10-25 22:11:04912 require.Equal(th.t, val, th.value(), "key=%q", th.key(i))
913 return nil
914
Martin Martinez Riveraf7963032019-01-04 01:53:25915 })
Manish R Jain1d37a102018-10-25 22:11:04916 })
Manish R Jain6c8b15c2018-10-27 15:26:35917 require.NoError(th.t, err, "key=%q", th.key(i))
Manish R Jain1d37a102018-10-25 22:11:04918 }
919}
920
Manish R Jainaf99e5f2018-09-23 17:43:37921// Test Bug #578, which showed that if a value is moved during value log GC, an
922// older version can end up at a higher level in the LSM tree than a newer
923// version, causing the data to not be returned.
924func TestBug578(t *testing.T) {
Ashish Goswami8c85c272019-06-19 08:21:22925 dir, err := ioutil.TempDir("", "badger-test")
Manish R Jainaf99e5f2018-09-23 17:43:37926 y.Check(err)
Ibrahim Jarif00039ea2019-10-17 13:02:32927 defer removeDir(dir)
Manish R Jainaf99e5f2018-09-23 17:43:37928
Francesc Campoy2bb87c02019-06-24 21:57:56929 db, err := Open(DefaultOptions(dir).
930 WithValueLogMaxEntries(64).
Manish R Jain45bca182020-10-26 21:12:58931 WithBaseTableSize(1 << 13))
Manish R Jainaf99e5f2018-09-23 17:43:37932 require.NoError(t, err)
933
Manish R Jain1d37a102018-10-25 22:11:04934 h := testHelper{db: db, t: t}
Manish R Jainaf99e5f2018-09-23 17:43:37935
936 // Let's run this whole thing a few times.
937 for j := 0; j < 10; j++ {
938 t.Logf("Cycle: %d\n", j)
Manish R Jain1d37a102018-10-25 22:11:04939 h.writeRange(0, 32)
940 h.writeRange(0, 10)
941 h.writeRange(50, 72)
942 h.writeRange(40, 72)
943 h.writeRange(40, 72)
Manish R Jainaf99e5f2018-09-23 17:43:37944
945 // Run value log GC a few times.
946 for i := 0; i < 5; i++ {
947 db.RunValueLogGC(0.5)
948 }
Manish R Jain1d37a102018-10-25 22:11:04949 h.readRange(0, 10)
Manish R Jainaf99e5f2018-09-23 17:43:37950 }
Ibrahim Jarif3d5fe0b2019-08-08 09:18:25951 require.NoError(t, db.Close())
Manish R Jainaf99e5f2018-09-23 17:43:37952}
953
Manish R Jain61b81192017-01-26 07:17:18954func BenchmarkReadWrite(b *testing.B) {
955 rwRatio := []float32{
956 0.1, 0.2, 0.5, 1.0,
957 }
958 valueSize := []int{
Manish R Jain86ce19b2017-01-27 11:17:28959 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384,
Manish R Jain61b81192017-01-26 07:17:18960 }
961
962 for _, vsz := range valueSize {
963 for _, rw := range rwRatio {
Manish R Jain1fcb50f2017-01-27 09:19:24964 b.Run(fmt.Sprintf("%3.1f,%04d", rw, vsz), func(b *testing.B) {
Manish R Jain1d37a102018-10-25 22:11:04965 dir, err := ioutil.TempDir("", "vlog-benchmark")
Motakjuq4b1e5332017-06-27 09:04:01966 y.Check(err)
Ibrahim Jarif00039ea2019-10-17 13:02:32967 defer removeDir(dir)
Naman Jain292a4be2021-09-14 14:04:35968 opts := getTestOptions(dir)
969 opts.ValueThreshold = 0
970 db, err := Open(opts)
Motakjuq4b1e5332017-06-27 09:04:01971 y.Check(err)
Manish R Jain1d37a102018-10-25 22:11:04972
Martin Martinez Riverac301fa12019-01-04 19:08:16973 vl := &db.vlog
Manish R Jain61b81192017-01-26 07:17:18974 b.ResetTimer()
975
Manish R Jaindadcb4d2017-04-21 10:03:05976 for i := 0; i < b.N; i++ {
Deepak Jois5bc2e162017-11-30 03:04:57977 e := new(Entry)
Manish R Jain61b81192017-01-26 07:17:18978 e.Key = make([]byte, 16)
979 e.Value = make([]byte, vsz)
Manish R Jain9230dd02017-04-29 01:57:00980 bl := new(request)
Deepak Jois5bc2e162017-11-30 03:04:57981 bl.Entries = []*Entry{e}
Manish R Jain61b81192017-01-26 07:17:18982
Manish R Jain22e69902017-04-24 02:02:01983 var ptrs []valuePointer
Manish R Jain61b81192017-01-26 07:17:18984
Manish R Jain20849402017-05-17 08:36:32985 vl.write([]*request{bl})
Manish R Jaindadcb4d2017-04-21 10:03:05986 ptrs = append(ptrs, bl.Ptrs...)
Manish R Jain61b81192017-01-26 07:17:18987
Manish R Jaindadcb4d2017-04-21 10:03:05988 f := rand.Float32()
989 if f < rw {
Manish R Jain20849402017-05-17 08:36:32990 vl.write([]*request{bl})
Manish R Jain61b81192017-01-26 07:17:18991
Manish R Jaindadcb4d2017-04-21 10:03:05992 } else {
993 ln := len(ptrs)
994 if ln == 0 {
995 b.Fatalf("Zero length of ptrs")
996 }
997 idx := rand.Intn(ln)
Naman Jaina35b08d2021-01-14 06:01:03998 buf, lf, err := vl.readValueBytes(ptrs[idx])
Manish R Jaindadcb4d2017-04-21 10:03:05999 if err != nil {
Manish R Jainfb0f0aa2017-05-03 01:17:041000 b.Fatalf("Benchmark Read: %v", err)
Manish R Jaindadcb4d2017-04-21 10:03:051001 }
Manish R Jain83aa09d2017-10-04 07:20:271002
balajia425b0e2019-09-24 14:06:361003 e, err := lf.decodeEntry(buf, ptrs[idx].Offset)
1004 require.NoError(b, err)
Manish R Jain83aa09d2017-10-04 07:20:271005 if len(e.Key) != 16 {
1006 b.Fatalf("Key is invalid")
1007 }
1008 if len(e.Value) != vsz {
1009 b.Fatalf("Value is invalid")
1010 }
balajia425b0e2019-09-24 14:06:361011 runCallback(db.vlog.getUnlockCallback(lf))
Manish R Jain61b81192017-01-26 07:17:181012 }
Manish R Jaindadcb4d2017-04-21 10:03:051013 }
Manish R Jain61b81192017-01-26 07:17:181014 })
1015 }
1016 }
1017}
Ibrahim Jarif22a9a1e2019-05-31 08:05:251018
1019// Regression test for https://github.com/dgraph-io/badger/issues/817
Ibrahim Jarif0506f782020-11-02 08:01:521020// This test verifies if fully corrupted memtables are deleted on reopen.
Ibrahim Jarif22a9a1e2019-05-31 08:05:251021func TestValueLogTruncate(t *testing.T) {
Ashish Goswami8c85c272019-06-19 08:21:221022 dir, err := ioutil.TempDir("", "badger-test")
Ibrahim Jarif22a9a1e2019-05-31 08:05:251023 require.NoError(t, err)
Ibrahim Jarif00039ea2019-10-17 13:02:321024 defer removeDir(dir)
Ibrahim Jarif22a9a1e2019-05-31 08:05:251025
Manish R Jaine3a0d292020-10-07 01:41:411026 // Initialize the data directory.
1027 db, err := Open(DefaultOptions(dir))
Ibrahim Jarif22a9a1e2019-05-31 08:05:251028 require.NoError(t, err)
Manish R Jaine3a0d292020-10-07 01:41:411029 // Insert 1 entry so that we have valid data in first mem file
Ibrahim Jarif22a9a1e2019-05-31 08:05:251030 require.NoError(t, db.Update(func(txn *Txn) error {
1031 return txn.Set([]byte("foo"), nil)
1032 }))
1033
Manish R Jaine3a0d292020-10-07 01:41:411034 fileCountBeforeCorruption := 1
Ibrahim Jarif22a9a1e2019-05-31 08:05:251035 require.NoError(t, db.Close())
1036
Manish R Jaine3a0d292020-10-07 01:41:411037 // Create two mem files with corrupted data. These will be truncated when DB starts next time
1038 require.NoError(t, ioutil.WriteFile(db.mtFilePath(2), []byte("foo"), 0664))
1039 require.NoError(t, ioutil.WriteFile(db.mtFilePath(3), []byte("foo"), 0664))
Ibrahim Jarif22a9a1e2019-05-31 08:05:251040
Manish R Jaine3a0d292020-10-07 01:41:411041 db, err = Open(DefaultOptions(dir))
Ibrahim Jarif22a9a1e2019-05-31 08:05:251042 require.NoError(t, err)
1043
Manish R Jaine3a0d292020-10-07 01:41:411044 // Ensure we have only one SST file.
1045 require.Equal(t, 1, len(db.Tables()))
Ibrahim Jarif22a9a1e2019-05-31 08:05:251046
Manish R Jaine3a0d292020-10-07 01:41:411047 // Ensure mem file with ID 4 is zero.
1048 require.Equal(t, 4, int(db.mt.wal.fid))
1049 fileStat, err := db.mt.wal.Fd.Stat()
Ibrahim Jarif22a9a1e2019-05-31 08:05:251050 require.NoError(t, err)
Manish R Jain741de052020-11-12 23:05:111051 require.Equal(t, 2*db.opt.MemTableSize, fileStat.Size())
Ibrahim Jarif22a9a1e2019-05-31 08:05:251052
Manish R Jaine3a0d292020-10-07 01:41:411053 fileCountAfterCorruption := len(db.Tables()) + len(db.imm) + 1 // +1 for db.mt
1054 // We should have one memtable and one sst file.
Ibrahim Jarif22a9a1e2019-05-31 08:05:251055 require.Equal(t, fileCountBeforeCorruption+1, fileCountAfterCorruption)
Manish R Jaine3a0d292020-10-07 01:41:411056 // maxFid will be 2 because we increment the max fid on DB open everytime.
Ibrahim Jarif22a9a1e2019-05-31 08:05:251057 require.Equal(t, 2, int(db.vlog.maxFid))
1058 require.NoError(t, db.Close())
1059}
Ibrahim Jarifc5bf7e32019-07-18 09:27:211060
Ibrahim Jarifd8e1fcf2019-07-19 05:07:031061func TestSafeEntry(t *testing.T) {
1062 var s safeRead
balajia425b0e2019-09-24 14:06:361063 s.lf = &logFile{}
Ibrahim Jarifd8e1fcf2019-07-19 05:07:031064 e := NewEntry([]byte("foo"), []byte("bar"))
1065 buf := bytes.NewBuffer(nil)
Manish R Jaine3a0d292020-10-07 01:41:411066 _, err := s.lf.encodeEntry(buf, e, 0)
Ibrahim Jarifd8e1fcf2019-07-19 05:07:031067 require.NoError(t, err)
1068
1069 ne, err := s.Entry(buf)
1070 require.NoError(t, err)
1071 require.Equal(t, e.Key, ne.Key, "key mismatch")
1072 require.Equal(t, e.Value, ne.Value, "value mismatch")
1073 require.Equal(t, e.meta, ne.meta, "meta mismatch")
1074 require.Equal(t, e.UserMeta, ne.UserMeta, "usermeta mismatch")
1075 require.Equal(t, e.ExpiresAt, ne.ExpiresAt, "expiresAt mismatch")
1076}
1077
Ibrahim Jarif1ee50f72019-12-06 11:49:031078func TestValueEntryChecksum(t *testing.T) {
Ibrahim Jarif0ef91a82019-10-04 09:30:171079 k := []byte("KEY")
1080 v := []byte(fmt.Sprintf("val%100d", 10))
Ibrahim Jarif1ee50f72019-12-06 11:49:031081 t.Run("ok", func(t *testing.T) {
1082 dir, err := ioutil.TempDir("", "badger-test")
1083 require.NoError(t, err)
1084 defer removeDir(dir)
Ibrahim Jarif0ef91a82019-10-04 09:30:171085
Ibrahim Jarif1ee50f72019-12-06 11:49:031086 opt := getTestOptions(dir)
1087 opt.VerifyValueChecksum = true
Ibrahim Jarif675efcd2020-06-24 15:16:121088 opt.ValueThreshold = 32
Ibrahim Jarif1ee50f72019-12-06 11:49:031089 db, err := Open(opt)
1090 require.NoError(t, err)
Ibrahim Jarif0ef91a82019-10-04 09:30:171091
aman bansal5dbae102021-02-04 12:59:511092 require.Greater(t, int64(len(v)), db.vlog.db.valueThreshold())
Ibrahim Jarif1ee50f72019-12-06 11:49:031093 txnSet(t, db, k, v, 0)
1094 require.NoError(t, db.Close())
Ibrahim Jarif0ef91a82019-10-04 09:30:171095
Ibrahim Jarif1ee50f72019-12-06 11:49:031096 db, err = Open(opt)
1097 require.NoError(t, err)
Ibrahim Jarif0ef91a82019-10-04 09:30:171098
Ibrahim Jarif1ee50f72019-12-06 11:49:031099 txn := db.NewTransaction(false)
1100 entry, err := txn.Get(k)
1101 require.NoError(t, err)
Ibrahim Jarif0ef91a82019-10-04 09:30:171102
Ibrahim Jarif1ee50f72019-12-06 11:49:031103 x, err := entry.ValueCopy(nil)
1104 require.NoError(t, err)
1105 require.Equal(t, v, x)
Ibrahim Jarif0ef91a82019-10-04 09:30:171106
Ibrahim Jarif1ee50f72019-12-06 11:49:031107 require.NoError(t, db.Close())
1108 })
1109 // Regression test for https://github.com/dgraph-io/badger/issues/1049
1110 t.Run("Corruption", func(t *testing.T) {
1111 dir, err := ioutil.TempDir("", "badger-test")
1112 require.NoError(t, err)
1113 defer removeDir(dir)
1114
1115 opt := getTestOptions(dir)
1116 opt.VerifyValueChecksum = true
Ibrahim Jarif675efcd2020-06-24 15:16:121117 opt.ValueThreshold = 32
Ibrahim Jarif1ee50f72019-12-06 11:49:031118 db, err := Open(opt)
1119 require.NoError(t, err)
1120
aman bansal5dbae102021-02-04 12:59:511121 require.Greater(t, int64(len(v)), db.vlog.db.valueThreshold())
Ibrahim Jarif1ee50f72019-12-06 11:49:031122 txnSet(t, db, k, v, 0)
1123
Manish R Jaine3a0d292020-10-07 01:41:411124 path := db.vlog.fpath(1)
Ibrahim Jarif1ee50f72019-12-06 11:49:031125 require.NoError(t, db.Close())
1126
1127 file, err := os.OpenFile(path, os.O_RDWR, 0644)
1128 require.NoError(t, err)
1129 offset := 50
1130 orig := make([]byte, 1)
1131 _, err = file.ReadAt(orig, int64(offset))
1132 require.NoError(t, err)
1133 // Corrupt a single bit.
1134 _, err = file.WriteAt([]byte{7}, int64(offset))
1135 require.NoError(t, err)
1136 require.NoError(t, file.Close())
1137
1138 db, err = Open(opt)
1139 require.NoError(t, err)
1140
1141 txn := db.NewTransaction(false)
1142 entry, err := txn.Get(k)
1143 require.NoError(t, err)
1144
Ibrahim Jariffeb1f5f2020-11-25 13:15:521145 // TODO(ibrahim): This test is broken since we're not returning errors
1146 // in case we cannot read the values. This is incorrect behavior but
1147 // we're doing this to debug an issue where the values are being read
1148 // from old vlog files.
1149 _, _ = entry.ValueCopy(nil)
1150 // require.Error(t, err)
1151 // require.Contains(t, err.Error(), "ErrEOF")
1152 // require.Nil(t, x)
Ibrahim Jarif1ee50f72019-12-06 11:49:031153
1154 require.NoError(t, db.Close())
1155 })
Ibrahim Jarif0ef91a82019-10-04 09:30:171156}
Balaji Jinnahd981f472020-07-10 06:25:021157
1158func TestValidateWrite(t *testing.T) {
1159 // Mocking the file size, so that we don't allocate big memory while running test.
1160 maxVlogFileSize = 400
1161 defer func() {
1162 maxVlogFileSize = math.MaxUint32
1163 }()
1164
1165 bigBuf := make([]byte, maxVlogFileSize+1)
1166 log := &valueLog{
1167 opt: DefaultOptions("."),
1168 }
1169
1170 // Sending a request with big values which will overflow uint32.
1171 key := []byte("HelloKey")
1172 req := &request{
1173 Entries: []*Entry{
1174 {
1175 Key: key,
1176 Value: bigBuf,
1177 },
1178 {
1179 Key: key,
1180 Value: bigBuf,
1181 },
1182 {
1183 Key: key,
1184 Value: bigBuf,
1185 },
1186 },
1187 }
1188
1189 err := log.validateWrites([]*request{req})
1190 require.Error(t, err)
1191
1192 // Testing with small values.
1193 smallBuf := make([]byte, 4)
1194 req1 := &request{
1195 Entries: []*Entry{
1196 {
1197 Key: key,
1198 Value: smallBuf,
1199 },
1200 {
1201 Key: key,
1202 Value: smallBuf,
1203 },
1204 {
1205 Key: key,
1206 Value: smallBuf,
1207 },
1208 },
1209 }
1210
1211 err = log.validateWrites([]*request{req1})
1212 require.NoError(t, err)
1213
1214 // Batching small and big request.
1215 err = log.validateWrites([]*request{req1, req})
1216 require.Error(t, err)
1217}
Naman Jain34f2d042020-10-15 09:24:371218
1219func TestValueLogMeta(t *testing.T) {
1220 dir, err := ioutil.TempDir("", "badger-test")
1221 y.Check(err)
1222 defer removeDir(dir)
1223
1224 opt := getTestOptions(dir).WithValueThreshold(16)
1225 db, _ := Open(opt)
1226 defer db.Close()
1227 txn := db.NewTransaction(true)
1228 for i := 0; i < 10; i++ {
1229 k := []byte(fmt.Sprintf("key=%d", i))
1230 v := []byte(fmt.Sprintf("val=%020d", i))
1231 require.NoError(t, txn.SetEntry(NewEntry(k, v)))
1232 }
1233 require.NoError(t, txn.Commit())
1234 fids := db.vlog.sortedFids()
1235 require.Equal(t, 1, len(fids))
1236
1237 // vlog entries must not have txn meta.
1238 db.vlog.filesMap[fids[0]].iterate(true, 0, func(e Entry, vp valuePointer) error {
1239 require.Zero(t, e.meta&(bitTxn|bitFinTxn))
1240 return nil
1241 })
1242
1243 // Entries in LSM tree must have txn bit of meta set
1244 txn = db.NewTransaction(false)
1245 defer txn.Discard()
1246 iopt := DefaultIteratorOptions
1247 key := []byte("key")
1248 iopt.Prefix = key
1249 itr := txn.NewIterator(iopt)
1250 defer itr.Close()
1251 var count int
1252 for itr.Seek(key); itr.ValidForPrefix(key); itr.Next() {
1253 item := itr.Item()
1254 require.Equal(t, bitTxn, item.meta&(bitTxn|bitFinTxn))
1255 count++
1256 }
1257 require.Equal(t, 10, count)
1258}
1259
1260// This tests asserts the condition that vlog fids start from 1.
1261// TODO(naman): should this be changed to assert instead?
1262func TestFirstVlogFile(t *testing.T) {
1263 dir, err := ioutil.TempDir("", "badger-test")
1264 require.NoError(t, err)
1265 defer removeDir(dir)
1266
1267 opt := DefaultOptions(dir)
1268 db, err := Open(opt)
1269 defer db.Close()
1270
1271 fids := db.vlog.sortedFids()
1272 require.NotZero(t, len(fids))
1273 require.Equal(t, uint32(1), fids[0])
1274}