Manish R Jain | 9b4612b | 2017-03-14 11:30:04 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2017 Dgraph Labs, Inc. and Contributors |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
Manish R Jain | 22e6990 | 2017-04-24 02:02:01 | [diff] [blame] | 17 | package badger |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 18 | |
| 19 | import ( |
Ibrahim Jarif | d8e1fcf | 2019-07-19 05:07:03 | [diff] [blame] | 20 | "bytes" |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 21 | "fmt" |
Ganesh Acharya | 080745a | 2017-04-04 05:15:40 | [diff] [blame] | 22 | "io/ioutil" |
Balaji Jinnah | d981f47 | 2020-07-10 06:25:02 | [diff] [blame] | 23 | "math" |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 24 | "math/rand" |
| 25 | "os" |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 26 | "reflect" |
Manish R Jain | d22c0e8 | 2018-09-19 17:44:01 | [diff] [blame] | 27 | "sync" |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 28 | "testing" |
Manish R Jain | 2237832 | 2019-05-12 07:24:21 | [diff] [blame] | 29 | "time" |
Ganesh Acharya | 080745a | 2017-04-04 05:15:40 | [diff] [blame] | 30 | |
aman bansal | b69163b | 2021-01-13 05:25:37 | [diff] [blame] | 31 | "github.com/dgraph-io/badger/v3/y" |
Manish R Jain | d22c0e8 | 2018-09-19 17:44:01 | [diff] [blame] | 32 | humanize "github.com/dustin/go-humanize" |
Motakjuq | ff18eb0 | 2017-06-27 07:39:15 | [diff] [blame] | 33 | "github.com/stretchr/testify/require" |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 34 | ) |
| 35 | |
aman bansal | 5dbae10 | 2021-02-04 12:59:51 | [diff] [blame] | 36 | func TestDynamicValueThreshold(t *testing.T) { |
Ibrahim Jarif | 6c35ad6 | 2021-02-09 20:11:11 | [diff] [blame] | 37 | t.Skip() |
aman bansal | 5dbae10 | 2021-02-04 12:59:51 | [diff] [blame] | 38 | dir, err := ioutil.TempDir("", "badger-test") |
| 39 | y.Check(err) |
| 40 | defer removeDir(dir) |
| 41 | kv, _ := Open(getTestOptions(dir).WithValueThreshold(32).WithVLogPercentile(0.99)) |
| 42 | defer kv.Close() |
| 43 | log := &kv.vlog |
| 44 | for vl := 32; vl <= 1024; vl = vl + 4 { |
| 45 | for i := 0; i < 1000; i++ { |
| 46 | val := make([]byte, vl) |
| 47 | y.Check2(rand.Read(val)) |
| 48 | e1 := &Entry{ |
| 49 | Key: []byte(fmt.Sprintf("samplekey_%d_%d", vl, i)), |
| 50 | Value: val, |
| 51 | meta: bitValuePointer, |
| 52 | } |
| 53 | b := new(request) |
| 54 | b.Entries = []*Entry{e1} |
| 55 | log.write([]*request{b}) |
| 56 | } |
| 57 | t.Logf("value threshold is %d \n", log.db.valueThreshold()) |
| 58 | } |
| 59 | |
| 60 | for vl := 511; vl >= 31; vl = vl - 4 { |
| 61 | for i := 0; i < 5000; i++ { |
| 62 | val := make([]byte, vl) |
| 63 | y.Check2(rand.Read(val)) |
| 64 | e1 := &Entry{ |
| 65 | Key: []byte(fmt.Sprintf("samplekey_%d_%d", vl, i)), |
| 66 | Value: val, |
| 67 | meta: bitValuePointer, |
| 68 | } |
| 69 | b := new(request) |
| 70 | b.Entries = []*Entry{e1} |
| 71 | log.write([]*request{b}) |
| 72 | } |
| 73 | t.Logf("value threshold is %d \n", log.db.valueThreshold()) |
| 74 | } |
| 75 | require.Equal(t, log.db.valueThreshold(), int64(995)) |
| 76 | } |
| 77 | |
Ganesh Acharya | bd27063 | 2017-04-27 16:31:24 | [diff] [blame] | 78 | func TestValueBasic(t *testing.T) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 79 | dir, err := ioutil.TempDir("", "badger-test") |
Ganesh Acharya | 080745a | 2017-04-04 05:15:40 | [diff] [blame] | 80 | y.Check(err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 81 | defer removeDir(dir) |
Ganesh Acharya | 080745a | 2017-04-04 05:15:40 | [diff] [blame] | 82 | |
Ibrahim Jarif | 675efcd | 2020-06-24 15:16:12 | [diff] [blame] | 83 | kv, _ := Open(getTestOptions(dir).WithValueThreshold(32)) |
Manish R Jain | 4cab233 | 2017-04-24 07:32:15 | [diff] [blame] | 84 | defer kv.Close() |
Manish R Jain | fb0f0aa | 2017-05-03 01:17:04 | [diff] [blame] | 85 | log := &kv.vlog |
Manish R Jain | f454220 | 2017-04-14 01:56:49 | [diff] [blame] | 86 | |
Sam Hughes | 8de624f | 2017-09-20 23:30:16 | [diff] [blame] | 87 | // Use value big enough that the value log writes them even if SyncWrites is false. |
| 88 | const val1 = "sampleval012345678901234567890123" |
| 89 | const val2 = "samplevalb012345678901234567890123" |
aman bansal | 5dbae10 | 2021-02-04 12:59:51 | [diff] [blame] | 90 | require.True(t, int64(len(val1)) >= kv.vlog.db.valueThreshold()) |
Sam Hughes | 8de624f | 2017-09-20 23:30:16 | [diff] [blame] | 91 | |
Ibrahim Jarif | d8e1fcf | 2019-07-19 05:07:03 | [diff] [blame] | 92 | e1 := &Entry{ |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 93 | Key: []byte("samplekey"), |
| 94 | Value: []byte(val1), |
Deepak Jois | a5499e5 | 2017-11-02 02:03:14 | [diff] [blame] | 95 | meta: bitValuePointer, |
Manish R Jain | b61131e | 2017-04-19 09:53:35 | [diff] [blame] | 96 | } |
Deepak Jois | 5bc2e16 | 2017-11-30 03:04:57 | [diff] [blame] | 97 | e2 := &Entry{ |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 98 | Key: []byte("samplekeyb"), |
| 99 | Value: []byte(val2), |
Deepak Jois | a5499e5 | 2017-11-02 02:03:14 | [diff] [blame] | 100 | meta: bitValuePointer, |
Ganesh Acharya | f9e1471 | 2017-04-30 20:39:14 | [diff] [blame] | 101 | } |
| 102 | |
Manish R Jain | 9230dd0 | 2017-04-29 01:57:00 | [diff] [blame] | 103 | b := new(request) |
Ibrahim Jarif | d8e1fcf | 2019-07-19 05:07:03 | [diff] [blame] | 104 | b.Entries = []*Entry{e1, e2} |
Manish R Jain | dadcb4d | 2017-04-21 10:03:05 | [diff] [blame] | 105 | |
Manish R Jain | 2084940 | 2017-05-17 08:36:32 | [diff] [blame] | 106 | log.write([]*request{b}) |
Ganesh Acharya | f9e1471 | 2017-04-30 20:39:14 | [diff] [blame] | 107 | require.Len(t, b.Ptrs, 2) |
Manish R Jain | a6860cb | 2017-10-02 07:32:11 | [diff] [blame] | 108 | t.Logf("Pointer written: %+v %+v\n", b.Ptrs[0], b.Ptrs[1]) |
Manish R Jain | 526b665 | 2017-04-13 08:22:46 | [diff] [blame] | 109 | |
Naman Jain | a35b08d | 2021-01-14 06:01:03 | [diff] [blame] | 110 | buf1, lf1, err1 := log.readValueBytes(b.Ptrs[0]) |
| 111 | buf2, lf2, err2 := log.readValueBytes(b.Ptrs[1]) |
Deepak Jois | b9aae1b | 2017-08-31 05:06:49 | [diff] [blame] | 112 | require.NoError(t, err1) |
| 113 | require.NoError(t, err2) |
balaji | a425b0e | 2019-09-24 14:06:36 | [diff] [blame] | 114 | defer runCallback(log.getUnlockCallback(lf1)) |
| 115 | defer runCallback(log.getUnlockCallback(lf2)) |
| 116 | e1, err = lf1.decodeEntry(buf1, b.Ptrs[0].Offset) |
| 117 | require.NoError(t, err) |
| 118 | e2, err = lf1.decodeEntry(buf2, b.Ptrs[1].Offset) |
| 119 | require.NoError(t, err) |
| 120 | readEntries := []Entry{*e1, *e2} |
Deepak Jois | 5bc2e16 | 2017-11-30 03:04:57 | [diff] [blame] | 121 | require.EqualValues(t, []Entry{ |
Ganesh Acharya | 080745a | 2017-04-04 05:15:40 | [diff] [blame] | 122 | { |
balaji | a425b0e | 2019-09-24 14:06:36 | [diff] [blame] | 123 | Key: []byte("samplekey"), |
| 124 | Value: []byte(val1), |
| 125 | meta: bitValuePointer, |
| 126 | offset: b.Ptrs[0].Offset, |
Ganesh Acharya | f9e1471 | 2017-04-30 20:39:14 | [diff] [blame] | 127 | }, |
| 128 | { |
balaji | a425b0e | 2019-09-24 14:06:36 | [diff] [blame] | 129 | Key: []byte("samplekeyb"), |
| 130 | Value: []byte(val2), |
| 131 | meta: bitValuePointer, |
| 132 | offset: b.Ptrs[1].Offset, |
Ganesh Acharya | 080745a | 2017-04-04 05:15:40 | [diff] [blame] | 133 | }, |
| 134 | }, readEntries) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 135 | |
Ganesh Acharya | 080745a | 2017-04-04 05:15:40 | [diff] [blame] | 136 | } |
| 137 | |
Manish R Jain | d22c0e8 | 2018-09-19 17:44:01 | [diff] [blame] | 138 | func TestValueGCManaged(t *testing.T) { |
Manish R Jain | b21f591 | 2021-04-27 04:37:45 | [diff] [blame] | 139 | t.Skipf("Value Log is not used in managed mode.") |
| 140 | |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 141 | dir, err := ioutil.TempDir("", "badger-test") |
Manish R Jain | d22c0e8 | 2018-09-19 17:44:01 | [diff] [blame] | 142 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 143 | defer removeDir(dir) |
Manish R Jain | d22c0e8 | 2018-09-19 17:44:01 | [diff] [blame] | 144 | |
| 145 | N := 10000 |
Ibrahim Jarif | 6c35ad6 | 2021-02-09 20:11:11 | [diff] [blame] | 146 | |
Manish R Jain | d22c0e8 | 2018-09-19 17:44:01 | [diff] [blame] | 147 | opt := getTestOptions(dir) |
| 148 | opt.ValueLogMaxEntries = uint32(N / 10) |
Manish R Jain | 970454a | 2018-10-02 22:05:04 | [diff] [blame] | 149 | opt.managedTxns = true |
Ibrahim Jarif | 6c35ad6 | 2021-02-09 20:11:11 | [diff] [blame] | 150 | opt.BaseTableSize = 1 << 15 |
| 151 | opt.ValueThreshold = 1 << 10 |
| 152 | opt.MemTableSize = 1 << 15 |
| 153 | |
Manish R Jain | d22c0e8 | 2018-09-19 17:44:01 | [diff] [blame] | 154 | db, err := Open(opt) |
| 155 | require.NoError(t, err) |
| 156 | defer db.Close() |
| 157 | |
| 158 | var ts uint64 |
| 159 | newTs := func() uint64 { |
marigonzes | 28ef9bf | 2019-01-31 17:54:06 | [diff] [blame] | 160 | ts++ |
Manish R Jain | d22c0e8 | 2018-09-19 17:44:01 | [diff] [blame] | 161 | return ts |
| 162 | } |
| 163 | |
| 164 | sz := 64 << 10 |
| 165 | var wg sync.WaitGroup |
| 166 | for i := 0; i < N; i++ { |
| 167 | v := make([]byte, sz) |
| 168 | rand.Read(v[:rand.Intn(sz)]) |
| 169 | |
| 170 | wg.Add(1) |
| 171 | txn := db.NewTransactionAt(newTs(), true) |
Ashish Goswami | e9447c9 | 2019-05-28 05:28:37 | [diff] [blame] | 172 | require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v))) |
Manish R Jain | d22c0e8 | 2018-09-19 17:44:01 | [diff] [blame] | 173 | require.NoError(t, txn.CommitAt(newTs(), func(err error) { |
| 174 | wg.Done() |
| 175 | require.NoError(t, err) |
| 176 | })) |
| 177 | } |
| 178 | |
| 179 | for i := 0; i < N; i++ { |
| 180 | wg.Add(1) |
| 181 | txn := db.NewTransactionAt(newTs(), true) |
| 182 | require.NoError(t, txn.Delete([]byte(fmt.Sprintf("key%d", i)))) |
| 183 | require.NoError(t, txn.CommitAt(newTs(), func(err error) { |
| 184 | wg.Done() |
| 185 | require.NoError(t, err) |
| 186 | })) |
| 187 | } |
| 188 | wg.Wait() |
| 189 | files, err := ioutil.ReadDir(dir) |
| 190 | require.NoError(t, err) |
| 191 | for _, fi := range files { |
Naman Jain | 7770c6e | 2021-03-30 13:56:44 | [diff] [blame] | 192 | t.Logf("File: %s. Size: %s\n", fi.Name(), humanize.IBytes(uint64(fi.Size()))) |
Manish R Jain | d22c0e8 | 2018-09-19 17:44:01 | [diff] [blame] | 193 | } |
| 194 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 195 | db.SetDiscardTs(math.MaxUint32) |
| 196 | db.Flatten(3) |
| 197 | |
Manish R Jain | d22c0e8 | 2018-09-19 17:44:01 | [diff] [blame] | 198 | for i := 0; i < 100; i++ { |
| 199 | // Try at max 100 times to GC even a single value log file. |
| 200 | if err := db.RunValueLogGC(0.0001); err == nil { |
| 201 | return // Done |
| 202 | } |
| 203 | } |
| 204 | require.Fail(t, "Unable to GC even a single value log file.") |
| 205 | } |
| 206 | |
Ganesh Acharya | bd27063 | 2017-04-27 16:31:24 | [diff] [blame] | 207 | func TestValueGC(t *testing.T) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 208 | dir, err := ioutil.TempDir("", "badger-test") |
Manish R Jain | 4cab233 | 2017-04-24 07:32:15 | [diff] [blame] | 209 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 210 | defer removeDir(dir) |
Szymon Matejczyk | 0fdb95e | 2017-05-08 14:55:14 | [diff] [blame] | 211 | opt := getTestOptions(dir) |
| 212 | opt.ValueLogFileSize = 1 << 20 |
Ibrahim Jarif | 6c35ad6 | 2021-02-09 20:11:11 | [diff] [blame] | 213 | opt.BaseTableSize = 1 << 15 |
| 214 | opt.ValueThreshold = 1 << 10 |
Szymon Matejczyk | 0fdb95e | 2017-05-08 14:55:14 | [diff] [blame] | 215 | |
Manish R Jain | abaad90 | 2017-10-04 10:55:56 | [diff] [blame] | 216 | kv, _ := Open(opt) |
Ganesh Acharya | f9e1471 | 2017-04-30 20:39:14 | [diff] [blame] | 217 | defer kv.Close() |
Manish R Jain | 4cab233 | 2017-04-24 07:32:15 | [diff] [blame] | 218 | |
Szymon Matejczyk | 0fdb95e | 2017-05-08 14:55:14 | [diff] [blame] | 219 | sz := 32 << 10 |
Manish R Jain | a6860cb | 2017-10-02 07:32:11 | [diff] [blame] | 220 | txn := kv.NewTransaction(true) |
Manish R Jain | 4cab233 | 2017-04-24 07:32:15 | [diff] [blame] | 221 | for i := 0; i < 100; i++ { |
Manish R Jain | 97cc434 | 2017-04-25 05:03:15 | [diff] [blame] | 222 | v := make([]byte, sz) |
Peter Stace | ec0f11d | 2017-08-04 04:57:35 | [diff] [blame] | 223 | rand.Read(v[:rand.Intn(sz)]) |
Ashish Goswami | e9447c9 | 2019-05-28 05:28:37 | [diff] [blame] | 224 | require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v))) |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 225 | if i%20 == 0 { |
Manish R Jain | 6daccf9 | 2018-10-09 01:05:00 | [diff] [blame] | 226 | require.NoError(t, txn.Commit()) |
Manish R Jain | a6860cb | 2017-10-02 07:32:11 | [diff] [blame] | 227 | txn = kv.NewTransaction(true) |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 228 | } |
Manish R Jain | 4cab233 | 2017-04-24 07:32:15 | [diff] [blame] | 229 | } |
Manish R Jain | 6daccf9 | 2018-10-09 01:05:00 | [diff] [blame] | 230 | require.NoError(t, txn.Commit()) |
Manish R Jain | 4cab233 | 2017-04-24 07:32:15 | [diff] [blame] | 231 | |
| 232 | for i := 0; i < 45; i++ { |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 233 | txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i))) |
Manish R Jain | 4cab233 | 2017-04-24 07:32:15 | [diff] [blame] | 234 | } |
| 235 | |
Sam Hughes | edbce1b | 2017-09-01 17:08:23 | [diff] [blame] | 236 | kv.vlog.filesLock.RLock() |
Sam Hughes | 2a6fc67 | 2017-09-03 22:08:47 | [diff] [blame] | 237 | lf := kv.vlog.filesMap[kv.vlog.sortedFids()[0]] |
Sam Hughes | edbce1b | 2017-09-01 17:08:23 | [diff] [blame] | 238 | kv.vlog.filesLock.RUnlock() |
Manish R Jain | 4cab233 | 2017-04-24 07:32:15 | [diff] [blame] | 239 | |
Ganesh Acharya | f9e1471 | 2017-04-30 20:39:14 | [diff] [blame] | 240 | // lf.iterate(0, func(e Entry) bool { |
| 241 | // e.print("lf") |
| 242 | // return true |
| 243 | // }) |
Manish R Jain | 4cab233 | 2017-04-24 07:32:15 | [diff] [blame] | 244 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 245 | kv.vlog.rewrite(lf) |
Manish R Jain | 4cab233 | 2017-04-24 07:32:15 | [diff] [blame] | 246 | for i := 45; i < 100; i++ { |
Manish R Jain | 55c350d | 2017-05-30 10:04:50 | [diff] [blame] | 247 | key := []byte(fmt.Sprintf("key%d", i)) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 248 | |
| 249 | require.NoError(t, kv.View(func(txn *Txn) error { |
| 250 | item, err := txn.Get(key) |
| 251 | require.NoError(t, err) |
Manish R Jain | cfd203d | 2017-10-05 02:50:34 | [diff] [blame] | 252 | val := getItemValue(t, item) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 253 | require.NotNil(t, val) |
| 254 | require.True(t, len(val) == sz, "Size found: %d", len(val)) |
| 255 | return nil |
| 256 | })) |
Manish R Jain | 4cab233 | 2017-04-24 07:32:15 | [diff] [blame] | 257 | } |
| 258 | } |
| 259 | |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 260 | func TestValueGC2(t *testing.T) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 261 | dir, err := ioutil.TempDir("", "badger-test") |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 262 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 263 | defer removeDir(dir) |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 264 | opt := getTestOptions(dir) |
| 265 | opt.ValueLogFileSize = 1 << 20 |
Ibrahim Jarif | 6c35ad6 | 2021-02-09 20:11:11 | [diff] [blame] | 266 | opt.BaseTableSize = 1 << 15 |
| 267 | opt.ValueThreshold = 1 << 10 |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 268 | |
Manish R Jain | abaad90 | 2017-10-04 10:55:56 | [diff] [blame] | 269 | kv, _ := Open(opt) |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 270 | defer kv.Close() |
| 271 | |
| 272 | sz := 32 << 10 |
Manish R Jain | a6860cb | 2017-10-02 07:32:11 | [diff] [blame] | 273 | txn := kv.NewTransaction(true) |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 274 | for i := 0; i < 100; i++ { |
| 275 | v := make([]byte, sz) |
Peter Stace | ec0f11d | 2017-08-04 04:57:35 | [diff] [blame] | 276 | rand.Read(v[:rand.Intn(sz)]) |
Ashish Goswami | e9447c9 | 2019-05-28 05:28:37 | [diff] [blame] | 277 | require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v))) |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 278 | if i%20 == 0 { |
Manish R Jain | 6daccf9 | 2018-10-09 01:05:00 | [diff] [blame] | 279 | require.NoError(t, txn.Commit()) |
Manish R Jain | a6860cb | 2017-10-02 07:32:11 | [diff] [blame] | 280 | txn = kv.NewTransaction(true) |
Janardhan Reddy | f407625 | 2017-09-05 01:28:57 | [diff] [blame] | 281 | } |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 282 | } |
Manish R Jain | 6daccf9 | 2018-10-09 01:05:00 | [diff] [blame] | 283 | require.NoError(t, txn.Commit()) |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 284 | |
| 285 | for i := 0; i < 5; i++ { |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 286 | txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i))) |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 287 | } |
| 288 | |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 289 | for i := 5; i < 10; i++ { |
| 290 | v := []byte(fmt.Sprintf("value%d", i)) |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 291 | txnSet(t, kv, []byte(fmt.Sprintf("key%d", i)), v, 0) |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 292 | } |
| 293 | |
Sam Hughes | edbce1b | 2017-09-01 17:08:23 | [diff] [blame] | 294 | kv.vlog.filesLock.RLock() |
Sam Hughes | 2a6fc67 | 2017-09-03 22:08:47 | [diff] [blame] | 295 | lf := kv.vlog.filesMap[kv.vlog.sortedFids()[0]] |
Sam Hughes | edbce1b | 2017-09-01 17:08:23 | [diff] [blame] | 296 | kv.vlog.filesLock.RUnlock() |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 297 | |
| 298 | // lf.iterate(0, func(e Entry) bool { |
| 299 | // e.print("lf") |
| 300 | // return true |
| 301 | // }) |
| 302 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 303 | kv.vlog.rewrite(lf) |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 304 | for i := 0; i < 5; i++ { |
| 305 | key := []byte(fmt.Sprintf("key%d", i)) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 306 | require.NoError(t, kv.View(func(txn *Txn) error { |
| 307 | _, err := txn.Get(key) |
Manish R Jain | c3ac2ff | 2018-04-30 15:43:55 | [diff] [blame] | 308 | require.Equal(t, ErrKeyNotFound, err) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 309 | return nil |
| 310 | })) |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 311 | } |
| 312 | for i := 5; i < 10; i++ { |
| 313 | key := []byte(fmt.Sprintf("key%d", i)) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 314 | require.NoError(t, kv.View(func(txn *Txn) error { |
| 315 | item, err := txn.Get(key) |
| 316 | require.NoError(t, err) |
Manish R Jain | cfd203d | 2017-10-05 02:50:34 | [diff] [blame] | 317 | val := getItemValue(t, item) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 318 | require.NotNil(t, val) |
| 319 | require.Equal(t, string(val), fmt.Sprintf("value%d", i)) |
| 320 | return nil |
| 321 | })) |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 322 | } |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 323 | // Moved entries. |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 324 | for i := 10; i < 100; i++ { |
| 325 | key := []byte(fmt.Sprintf("key%d", i)) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 326 | require.NoError(t, kv.View(func(txn *Txn) error { |
| 327 | item, err := txn.Get(key) |
| 328 | require.NoError(t, err) |
Manish R Jain | cfd203d | 2017-10-05 02:50:34 | [diff] [blame] | 329 | val := getItemValue(t, item) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 330 | require.NotNil(t, val) |
| 331 | require.True(t, len(val) == sz, "Size found: %d", len(val)) |
| 332 | return nil |
| 333 | })) |
Janardhan Reddy | e0c226b | 2017-07-24 07:42:09 | [diff] [blame] | 334 | } |
| 335 | } |
| 336 | |
Sam Hughes | 9ed12b9 | 2017-09-14 05:15:36 | [diff] [blame] | 337 | func TestValueGC3(t *testing.T) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 338 | dir, err := ioutil.TempDir("", "badger-test") |
Sam Hughes | 9ed12b9 | 2017-09-14 05:15:36 | [diff] [blame] | 339 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 340 | defer removeDir(dir) |
Sam Hughes | 9ed12b9 | 2017-09-14 05:15:36 | [diff] [blame] | 341 | opt := getTestOptions(dir) |
| 342 | opt.ValueLogFileSize = 1 << 20 |
Ibrahim Jarif | 6c35ad6 | 2021-02-09 20:11:11 | [diff] [blame] | 343 | opt.BaseTableSize = 1 << 15 |
| 344 | opt.ValueThreshold = 1 << 10 |
Sam Hughes | 9ed12b9 | 2017-09-14 05:15:36 | [diff] [blame] | 345 | |
Manish R Jain | abaad90 | 2017-10-04 10:55:56 | [diff] [blame] | 346 | kv, err := Open(opt) |
Sam Hughes | 9ed12b9 | 2017-09-14 05:15:36 | [diff] [blame] | 347 | require.NoError(t, err) |
| 348 | defer kv.Close() |
| 349 | |
| 350 | // We want to test whether an iterator can continue through a value log GC. |
| 351 | |
| 352 | valueSize := 32 << 10 |
| 353 | |
| 354 | var value3 []byte |
Manish R Jain | a6860cb | 2017-10-02 07:32:11 | [diff] [blame] | 355 | txn := kv.NewTransaction(true) |
Sam Hughes | 9ed12b9 | 2017-09-14 05:15:36 | [diff] [blame] | 356 | for i := 0; i < 100; i++ { |
| 357 | v := make([]byte, valueSize) // 32K * 100 will take >=3'276'800 B. |
| 358 | if i == 3 { |
| 359 | value3 = v |
| 360 | } |
| 361 | rand.Read(v[:]) |
| 362 | // Keys key000, key001, key002, such that sorted order matches insertion order |
Ashish Goswami | e9447c9 | 2019-05-28 05:28:37 | [diff] [blame] | 363 | require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%03d", i)), v))) |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 364 | if i%20 == 0 { |
Manish R Jain | 6daccf9 | 2018-10-09 01:05:00 | [diff] [blame] | 365 | require.NoError(t, txn.Commit()) |
Manish R Jain | a6860cb | 2017-10-02 07:32:11 | [diff] [blame] | 366 | txn = kv.NewTransaction(true) |
Sam Hughes | 9ed12b9 | 2017-09-14 05:15:36 | [diff] [blame] | 367 | } |
Sam Hughes | 9ed12b9 | 2017-09-14 05:15:36 | [diff] [blame] | 368 | } |
Manish R Jain | 6daccf9 | 2018-10-09 01:05:00 | [diff] [blame] | 369 | require.NoError(t, txn.Commit()) |
Sam Hughes | 9ed12b9 | 2017-09-14 05:15:36 | [diff] [blame] | 370 | |
| 371 | // Start an iterator to keys in the first value log file |
| 372 | itOpt := IteratorOptions{ |
| 373 | PrefetchValues: false, |
| 374 | PrefetchSize: 0, |
| 375 | Reverse: false, |
| 376 | } |
| 377 | |
Manish R Jain | a6860cb | 2017-10-02 07:32:11 | [diff] [blame] | 378 | txn = kv.NewTransaction(true) |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 379 | it := txn.NewIterator(itOpt) |
Sam Hughes | 9ed12b9 | 2017-09-14 05:15:36 | [diff] [blame] | 380 | defer it.Close() |
| 381 | // Walk a few keys |
| 382 | it.Rewind() |
| 383 | require.True(t, it.Valid()) |
| 384 | item := it.Item() |
| 385 | require.Equal(t, []byte("key000"), item.Key()) |
| 386 | it.Next() |
| 387 | require.True(t, it.Valid()) |
| 388 | item = it.Item() |
| 389 | require.Equal(t, []byte("key001"), item.Key()) |
| 390 | it.Next() |
| 391 | require.True(t, it.Valid()) |
| 392 | item = it.Item() |
| 393 | require.Equal(t, []byte("key002"), item.Key()) |
| 394 | |
| 395 | // Like other tests, we pull out a logFile to rewrite it directly |
| 396 | |
| 397 | kv.vlog.filesLock.RLock() |
| 398 | logFile := kv.vlog.filesMap[kv.vlog.sortedFids()[0]] |
| 399 | kv.vlog.filesLock.RUnlock() |
| 400 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 401 | kv.vlog.rewrite(logFile) |
Sam Hughes | 9ed12b9 | 2017-09-14 05:15:36 | [diff] [blame] | 402 | it.Next() |
| 403 | require.True(t, it.Valid()) |
| 404 | item = it.Item() |
| 405 | require.Equal(t, []byte("key003"), item.Key()) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 406 | |
Manish R Jain | c10276c | 2018-09-25 04:13:48 | [diff] [blame] | 407 | v3, err := item.ValueCopy(nil) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 408 | require.NoError(t, err) |
Sam Hughes | 9ed12b9 | 2017-09-14 05:15:36 | [diff] [blame] | 409 | require.Equal(t, value3, v3) |
| 410 | } |
| 411 | |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 412 | func TestValueGC4(t *testing.T) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 413 | dir, err := ioutil.TempDir("", "badger-test") |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 414 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 415 | defer removeDir(dir) |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 416 | opt := getTestOptions(dir) |
| 417 | opt.ValueLogFileSize = 1 << 20 |
Ibrahim Jarif | 6c35ad6 | 2021-02-09 20:11:11 | [diff] [blame] | 418 | opt.BaseTableSize = 1 << 15 |
| 419 | opt.ValueThreshold = 1 << 10 |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 420 | |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 421 | kv, err := Open(opt) |
| 422 | require.NoError(t, err) |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 423 | |
| 424 | sz := 128 << 10 // 5 entries per value log file. |
| 425 | txn := kv.NewTransaction(true) |
| 426 | for i := 0; i < 24; i++ { |
| 427 | v := make([]byte, sz) |
| 428 | rand.Read(v[:rand.Intn(sz)]) |
Ashish Goswami | e9447c9 | 2019-05-28 05:28:37 | [diff] [blame] | 429 | require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v))) |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 430 | if i%3 == 0 { |
Manish R Jain | 6daccf9 | 2018-10-09 01:05:00 | [diff] [blame] | 431 | require.NoError(t, txn.Commit()) |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 432 | txn = kv.NewTransaction(true) |
| 433 | } |
| 434 | } |
Manish R Jain | 6daccf9 | 2018-10-09 01:05:00 | [diff] [blame] | 435 | require.NoError(t, txn.Commit()) |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 436 | |
| 437 | for i := 0; i < 8; i++ { |
| 438 | txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i))) |
| 439 | } |
| 440 | |
| 441 | for i := 8; i < 16; i++ { |
| 442 | v := []byte(fmt.Sprintf("value%d", i)) |
| 443 | txnSet(t, kv, []byte(fmt.Sprintf("key%d", i)), v, 0) |
| 444 | } |
| 445 | |
| 446 | kv.vlog.filesLock.RLock() |
| 447 | lf0 := kv.vlog.filesMap[kv.vlog.sortedFids()[0]] |
| 448 | lf1 := kv.vlog.filesMap[kv.vlog.sortedFids()[1]] |
| 449 | kv.vlog.filesLock.RUnlock() |
| 450 | |
| 451 | // lf.iterate(0, func(e Entry) bool { |
| 452 | // e.print("lf") |
| 453 | // return true |
| 454 | // }) |
| 455 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 456 | kv.vlog.rewrite(lf0) |
| 457 | kv.vlog.rewrite(lf1) |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 458 | |
Ibrahim Jarif | 509de73 | 2020-06-30 15:30:38 | [diff] [blame] | 459 | require.NoError(t, kv.Close()) |
Martin Martinez Rivera | 1dd99e4 | 2019-01-03 18:06:36 | [diff] [blame] | 460 | |
Ibrahim Jarif | 509de73 | 2020-06-30 15:30:38 | [diff] [blame] | 461 | kv, err = Open(opt) |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 462 | require.NoError(t, err) |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 463 | |
| 464 | for i := 0; i < 8; i++ { |
| 465 | key := []byte(fmt.Sprintf("key%d", i)) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 466 | require.NoError(t, kv.View(func(txn *Txn) error { |
| 467 | _, err := txn.Get(key) |
| 468 | require.Equal(t, ErrKeyNotFound, err) |
| 469 | return nil |
| 470 | })) |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 471 | } |
| 472 | for i := 8; i < 16; i++ { |
| 473 | key := []byte(fmt.Sprintf("key%d", i)) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 474 | require.NoError(t, kv.View(func(txn *Txn) error { |
| 475 | item, err := txn.Get(key) |
| 476 | require.NoError(t, err) |
Manish R Jain | cfd203d | 2017-10-05 02:50:34 | [diff] [blame] | 477 | val := getItemValue(t, item) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 478 | require.NotNil(t, val) |
| 479 | require.Equal(t, string(val), fmt.Sprintf("value%d", i)) |
| 480 | return nil |
| 481 | })) |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 482 | } |
Ibrahim Jarif | 509de73 | 2020-06-30 15:30:38 | [diff] [blame] | 483 | require.NoError(t, kv.Close()) |
Janardhan Reddy | 9b31d1a | 2017-10-03 23:53:02 | [diff] [blame] | 484 | } |
| 485 | |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 486 | func TestPersistLFDiscardStats(t *testing.T) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 487 | dir, err := ioutil.TempDir("", "badger-test") |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 488 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 489 | defer removeDir(dir) |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 490 | opt := getTestOptions(dir) |
Ibrahim Jarif | 0506f78 | 2020-11-02 08:01:52 | [diff] [blame] | 491 | // Force more compaction by reducing the number of L0 tables. |
| 492 | opt.NumLevelZeroTables = 1 |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 493 | opt.ValueLogFileSize = 1 << 20 |
Ibrahim Jarif | 0506f78 | 2020-11-02 08:01:52 | [diff] [blame] | 494 | // Avoid compaction on close so that the discard map remains the same. |
Manish R Jain | 2237832 | 2019-05-12 07:24:21 | [diff] [blame] | 495 | opt.CompactL0OnClose = false |
Ibrahim Jarif | 6c35ad6 | 2021-02-09 20:11:11 | [diff] [blame] | 496 | opt.MemTableSize = 1 << 15 |
| 497 | opt.ValueThreshold = 1 << 10 |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 498 | |
| 499 | db, err := Open(opt) |
| 500 | require.NoError(t, err) |
| 501 | |
| 502 | sz := 128 << 10 // 5 entries per value log file. |
| 503 | v := make([]byte, sz) |
| 504 | rand.Read(v[:rand.Intn(sz)]) |
| 505 | txn := db.NewTransaction(true) |
| 506 | for i := 0; i < 500; i++ { |
Ashish Goswami | e9447c9 | 2019-05-28 05:28:37 | [diff] [blame] | 507 | require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v))) |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 508 | if i%3 == 0 { |
| 509 | require.NoError(t, txn.Commit()) |
| 510 | txn = db.NewTransaction(true) |
| 511 | } |
| 512 | } |
Martin Martinez Rivera | a712a3f | 2019-05-30 18:14:00 | [diff] [blame] | 513 | require.NoError(t, txn.Commit(), "error while committing txn") |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 514 | |
| 515 | for i := 0; i < 500; i++ { |
Ashish Goswami | cd5884e | 2019-06-04 13:15:39 | [diff] [blame] | 516 | // use Entry.WithDiscard() to delete entries, because this causes data to be flushed on |
| 517 | // disk, creating SSTs. Simple Delete was having data in Memtables only. |
Manish R Jain | 2237832 | 2019-05-12 07:24:21 | [diff] [blame] | 518 | err = db.Update(func(txn *Txn) error { |
Ashish Goswami | e9447c9 | 2019-05-28 05:28:37 | [diff] [blame] | 519 | return txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v).WithDiscard()) |
Manish R Jain | 2237832 | 2019-05-12 07:24:21 | [diff] [blame] | 520 | }) |
| 521 | require.NoError(t, err) |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 522 | } |
| 523 | |
Ibrahim Jarif | 675efcd | 2020-06-24 15:16:12 | [diff] [blame] | 524 | time.Sleep(2 * time.Second) // wait for compaction to complete |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 525 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 526 | persistedMap := make(map[uint64]uint64) |
| 527 | db.vlog.discardStats.Lock() |
| 528 | require.True(t, db.vlog.discardStats.Len() > 1, "some discardStats should be generated") |
Manish R Jain | a78c086 | 2021-02-05 02:38:20 | [diff] [blame] | 529 | db.vlog.discardStats.Iterate(func(fid, val uint64) { |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 530 | persistedMap[fid] = val |
| 531 | }) |
Ashish Goswami | c1cf0d7 | 2019-10-16 10:38:10 | [diff] [blame] | 532 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 533 | require.NoError(t, db.Close()) |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 534 | |
Ibrahim Jarif | 0b8eb4c | 2020-08-19 13:12:06 | [diff] [blame] | 535 | // Avoid running compactors on reopening badger. |
| 536 | opt.NumCompactors = 0 |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 537 | db, err = Open(opt) |
| 538 | require.NoError(t, err) |
| 539 | defer db.Close() |
Ashish Goswami | c1cf0d7 | 2019-10-16 10:38:10 | [diff] [blame] | 540 | time.Sleep(1 * time.Second) // Wait for discardStats to be populated by populateDiscardStats(). |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 541 | db.vlog.discardStats.Lock() |
| 542 | statsMap := make(map[uint64]uint64) |
Manish R Jain | a78c086 | 2021-02-05 02:38:20 | [diff] [blame] | 543 | db.vlog.discardStats.Iterate(func(fid, val uint64) { |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 544 | statsMap[fid] = val |
| 545 | }) |
| 546 | require.True(t, reflect.DeepEqual(persistedMap, statsMap), "Discard maps are not equal") |
| 547 | db.vlog.discardStats.Unlock() |
ashish | f7bbdb8 | 2019-03-19 15:27:08 | [diff] [blame] | 548 | } |
| 549 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 550 | func TestValueChecksums(t *testing.T) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 551 | dir, err := ioutil.TempDir("", "badger-test") |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 552 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 553 | defer removeDir(dir) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 554 | |
| 555 | // Set up SST with K1=V1 |
Manish R Jain | 273f40c | 2017-09-21 05:44:36 | [diff] [blame] | 556 | opts := getTestOptions(dir) |
| 557 | opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 558 | opts.VerifyValueChecksum = true |
Manish R Jain | abaad90 | 2017-10-04 10:55:56 | [diff] [blame] | 559 | kv, err := Open(opts) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 560 | require.NoError(t, err) |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 561 | require.NoError(t, kv.Close()) |
Sam Hughes | 8de624f | 2017-09-20 23:30:16 | [diff] [blame] | 562 | |
| 563 | var ( |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 564 | k0 = []byte("k0") |
Sam Hughes | 8de624f | 2017-09-20 23:30:16 | [diff] [blame] | 565 | k1 = []byte("k1") |
| 566 | k2 = []byte("k2") |
| 567 | k3 = []byte("k3") |
Manish R Jain | 7af0076 | 2018-05-08 23:42:00 | [diff] [blame] | 568 | v0 = []byte("value0-012345678901234567890123012345678901234567890123") |
| 569 | v1 = []byte("value1-012345678901234567890123012345678901234567890123") |
| 570 | v2 = []byte("value2-012345678901234567890123012345678901234567890123") |
| 571 | v3 = []byte("value3-012345678901234567890123012345678901234567890123") |
Sam Hughes | 8de624f | 2017-09-20 23:30:16 | [diff] [blame] | 572 | ) |
Sam Hughes | 8de624f | 2017-09-20 23:30:16 | [diff] [blame] | 573 | |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 574 | // Use a vlog with K0=V0 and a (corrupted) second transaction(k1,k2) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 575 | buf, offset := createMemFile(t, []*Entry{ |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 576 | {Key: k0, Value: v0}, |
Deepak Jois | 3e99987 | 2017-09-07 08:11:00 | [diff] [blame] | 577 | {Key: k1, Value: v1}, |
| 578 | {Key: k2, Value: v2}, |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 579 | }) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 580 | buf[offset-1]++ // Corrupt last byte |
| 581 | require.NoError(t, ioutil.WriteFile(kv.mtFilePath(1), buf, 0777)) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 582 | |
| 583 | // K1 should exist, but K2 shouldn't. |
Manish R Jain | abaad90 | 2017-10-04 10:55:56 | [diff] [blame] | 584 | kv, err = Open(opts) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 585 | require.NoError(t, err) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 586 | |
| 587 | require.NoError(t, kv.View(func(txn *Txn) error { |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 588 | // Replay should have added K0. |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 589 | item, err := txn.Get(k0) |
| 590 | require.NoError(t, err) |
Manish R Jain | cfd203d | 2017-10-05 02:50:34 | [diff] [blame] | 591 | require.Equal(t, getItemValue(t, item), v0) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 592 | |
| 593 | _, err = txn.Get(k1) |
Manish R Jain | c3ac2ff | 2018-04-30 15:43:55 | [diff] [blame] | 594 | require.Equal(t, ErrKeyNotFound, err) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 595 | |
| 596 | _, err = txn.Get(k2) |
Manish R Jain | c3ac2ff | 2018-04-30 15:43:55 | [diff] [blame] | 597 | require.Equal(t, ErrKeyNotFound, err) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 598 | return nil |
| 599 | })) |
| 600 | |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 601 | // Write K3 at the end of the vlog. |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 602 | txnSet(t, kv, k3, v3, 0) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 603 | require.NoError(t, kv.Close()) |
| 604 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 605 | // The DB should contain K0 and K3 (K1 and k2 was lost when Badger started up |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 606 | // last due to checksum failure). |
Manish R Jain | abaad90 | 2017-10-04 10:55:56 | [diff] [blame] | 607 | kv, err = Open(opts) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 608 | require.NoError(t, err) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 609 | |
| 610 | { |
| 611 | txn := kv.NewTransaction(false) |
| 612 | |
| 613 | iter := txn.NewIterator(DefaultIteratorOptions) |
| 614 | iter.Seek(k0) |
| 615 | require.True(t, iter.Valid()) |
| 616 | it := iter.Item() |
| 617 | require.Equal(t, it.Key(), k0) |
| 618 | require.Equal(t, getItemValue(t, it), v0) |
| 619 | iter.Next() |
| 620 | require.True(t, iter.Valid()) |
| 621 | it = iter.Item() |
| 622 | require.Equal(t, it.Key(), k3) |
| 623 | require.Equal(t, getItemValue(t, it), v3) |
| 624 | |
| 625 | iter.Close() |
| 626 | txn.Discard() |
| 627 | } |
| 628 | |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 629 | require.NoError(t, kv.Close()) |
| 630 | } |
| 631 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 632 | // TODO: Do we need this test? |
| 633 | func TestPartialAppendToWAL(t *testing.T) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 634 | dir, err := ioutil.TempDir("", "badger-test") |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 635 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 636 | defer removeDir(dir) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 637 | |
| 638 | // Create skeleton files. |
Manish R Jain | 273f40c | 2017-09-21 05:44:36 | [diff] [blame] | 639 | opts := getTestOptions(dir) |
| 640 | opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb |
Ibrahim Jarif | 675efcd | 2020-06-24 15:16:12 | [diff] [blame] | 641 | opts.ValueThreshold = 32 |
Manish R Jain | abaad90 | 2017-10-04 10:55:56 | [diff] [blame] | 642 | kv, err := Open(opts) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 643 | require.NoError(t, err) |
| 644 | require.NoError(t, kv.Close()) |
| 645 | |
Sam Hughes | 8de624f | 2017-09-20 23:30:16 | [diff] [blame] | 646 | var ( |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 647 | k0 = []byte("k0") |
Sam Hughes | 8de624f | 2017-09-20 23:30:16 | [diff] [blame] | 648 | k1 = []byte("k1") |
| 649 | k2 = []byte("k2") |
| 650 | k3 = []byte("k3") |
Manish R Jain | 7af0076 | 2018-05-08 23:42:00 | [diff] [blame] | 651 | v0 = []byte("value0-01234567890123456789012012345678901234567890123") |
| 652 | v1 = []byte("value1-01234567890123456789012012345678901234567890123") |
| 653 | v2 = []byte("value2-01234567890123456789012012345678901234567890123") |
| 654 | v3 = []byte("value3-01234567890123456789012012345678901234567890123") |
Sam Hughes | 8de624f | 2017-09-20 23:30:16 | [diff] [blame] | 655 | ) |
| 656 | // Values need to be long enough to actually get written to value log. |
aman bansal | 5dbae10 | 2021-02-04 12:59:51 | [diff] [blame] | 657 | require.True(t, int64(len(v3)) >= kv.vlog.db.valueThreshold()) |
Sam Hughes | 8de624f | 2017-09-20 23:30:16 | [diff] [blame] | 658 | |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 659 | // Create truncated vlog to simulate a partial append. |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 660 | // k0 - single transaction, k1 and k2 in another transaction |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 661 | buf, offset := createMemFile(t, []*Entry{ |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 662 | {Key: k0, Value: v0}, |
Deepak Jois | 3e99987 | 2017-09-07 08:11:00 | [diff] [blame] | 663 | {Key: k1, Value: v1}, |
| 664 | {Key: k2, Value: v2}, |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 665 | }) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 666 | buf = buf[:offset-6] |
| 667 | require.NoError(t, ioutil.WriteFile(kv.mtFilePath(1), buf, 0777)) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 668 | |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 669 | // Badger should now start up |
Manish R Jain | abaad90 | 2017-10-04 10:55:56 | [diff] [blame] | 670 | kv, err = Open(opts) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 671 | require.NoError(t, err) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 672 | |
| 673 | require.NoError(t, kv.View(func(txn *Txn) error { |
| 674 | item, err := txn.Get(k0) |
| 675 | require.NoError(t, err) |
Manish R Jain | cfd203d | 2017-10-05 02:50:34 | [diff] [blame] | 676 | require.Equal(t, v0, getItemValue(t, item)) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 677 | |
| 678 | _, err = txn.Get(k1) |
Janardhan Reddy | 101a2dc | 2018-03-01 04:01:15 | [diff] [blame] | 679 | require.Equal(t, ErrKeyNotFound, err) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 680 | _, err = txn.Get(k2) |
Janardhan Reddy | 101a2dc | 2018-03-01 04:01:15 | [diff] [blame] | 681 | require.Equal(t, ErrKeyNotFound, err) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 682 | return nil |
| 683 | })) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 684 | |
| 685 | // When K3 is set, it should be persisted after a restart. |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 686 | txnSet(t, kv, k3, v3, 0) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 687 | require.NoError(t, kv.Close()) |
Martin Martinez Rivera | f937f21 | 2019-01-10 19:23:42 | [diff] [blame] | 688 | kv, err = Open(opts) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 689 | require.NoError(t, err) |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 690 | checkKeys(t, kv, [][]byte{k3}) |
Janardhan Reddy | 101a2dc | 2018-03-01 04:01:15 | [diff] [blame] | 691 | // Replay value log from beginning, badger head is past k2. |
Martin Martinez Rivera | f937f21 | 2019-01-10 19:23:42 | [diff] [blame] | 692 | require.NoError(t, kv.vlog.Close()) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 693 | } |
| 694 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 695 | func TestReadOnlyOpenWithPartialAppendToWAL(t *testing.T) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 696 | dir, err := ioutil.TempDir("", "badger-test") |
Allen Luce | dc0df25 | 2018-03-08 03:45:33 | [diff] [blame] | 697 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 698 | defer removeDir(dir) |
Allen Luce | dc0df25 | 2018-03-08 03:45:33 | [diff] [blame] | 699 | |
| 700 | // Create skeleton files. |
| 701 | opts := getTestOptions(dir) |
| 702 | opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb |
| 703 | kv, err := Open(opts) |
| 704 | require.NoError(t, err) |
| 705 | require.NoError(t, kv.Close()) |
| 706 | |
| 707 | var ( |
| 708 | k0 = []byte("k0") |
| 709 | k1 = []byte("k1") |
| 710 | k2 = []byte("k2") |
| 711 | v0 = []byte("value0-012345678901234567890123") |
| 712 | v1 = []byte("value1-012345678901234567890123") |
| 713 | v2 = []byte("value2-012345678901234567890123") |
| 714 | ) |
| 715 | |
| 716 | // Create truncated vlog to simulate a partial append. |
| 717 | // k0 - single transaction, k1 and k2 in another transaction |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 718 | buf, offset := createMemFile(t, []*Entry{ |
Allen Luce | dc0df25 | 2018-03-08 03:45:33 | [diff] [blame] | 719 | {Key: k0, Value: v0}, |
| 720 | {Key: k1, Value: v1}, |
| 721 | {Key: k2, Value: v2}, |
| 722 | }) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 723 | buf = buf[:offset-6] |
Naman Jain | 16ec633 | 2020-11-03 08:33:52 | [diff] [blame] | 724 | require.NoError(t, ioutil.WriteFile(kv.mtFilePath(1), buf, 0777)) |
Allen Luce | dc0df25 | 2018-03-08 03:45:33 | [diff] [blame] | 725 | |
| 726 | opts.ReadOnly = true |
| 727 | // Badger should fail a read-only open with values to replay |
Martin Martinez Rivera | 50ddf21 | 2019-09-30 17:10:22 | [diff] [blame] | 728 | _, err = Open(opts) |
Allen Luce | dc0df25 | 2018-03-08 03:45:33 | [diff] [blame] | 729 | require.Error(t, err) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 730 | require.Regexp(t, "Log truncate required", err.Error()) |
Allen Luce | dc0df25 | 2018-03-08 03:45:33 | [diff] [blame] | 731 | } |
| 732 | |
Manish R Jain | 935ef5d | 2017-09-15 03:20:15 | [diff] [blame] | 733 | func TestValueLogTrigger(t *testing.T) { |
Deepak Jois | 37c2a90 | 2017-10-18 04:55:55 | [diff] [blame] | 734 | t.Skip("Difficult to trigger compaction, so skipping. Re-enable after fixing #226") |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 735 | dir, err := ioutil.TempDir("", "badger-test") |
Manish R Jain | 935ef5d | 2017-09-15 03:20:15 | [diff] [blame] | 736 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 737 | defer removeDir(dir) |
Manish R Jain | 935ef5d | 2017-09-15 03:20:15 | [diff] [blame] | 738 | |
| 739 | opt := getTestOptions(dir) |
| 740 | opt.ValueLogFileSize = 1 << 20 |
Manish R Jain | abaad90 | 2017-10-04 10:55:56 | [diff] [blame] | 741 | kv, err := Open(opt) |
Manish R Jain | 935ef5d | 2017-09-15 03:20:15 | [diff] [blame] | 742 | require.NoError(t, err) |
| 743 | |
| 744 | // Write a lot of data, so it creates some work for valug log GC. |
| 745 | sz := 32 << 10 |
Manish R Jain | a6860cb | 2017-10-02 07:32:11 | [diff] [blame] | 746 | txn := kv.NewTransaction(true) |
Manish R Jain | 935ef5d | 2017-09-15 03:20:15 | [diff] [blame] | 747 | for i := 0; i < 100; i++ { |
| 748 | v := make([]byte, sz) |
| 749 | rand.Read(v[:rand.Intn(sz)]) |
Ashish Goswami | e9447c9 | 2019-05-28 05:28:37 | [diff] [blame] | 750 | require.NoError(t, txn.SetEntry(NewEntry([]byte(fmt.Sprintf("key%d", i)), v))) |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 751 | if i%20 == 0 { |
Manish R Jain | 6daccf9 | 2018-10-09 01:05:00 | [diff] [blame] | 752 | require.NoError(t, txn.Commit()) |
Manish R Jain | a6860cb | 2017-10-02 07:32:11 | [diff] [blame] | 753 | txn = kv.NewTransaction(true) |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 754 | } |
Manish R Jain | 935ef5d | 2017-09-15 03:20:15 | [diff] [blame] | 755 | } |
Manish R Jain | 6daccf9 | 2018-10-09 01:05:00 | [diff] [blame] | 756 | require.NoError(t, txn.Commit()) |
Manish R Jain | 935ef5d | 2017-09-15 03:20:15 | [diff] [blame] | 757 | |
| 758 | for i := 0; i < 45; i++ { |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 759 | txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i))) |
Manish R Jain | 935ef5d | 2017-09-15 03:20:15 | [diff] [blame] | 760 | } |
| 761 | |
Deepak Jois | 37c2a90 | 2017-10-18 04:55:55 | [diff] [blame] | 762 | require.NoError(t, kv.RunValueLogGC(0.5)) |
| 763 | |
Manish R Jain | 935ef5d | 2017-09-15 03:20:15 | [diff] [blame] | 764 | require.NoError(t, kv.Close()) |
| 765 | |
| 766 | err = kv.RunValueLogGC(0.5) |
Manish R Jain | abaad90 | 2017-10-04 10:55:56 | [diff] [blame] | 767 | require.Equal(t, ErrRejected, err, "Error should be returned after closing DB.") |
Manish R Jain | 935ef5d | 2017-09-15 03:20:15 | [diff] [blame] | 768 | } |
| 769 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 770 | // createMemFile creates a new memFile and returns the last valid offset. |
| 771 | func createMemFile(t *testing.T, entries []*Entry) ([]byte, uint32) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 772 | dir, err := ioutil.TempDir("", "badger-test") |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 773 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 774 | defer removeDir(dir) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 775 | |
Manish R Jain | 273f40c | 2017-09-21 05:44:36 | [diff] [blame] | 776 | opts := getTestOptions(dir) |
| 777 | opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb |
Manish R Jain | abaad90 | 2017-10-04 10:55:56 | [diff] [blame] | 778 | kv, err := Open(opts) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 779 | require.NoError(t, err) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 780 | defer kv.Close() |
| 781 | |
Deepak Jois | a5499e5 | 2017-11-02 02:03:14 | [diff] [blame] | 782 | txnSet(t, kv, entries[0].Key, entries[0].Value, entries[0].meta) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 783 | |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 784 | entries = entries[1:] |
Manish R Jain | a6860cb | 2017-10-02 07:32:11 | [diff] [blame] | 785 | txn := kv.NewTransaction(true) |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 786 | for _, entry := range entries { |
Ashish Goswami | e9447c9 | 2019-05-28 05:28:37 | [diff] [blame] | 787 | require.NoError(t, txn.SetEntry(NewEntry(entry.Key, entry.Value).WithMeta(entry.meta))) |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 788 | } |
Manish R Jain | 6daccf9 | 2018-10-09 01:05:00 | [diff] [blame] | 789 | require.NoError(t, txn.Commit()) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 790 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 791 | filename := kv.mtFilePath(1) |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 792 | buf, err := ioutil.ReadFile(filename) |
| 793 | require.NoError(t, err) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 794 | return buf, kv.mt.wal.writeAt |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 795 | } |
| 796 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 797 | // This test creates two mem files and corrupts the last bit of the first file. |
| 798 | func TestPenultimateMemCorruption(t *testing.T) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 799 | dir, err := ioutil.TempDir("", "badger-test") |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 800 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 801 | defer removeDir(dir) |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 802 | opt := getTestOptions(dir) |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 803 | |
| 804 | db0, err := Open(opt) |
| 805 | require.NoError(t, err) |
Ibrahim Jarif | b762832 | 2020-06-09 10:46:19 | [diff] [blame] | 806 | defer func() { require.NoError(t, db0.Close()) }() |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 807 | |
| 808 | h := testHelper{db: db0, t: t} |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 809 | h.writeRange(0, 2) // 00001.mem |
| 810 | |
| 811 | // Move the current memtable to the db.imm and create a new memtable so |
| 812 | // that we can have more than one mem files. |
| 813 | require.Zero(t, len(db0.imm)) |
| 814 | db0.imm = append(db0.imm, db0.mt) |
| 815 | db0.mt, err = db0.newMemTable() |
| 816 | |
| 817 | h.writeRange(3, 7) // 00002.mem |
| 818 | |
| 819 | // Verify we have all the data we wrote. |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 820 | h.readRange(0, 7) |
| 821 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 822 | for i := 2; i >= 1; i-- { |
| 823 | fpath := db0.mtFilePath(i) |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 824 | fi, err := os.Stat(fpath) |
| 825 | require.NoError(t, err) |
| 826 | require.True(t, fi.Size() > 0, "Empty file at log=%d", i) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 827 | if i == 1 { |
| 828 | // This should corrupt the last entry in the first memtable (that is entry number 2) |
| 829 | wal := db0.imm[0].wal |
| 830 | wal.Fd.WriteAt([]byte{0}, int64(wal.writeAt-1)) |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 831 | require.NoError(t, err) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 832 | // We have corrupted the file. We can remove it. If we don't remove |
| 833 | // the imm here, the db.close in defer will crash since db0.mt != |
| 834 | // db0.imm[0] |
| 835 | db0.imm = db0.imm[:0] |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 836 | } |
| 837 | } |
| 838 | // Simulate a crash by not closing db0, but releasing the locks. |
| 839 | if db0.dirLockGuard != nil { |
| 840 | require.NoError(t, db0.dirLockGuard.release()) |
Ibrahim Jarif | b762832 | 2020-06-09 10:46:19 | [diff] [blame] | 841 | db0.dirLockGuard = nil |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 842 | } |
| 843 | if db0.valueDirGuard != nil { |
| 844 | require.NoError(t, db0.valueDirGuard.release()) |
Ibrahim Jarif | b762832 | 2020-06-09 10:46:19 | [diff] [blame] | 845 | db0.valueDirGuard = nil |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 846 | } |
| 847 | |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 848 | db1, err := Open(opt) |
| 849 | require.NoError(t, err) |
| 850 | h.db = db1 |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 851 | // Only 2 should be gone because it is at the end of 0001.mem (first memfile). |
| 852 | h.readRange(0, 1) |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 853 | h.readRange(3, 7) |
| 854 | err = db1.View(func(txn *Txn) error { |
| 855 | _, err := txn.Get(h.key(2)) // Verify that 2 is gone. |
| 856 | require.Equal(t, ErrKeyNotFound, err) |
| 857 | return nil |
| 858 | }) |
| 859 | require.NoError(t, err) |
| 860 | require.NoError(t, db1.Close()) |
| 861 | } |
| 862 | |
Manish R Jain | abaad90 | 2017-10-04 10:55:56 | [diff] [blame] | 863 | func checkKeys(t *testing.T, kv *DB, keys [][]byte) { |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 864 | i := 0 |
Manish R Jain | a6860cb | 2017-10-02 07:32:11 | [diff] [blame] | 865 | txn := kv.NewTransaction(false) |
Ibrahim Jarif | 0f2e629 | 2020-01-07 12:34:02 | [diff] [blame] | 866 | defer txn.Discard() |
Janardhan Reddy | ffaaa66 | 2017-10-02 06:47:46 | [diff] [blame] | 867 | iter := txn.NewIterator(IteratorOptions{}) |
Ibrahim Jarif | 0f2e629 | 2020-01-07 12:34:02 | [diff] [blame] | 868 | defer iter.Close() |
Peter Stace | 766bf27 | 2017-08-08 03:40:34 | [diff] [blame] | 869 | for iter.Seek(keys[0]); iter.Valid(); iter.Next() { |
| 870 | require.Equal(t, iter.Item().Key(), keys[i]) |
| 871 | i++ |
| 872 | } |
| 873 | require.Equal(t, i, len(keys)) |
| 874 | } |
| 875 | |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 876 | type testHelper struct { |
| 877 | db *DB |
| 878 | t *testing.T |
| 879 | val []byte |
| 880 | } |
| 881 | |
| 882 | func (th *testHelper) key(i int) []byte { |
| 883 | return []byte(fmt.Sprintf("%010d", i)) |
| 884 | } |
| 885 | func (th *testHelper) value() []byte { |
| 886 | if len(th.val) > 0 { |
| 887 | return th.val |
| 888 | } |
| 889 | th.val = make([]byte, 100) |
| 890 | y.Check2(rand.Read(th.val)) |
| 891 | return th.val |
| 892 | } |
| 893 | |
| 894 | // writeRange [from, to]. |
| 895 | func (th *testHelper) writeRange(from, to int) { |
| 896 | for i := from; i <= to; i++ { |
| 897 | err := th.db.Update(func(txn *Txn) error { |
Ashish Goswami | e9447c9 | 2019-05-28 05:28:37 | [diff] [blame] | 898 | return txn.SetEntry(NewEntry(th.key(i), th.value())) |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 899 | }) |
| 900 | require.NoError(th.t, err) |
| 901 | } |
| 902 | } |
| 903 | |
| 904 | func (th *testHelper) readRange(from, to int) { |
| 905 | for i := from; i <= to; i++ { |
| 906 | err := th.db.View(func(txn *Txn) error { |
| 907 | item, err := txn.Get(th.key(i)) |
| 908 | if err != nil { |
| 909 | return err |
| 910 | } |
Martin Martinez Rivera | f796303 | 2019-01-04 01:53:25 | [diff] [blame] | 911 | return item.Value(func(val []byte) error { |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 912 | require.Equal(th.t, val, th.value(), "key=%q", th.key(i)) |
| 913 | return nil |
| 914 | |
Martin Martinez Rivera | f796303 | 2019-01-04 01:53:25 | [diff] [blame] | 915 | }) |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 916 | }) |
Manish R Jain | 6c8b15c | 2018-10-27 15:26:35 | [diff] [blame] | 917 | require.NoError(th.t, err, "key=%q", th.key(i)) |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 918 | } |
| 919 | } |
| 920 | |
Manish R Jain | af99e5f | 2018-09-23 17:43:37 | [diff] [blame] | 921 | // Test Bug #578, which showed that if a value is moved during value log GC, an |
| 922 | // older version can end up at a higher level in the LSM tree than a newer |
| 923 | // version, causing the data to not be returned. |
| 924 | func TestBug578(t *testing.T) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 925 | dir, err := ioutil.TempDir("", "badger-test") |
Manish R Jain | af99e5f | 2018-09-23 17:43:37 | [diff] [blame] | 926 | y.Check(err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 927 | defer removeDir(dir) |
Manish R Jain | af99e5f | 2018-09-23 17:43:37 | [diff] [blame] | 928 | |
Francesc Campoy | 2bb87c0 | 2019-06-24 21:57:56 | [diff] [blame] | 929 | db, err := Open(DefaultOptions(dir). |
| 930 | WithValueLogMaxEntries(64). |
Manish R Jain | 45bca18 | 2020-10-26 21:12:58 | [diff] [blame] | 931 | WithBaseTableSize(1 << 13)) |
Manish R Jain | af99e5f | 2018-09-23 17:43:37 | [diff] [blame] | 932 | require.NoError(t, err) |
| 933 | |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 934 | h := testHelper{db: db, t: t} |
Manish R Jain | af99e5f | 2018-09-23 17:43:37 | [diff] [blame] | 935 | |
| 936 | // Let's run this whole thing a few times. |
| 937 | for j := 0; j < 10; j++ { |
| 938 | t.Logf("Cycle: %d\n", j) |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 939 | h.writeRange(0, 32) |
| 940 | h.writeRange(0, 10) |
| 941 | h.writeRange(50, 72) |
| 942 | h.writeRange(40, 72) |
| 943 | h.writeRange(40, 72) |
Manish R Jain | af99e5f | 2018-09-23 17:43:37 | [diff] [blame] | 944 | |
| 945 | // Run value log GC a few times. |
| 946 | for i := 0; i < 5; i++ { |
| 947 | db.RunValueLogGC(0.5) |
| 948 | } |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 949 | h.readRange(0, 10) |
Manish R Jain | af99e5f | 2018-09-23 17:43:37 | [diff] [blame] | 950 | } |
Ibrahim Jarif | 3d5fe0b | 2019-08-08 09:18:25 | [diff] [blame] | 951 | require.NoError(t, db.Close()) |
Manish R Jain | af99e5f | 2018-09-23 17:43:37 | [diff] [blame] | 952 | } |
| 953 | |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 954 | func BenchmarkReadWrite(b *testing.B) { |
| 955 | rwRatio := []float32{ |
| 956 | 0.1, 0.2, 0.5, 1.0, |
| 957 | } |
| 958 | valueSize := []int{ |
Manish R Jain | 86ce19b | 2017-01-27 11:17:28 | [diff] [blame] | 959 | 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 960 | } |
| 961 | |
| 962 | for _, vsz := range valueSize { |
| 963 | for _, rw := range rwRatio { |
Manish R Jain | 1fcb50f | 2017-01-27 09:19:24 | [diff] [blame] | 964 | b.Run(fmt.Sprintf("%3.1f,%04d", rw, vsz), func(b *testing.B) { |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 965 | dir, err := ioutil.TempDir("", "vlog-benchmark") |
Motakjuq | 4b1e533 | 2017-06-27 09:04:01 | [diff] [blame] | 966 | y.Check(err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 967 | defer removeDir(dir) |
Naman Jain | 292a4be | 2021-09-14 14:04:35 | [diff] [blame] | 968 | opts := getTestOptions(dir) |
| 969 | opts.ValueThreshold = 0 |
| 970 | db, err := Open(opts) |
Motakjuq | 4b1e533 | 2017-06-27 09:04:01 | [diff] [blame] | 971 | y.Check(err) |
Manish R Jain | 1d37a10 | 2018-10-25 22:11:04 | [diff] [blame] | 972 | |
Martin Martinez Rivera | c301fa1 | 2019-01-04 19:08:16 | [diff] [blame] | 973 | vl := &db.vlog |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 974 | b.ResetTimer() |
| 975 | |
Manish R Jain | dadcb4d | 2017-04-21 10:03:05 | [diff] [blame] | 976 | for i := 0; i < b.N; i++ { |
Deepak Jois | 5bc2e16 | 2017-11-30 03:04:57 | [diff] [blame] | 977 | e := new(Entry) |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 978 | e.Key = make([]byte, 16) |
| 979 | e.Value = make([]byte, vsz) |
Manish R Jain | 9230dd0 | 2017-04-29 01:57:00 | [diff] [blame] | 980 | bl := new(request) |
Deepak Jois | 5bc2e16 | 2017-11-30 03:04:57 | [diff] [blame] | 981 | bl.Entries = []*Entry{e} |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 982 | |
Manish R Jain | 22e6990 | 2017-04-24 02:02:01 | [diff] [blame] | 983 | var ptrs []valuePointer |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 984 | |
Manish R Jain | 2084940 | 2017-05-17 08:36:32 | [diff] [blame] | 985 | vl.write([]*request{bl}) |
Manish R Jain | dadcb4d | 2017-04-21 10:03:05 | [diff] [blame] | 986 | ptrs = append(ptrs, bl.Ptrs...) |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 987 | |
Manish R Jain | dadcb4d | 2017-04-21 10:03:05 | [diff] [blame] | 988 | f := rand.Float32() |
| 989 | if f < rw { |
Manish R Jain | 2084940 | 2017-05-17 08:36:32 | [diff] [blame] | 990 | vl.write([]*request{bl}) |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 991 | |
Manish R Jain | dadcb4d | 2017-04-21 10:03:05 | [diff] [blame] | 992 | } else { |
| 993 | ln := len(ptrs) |
| 994 | if ln == 0 { |
| 995 | b.Fatalf("Zero length of ptrs") |
| 996 | } |
| 997 | idx := rand.Intn(ln) |
Naman Jain | a35b08d | 2021-01-14 06:01:03 | [diff] [blame] | 998 | buf, lf, err := vl.readValueBytes(ptrs[idx]) |
Manish R Jain | dadcb4d | 2017-04-21 10:03:05 | [diff] [blame] | 999 | if err != nil { |
Manish R Jain | fb0f0aa | 2017-05-03 01:17:04 | [diff] [blame] | 1000 | b.Fatalf("Benchmark Read: %v", err) |
Manish R Jain | dadcb4d | 2017-04-21 10:03:05 | [diff] [blame] | 1001 | } |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 1002 | |
balaji | a425b0e | 2019-09-24 14:06:36 | [diff] [blame] | 1003 | e, err := lf.decodeEntry(buf, ptrs[idx].Offset) |
| 1004 | require.NoError(b, err) |
Manish R Jain | 83aa09d | 2017-10-04 07:20:27 | [diff] [blame] | 1005 | if len(e.Key) != 16 { |
| 1006 | b.Fatalf("Key is invalid") |
| 1007 | } |
| 1008 | if len(e.Value) != vsz { |
| 1009 | b.Fatalf("Value is invalid") |
| 1010 | } |
balaji | a425b0e | 2019-09-24 14:06:36 | [diff] [blame] | 1011 | runCallback(db.vlog.getUnlockCallback(lf)) |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 1012 | } |
Manish R Jain | dadcb4d | 2017-04-21 10:03:05 | [diff] [blame] | 1013 | } |
Manish R Jain | 61b8119 | 2017-01-26 07:17:18 | [diff] [blame] | 1014 | }) |
| 1015 | } |
| 1016 | } |
| 1017 | } |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1018 | |
| 1019 | // Regression test for https://github.com/dgraph-io/badger/issues/817 |
Ibrahim Jarif | 0506f78 | 2020-11-02 08:01:52 | [diff] [blame] | 1020 | // This test verifies if fully corrupted memtables are deleted on reopen. |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1021 | func TestValueLogTruncate(t *testing.T) { |
Ashish Goswami | 8c85c27 | 2019-06-19 08:21:22 | [diff] [blame] | 1022 | dir, err := ioutil.TempDir("", "badger-test") |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1023 | require.NoError(t, err) |
Ibrahim Jarif | 00039ea | 2019-10-17 13:02:32 | [diff] [blame] | 1024 | defer removeDir(dir) |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1025 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 1026 | // Initialize the data directory. |
| 1027 | db, err := Open(DefaultOptions(dir)) |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1028 | require.NoError(t, err) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 1029 | // Insert 1 entry so that we have valid data in first mem file |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1030 | require.NoError(t, db.Update(func(txn *Txn) error { |
| 1031 | return txn.Set([]byte("foo"), nil) |
| 1032 | })) |
| 1033 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 1034 | fileCountBeforeCorruption := 1 |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1035 | require.NoError(t, db.Close()) |
| 1036 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 1037 | // Create two mem files with corrupted data. These will be truncated when DB starts next time |
| 1038 | require.NoError(t, ioutil.WriteFile(db.mtFilePath(2), []byte("foo"), 0664)) |
| 1039 | require.NoError(t, ioutil.WriteFile(db.mtFilePath(3), []byte("foo"), 0664)) |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1040 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 1041 | db, err = Open(DefaultOptions(dir)) |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1042 | require.NoError(t, err) |
| 1043 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 1044 | // Ensure we have only one SST file. |
| 1045 | require.Equal(t, 1, len(db.Tables())) |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1046 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 1047 | // Ensure mem file with ID 4 is zero. |
| 1048 | require.Equal(t, 4, int(db.mt.wal.fid)) |
| 1049 | fileStat, err := db.mt.wal.Fd.Stat() |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1050 | require.NoError(t, err) |
Manish R Jain | 741de05 | 2020-11-12 23:05:11 | [diff] [blame] | 1051 | require.Equal(t, 2*db.opt.MemTableSize, fileStat.Size()) |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1052 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 1053 | fileCountAfterCorruption := len(db.Tables()) + len(db.imm) + 1 // +1 for db.mt |
| 1054 | // We should have one memtable and one sst file. |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1055 | require.Equal(t, fileCountBeforeCorruption+1, fileCountAfterCorruption) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 1056 | // maxFid will be 2 because we increment the max fid on DB open everytime. |
Ibrahim Jarif | 22a9a1e | 2019-05-31 08:05:25 | [diff] [blame] | 1057 | require.Equal(t, 2, int(db.vlog.maxFid)) |
| 1058 | require.NoError(t, db.Close()) |
| 1059 | } |
Ibrahim Jarif | c5bf7e3 | 2019-07-18 09:27:21 | [diff] [blame] | 1060 | |
Ibrahim Jarif | d8e1fcf | 2019-07-19 05:07:03 | [diff] [blame] | 1061 | func TestSafeEntry(t *testing.T) { |
| 1062 | var s safeRead |
balaji | a425b0e | 2019-09-24 14:06:36 | [diff] [blame] | 1063 | s.lf = &logFile{} |
Ibrahim Jarif | d8e1fcf | 2019-07-19 05:07:03 | [diff] [blame] | 1064 | e := NewEntry([]byte("foo"), []byte("bar")) |
| 1065 | buf := bytes.NewBuffer(nil) |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 1066 | _, err := s.lf.encodeEntry(buf, e, 0) |
Ibrahim Jarif | d8e1fcf | 2019-07-19 05:07:03 | [diff] [blame] | 1067 | require.NoError(t, err) |
| 1068 | |
| 1069 | ne, err := s.Entry(buf) |
| 1070 | require.NoError(t, err) |
| 1071 | require.Equal(t, e.Key, ne.Key, "key mismatch") |
| 1072 | require.Equal(t, e.Value, ne.Value, "value mismatch") |
| 1073 | require.Equal(t, e.meta, ne.meta, "meta mismatch") |
| 1074 | require.Equal(t, e.UserMeta, ne.UserMeta, "usermeta mismatch") |
| 1075 | require.Equal(t, e.ExpiresAt, ne.ExpiresAt, "expiresAt mismatch") |
| 1076 | } |
| 1077 | |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1078 | func TestValueEntryChecksum(t *testing.T) { |
Ibrahim Jarif | 0ef91a8 | 2019-10-04 09:30:17 | [diff] [blame] | 1079 | k := []byte("KEY") |
| 1080 | v := []byte(fmt.Sprintf("val%100d", 10)) |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1081 | t.Run("ok", func(t *testing.T) { |
| 1082 | dir, err := ioutil.TempDir("", "badger-test") |
| 1083 | require.NoError(t, err) |
| 1084 | defer removeDir(dir) |
Ibrahim Jarif | 0ef91a8 | 2019-10-04 09:30:17 | [diff] [blame] | 1085 | |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1086 | opt := getTestOptions(dir) |
| 1087 | opt.VerifyValueChecksum = true |
Ibrahim Jarif | 675efcd | 2020-06-24 15:16:12 | [diff] [blame] | 1088 | opt.ValueThreshold = 32 |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1089 | db, err := Open(opt) |
| 1090 | require.NoError(t, err) |
Ibrahim Jarif | 0ef91a8 | 2019-10-04 09:30:17 | [diff] [blame] | 1091 | |
aman bansal | 5dbae10 | 2021-02-04 12:59:51 | [diff] [blame] | 1092 | require.Greater(t, int64(len(v)), db.vlog.db.valueThreshold()) |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1093 | txnSet(t, db, k, v, 0) |
| 1094 | require.NoError(t, db.Close()) |
Ibrahim Jarif | 0ef91a8 | 2019-10-04 09:30:17 | [diff] [blame] | 1095 | |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1096 | db, err = Open(opt) |
| 1097 | require.NoError(t, err) |
Ibrahim Jarif | 0ef91a8 | 2019-10-04 09:30:17 | [diff] [blame] | 1098 | |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1099 | txn := db.NewTransaction(false) |
| 1100 | entry, err := txn.Get(k) |
| 1101 | require.NoError(t, err) |
Ibrahim Jarif | 0ef91a8 | 2019-10-04 09:30:17 | [diff] [blame] | 1102 | |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1103 | x, err := entry.ValueCopy(nil) |
| 1104 | require.NoError(t, err) |
| 1105 | require.Equal(t, v, x) |
Ibrahim Jarif | 0ef91a8 | 2019-10-04 09:30:17 | [diff] [blame] | 1106 | |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1107 | require.NoError(t, db.Close()) |
| 1108 | }) |
| 1109 | // Regression test for https://github.com/dgraph-io/badger/issues/1049 |
| 1110 | t.Run("Corruption", func(t *testing.T) { |
| 1111 | dir, err := ioutil.TempDir("", "badger-test") |
| 1112 | require.NoError(t, err) |
| 1113 | defer removeDir(dir) |
| 1114 | |
| 1115 | opt := getTestOptions(dir) |
| 1116 | opt.VerifyValueChecksum = true |
Ibrahim Jarif | 675efcd | 2020-06-24 15:16:12 | [diff] [blame] | 1117 | opt.ValueThreshold = 32 |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1118 | db, err := Open(opt) |
| 1119 | require.NoError(t, err) |
| 1120 | |
aman bansal | 5dbae10 | 2021-02-04 12:59:51 | [diff] [blame] | 1121 | require.Greater(t, int64(len(v)), db.vlog.db.valueThreshold()) |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1122 | txnSet(t, db, k, v, 0) |
| 1123 | |
Manish R Jain | e3a0d29 | 2020-10-07 01:41:41 | [diff] [blame] | 1124 | path := db.vlog.fpath(1) |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1125 | require.NoError(t, db.Close()) |
| 1126 | |
| 1127 | file, err := os.OpenFile(path, os.O_RDWR, 0644) |
| 1128 | require.NoError(t, err) |
| 1129 | offset := 50 |
| 1130 | orig := make([]byte, 1) |
| 1131 | _, err = file.ReadAt(orig, int64(offset)) |
| 1132 | require.NoError(t, err) |
| 1133 | // Corrupt a single bit. |
| 1134 | _, err = file.WriteAt([]byte{7}, int64(offset)) |
| 1135 | require.NoError(t, err) |
| 1136 | require.NoError(t, file.Close()) |
| 1137 | |
| 1138 | db, err = Open(opt) |
| 1139 | require.NoError(t, err) |
| 1140 | |
| 1141 | txn := db.NewTransaction(false) |
| 1142 | entry, err := txn.Get(k) |
| 1143 | require.NoError(t, err) |
| 1144 | |
Ibrahim Jarif | feb1f5f | 2020-11-25 13:15:52 | [diff] [blame] | 1145 | // TODO(ibrahim): This test is broken since we're not returning errors |
| 1146 | // in case we cannot read the values. This is incorrect behavior but |
| 1147 | // we're doing this to debug an issue where the values are being read |
| 1148 | // from old vlog files. |
| 1149 | _, _ = entry.ValueCopy(nil) |
| 1150 | // require.Error(t, err) |
| 1151 | // require.Contains(t, err.Error(), "ErrEOF") |
| 1152 | // require.Nil(t, x) |
Ibrahim Jarif | 1ee50f7 | 2019-12-06 11:49:03 | [diff] [blame] | 1153 | |
| 1154 | require.NoError(t, db.Close()) |
| 1155 | }) |
Ibrahim Jarif | 0ef91a8 | 2019-10-04 09:30:17 | [diff] [blame] | 1156 | } |
Balaji Jinnah | d981f47 | 2020-07-10 06:25:02 | [diff] [blame] | 1157 | |
| 1158 | func TestValidateWrite(t *testing.T) { |
| 1159 | // Mocking the file size, so that we don't allocate big memory while running test. |
| 1160 | maxVlogFileSize = 400 |
| 1161 | defer func() { |
| 1162 | maxVlogFileSize = math.MaxUint32 |
| 1163 | }() |
| 1164 | |
| 1165 | bigBuf := make([]byte, maxVlogFileSize+1) |
| 1166 | log := &valueLog{ |
| 1167 | opt: DefaultOptions("."), |
| 1168 | } |
| 1169 | |
| 1170 | // Sending a request with big values which will overflow uint32. |
| 1171 | key := []byte("HelloKey") |
| 1172 | req := &request{ |
| 1173 | Entries: []*Entry{ |
| 1174 | { |
| 1175 | Key: key, |
| 1176 | Value: bigBuf, |
| 1177 | }, |
| 1178 | { |
| 1179 | Key: key, |
| 1180 | Value: bigBuf, |
| 1181 | }, |
| 1182 | { |
| 1183 | Key: key, |
| 1184 | Value: bigBuf, |
| 1185 | }, |
| 1186 | }, |
| 1187 | } |
| 1188 | |
| 1189 | err := log.validateWrites([]*request{req}) |
| 1190 | require.Error(t, err) |
| 1191 | |
| 1192 | // Testing with small values. |
| 1193 | smallBuf := make([]byte, 4) |
| 1194 | req1 := &request{ |
| 1195 | Entries: []*Entry{ |
| 1196 | { |
| 1197 | Key: key, |
| 1198 | Value: smallBuf, |
| 1199 | }, |
| 1200 | { |
| 1201 | Key: key, |
| 1202 | Value: smallBuf, |
| 1203 | }, |
| 1204 | { |
| 1205 | Key: key, |
| 1206 | Value: smallBuf, |
| 1207 | }, |
| 1208 | }, |
| 1209 | } |
| 1210 | |
| 1211 | err = log.validateWrites([]*request{req1}) |
| 1212 | require.NoError(t, err) |
| 1213 | |
| 1214 | // Batching small and big request. |
| 1215 | err = log.validateWrites([]*request{req1, req}) |
| 1216 | require.Error(t, err) |
| 1217 | } |
Naman Jain | 34f2d04 | 2020-10-15 09:24:37 | [diff] [blame] | 1218 | |
| 1219 | func TestValueLogMeta(t *testing.T) { |
| 1220 | dir, err := ioutil.TempDir("", "badger-test") |
| 1221 | y.Check(err) |
| 1222 | defer removeDir(dir) |
| 1223 | |
| 1224 | opt := getTestOptions(dir).WithValueThreshold(16) |
| 1225 | db, _ := Open(opt) |
| 1226 | defer db.Close() |
| 1227 | txn := db.NewTransaction(true) |
| 1228 | for i := 0; i < 10; i++ { |
| 1229 | k := []byte(fmt.Sprintf("key=%d", i)) |
| 1230 | v := []byte(fmt.Sprintf("val=%020d", i)) |
| 1231 | require.NoError(t, txn.SetEntry(NewEntry(k, v))) |
| 1232 | } |
| 1233 | require.NoError(t, txn.Commit()) |
| 1234 | fids := db.vlog.sortedFids() |
| 1235 | require.Equal(t, 1, len(fids)) |
| 1236 | |
| 1237 | // vlog entries must not have txn meta. |
| 1238 | db.vlog.filesMap[fids[0]].iterate(true, 0, func(e Entry, vp valuePointer) error { |
| 1239 | require.Zero(t, e.meta&(bitTxn|bitFinTxn)) |
| 1240 | return nil |
| 1241 | }) |
| 1242 | |
| 1243 | // Entries in LSM tree must have txn bit of meta set |
| 1244 | txn = db.NewTransaction(false) |
| 1245 | defer txn.Discard() |
| 1246 | iopt := DefaultIteratorOptions |
| 1247 | key := []byte("key") |
| 1248 | iopt.Prefix = key |
| 1249 | itr := txn.NewIterator(iopt) |
| 1250 | defer itr.Close() |
| 1251 | var count int |
| 1252 | for itr.Seek(key); itr.ValidForPrefix(key); itr.Next() { |
| 1253 | item := itr.Item() |
| 1254 | require.Equal(t, bitTxn, item.meta&(bitTxn|bitFinTxn)) |
| 1255 | count++ |
| 1256 | } |
| 1257 | require.Equal(t, 10, count) |
| 1258 | } |
| 1259 | |
| 1260 | // This tests asserts the condition that vlog fids start from 1. |
| 1261 | // TODO(naman): should this be changed to assert instead? |
| 1262 | func TestFirstVlogFile(t *testing.T) { |
| 1263 | dir, err := ioutil.TempDir("", "badger-test") |
| 1264 | require.NoError(t, err) |
| 1265 | defer removeDir(dir) |
| 1266 | |
| 1267 | opt := DefaultOptions(dir) |
| 1268 | db, err := Open(opt) |
| 1269 | defer db.Close() |
| 1270 | |
| 1271 | fids := db.vlog.sortedFids() |
| 1272 | require.NotZero(t, len(fids)) |
| 1273 | require.Equal(t, uint32(1), fids[0]) |
| 1274 | } |