From 3b48b16290cd19e4f87ab90c72dd28b4072dc744 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Tue, 20 Aug 2024 14:06:00 +0200 Subject: [PATCH] core/rawdb: drop MigrateTable (#30331) These are the leftovers from #24028. --- core/rawdb/database.go | 6 -- core/rawdb/freezer.go | 114 ------------------------------- core/rawdb/freezer_memory.go | 7 -- core/rawdb/freezer_resettable.go | 9 --- core/rawdb/freezer_test.go | 83 ---------------------- core/rawdb/table.go | 6 -- ethdb/database.go | 5 -- ethdb/remotedb/remotedb.go | 4 -- 8 files changed, 234 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 831016da15..13233406fe 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -149,12 +149,6 @@ func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReaderOp) error) return fn(db) } -// MigrateTable processes the entries in a given table in sequence -// converting them to a new format if they're of an old format. -func (db *nofreezedb) MigrateTable(kind string, convert convertLegacyFn) error { - return errNotSupported -} - // AncientDatadir returns an error as we don't have a backing chain freezer. func (db *nofreezedb) AncientDatadir() (string, error) { return "", errNotSupported diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 0f28782db9..6c050ee625 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -24,9 +24,7 @@ import ( "path/filepath" "sync" "sync/atomic" - "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -389,115 +387,3 @@ func (f *Freezer) repair() error { f.tail.Store(tail) return nil } - -// convertLegacyFn takes a raw freezer entry in an older format and -// returns it in the new format. -type convertLegacyFn = func([]byte) ([]byte, error) - -// MigrateTable processes the entries in a given table in sequence -// converting them to a new format if they're of an old format. -func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error { - if f.readonly { - return errReadOnly - } - f.writeLock.Lock() - defer f.writeLock.Unlock() - - table, ok := f.tables[kind] - if !ok { - return errUnknownTable - } - // forEach iterates every entry in the table serially and in order, calling `fn` - // with the item as argument. If `fn` returns an error the iteration stops - // and that error will be returned. - forEach := func(t *freezerTable, offset uint64, fn func(uint64, []byte) error) error { - var ( - items = t.items.Load() - batchSize = uint64(1024) - maxBytes = uint64(1024 * 1024) - ) - for i := offset; i < items; { - if i+batchSize > items { - batchSize = items - i - } - data, err := t.RetrieveItems(i, batchSize, maxBytes) - if err != nil { - return err - } - for j, item := range data { - if err := fn(i+uint64(j), item); err != nil { - return err - } - } - i += uint64(len(data)) - } - return nil - } - // TODO(s1na): This is a sanity-check since as of now no process does tail-deletion. But the migration - // process assumes no deletion at tail and needs to be modified to account for that. - if table.itemOffset.Load() > 0 || table.itemHidden.Load() > 0 { - return errors.New("migration not supported for tail-deleted freezers") - } - ancientsPath := filepath.Dir(table.index.Name()) - // Set up new dir for the migrated table, the content of which - // we'll at the end move over to the ancients dir. - migrationPath := filepath.Join(ancientsPath, "migration") - newTable, err := newFreezerTable(migrationPath, kind, table.noCompression, false) - if err != nil { - return err - } - var ( - batch = newTable.newBatch() - out []byte - start = time.Now() - logged = time.Now() - offset = newTable.items.Load() - ) - if offset > 0 { - log.Info("found previous migration attempt", "migrated", offset) - } - // Iterate through entries and transform them - if err := forEach(table, offset, func(i uint64, blob []byte) error { - if i%10000 == 0 && time.Since(logged) > 16*time.Second { - log.Info("Processing legacy elements", "count", i, "elapsed", common.PrettyDuration(time.Since(start))) - logged = time.Now() - } - out, err = convert(blob) - if err != nil { - return err - } - if err := batch.AppendRaw(i, out); err != nil { - return err - } - return nil - }); err != nil { - return err - } - if err := batch.commit(); err != nil { - return err - } - log.Info("Replacing old table files with migrated ones", "elapsed", common.PrettyDuration(time.Since(start))) - // Release and delete old table files. Note this won't - // delete the index file. - table.releaseFilesAfter(0, true) - - if err := newTable.Close(); err != nil { - return err - } - files, err := os.ReadDir(migrationPath) - if err != nil { - return err - } - // Move migrated files to ancients dir. - for _, f := range files { - // This will replace the old index file as a side-effect. - if err := os.Rename(filepath.Join(migrationPath, f.Name()), filepath.Join(ancientsPath, f.Name())); err != nil { - return err - } - } - // Delete by now empty dir. - if err := os.Remove(migrationPath); err != nil { - return err - } - return nil -} diff --git a/core/rawdb/freezer_memory.go b/core/rawdb/freezer_memory.go index 954b58e874..cd39101514 100644 --- a/core/rawdb/freezer_memory.go +++ b/core/rawdb/freezer_memory.go @@ -394,13 +394,6 @@ func (f *MemoryFreezer) Sync() error { return nil } -// MigrateTable processes and migrates entries of a given table to a new format. -// The second argument is a function that takes a raw entry and returns it -// in the newest format. -func (f *MemoryFreezer) MigrateTable(string, func([]byte) ([]byte, error)) error { - return errors.New("not implemented") -} - // Close releases all the sources held by the memory freezer. It will panic if // any following invocation is made to a closed freezer. func (f *MemoryFreezer) Close() error { diff --git a/core/rawdb/freezer_resettable.go b/core/rawdb/freezer_resettable.go index 6f8541f43b..b147995066 100644 --- a/core/rawdb/freezer_resettable.go +++ b/core/rawdb/freezer_resettable.go @@ -202,15 +202,6 @@ func (f *resettableFreezer) Sync() error { return f.freezer.Sync() } -// MigrateTable processes the entries in a given table in sequence -// converting them to a new format if they're of an old format. -func (f *resettableFreezer) MigrateTable(kind string, convert convertLegacyFn) error { - f.lock.RLock() - defer f.lock.RUnlock() - - return f.freezer.MigrateTable(kind, convert) -} - // cleanup removes the directory located in the specified path // has the name with deletion marker suffix. func cleanup(path string) error { diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index 72d1417200..7d82ea305f 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -22,8 +22,6 @@ import ( "fmt" "math/big" "math/rand" - "os" - "path/filepath" "sync" "testing" @@ -379,87 +377,6 @@ func checkAncientCount(t *testing.T, f *Freezer, kind string, n uint64) { } } -func TestRenameWindows(t *testing.T) { - var ( - fname = "file.bin" - fname2 = "file2.bin" - data = []byte{1, 2, 3, 4} - data2 = []byte{2, 3, 4, 5} - data3 = []byte{3, 5, 6, 7} - dataLen = 4 - ) - - // Create 2 temp dirs - dir1 := t.TempDir() - dir2 := t.TempDir() - - // Create file in dir1 and fill with data - f, err := os.Create(filepath.Join(dir1, fname)) - if err != nil { - t.Fatal(err) - } - f2, err := os.Create(filepath.Join(dir1, fname2)) - if err != nil { - t.Fatal(err) - } - f3, err := os.Create(filepath.Join(dir2, fname2)) - if err != nil { - t.Fatal(err) - } - if _, err := f.Write(data); err != nil { - t.Fatal(err) - } - if _, err := f2.Write(data2); err != nil { - t.Fatal(err) - } - if _, err := f3.Write(data3); err != nil { - t.Fatal(err) - } - if err := f.Close(); err != nil { - t.Fatal(err) - } - if err := f2.Close(); err != nil { - t.Fatal(err) - } - if err := f3.Close(); err != nil { - t.Fatal(err) - } - if err := os.Rename(f.Name(), filepath.Join(dir2, fname)); err != nil { - t.Fatal(err) - } - if err := os.Rename(f2.Name(), filepath.Join(dir2, fname2)); err != nil { - t.Fatal(err) - } - - // Check file contents - f, err = os.Open(filepath.Join(dir2, fname)) - if err != nil { - t.Fatal(err) - } - defer f.Close() - defer os.Remove(f.Name()) - buf := make([]byte, dataLen) - if _, err := f.Read(buf); err != nil { - t.Fatal(err) - } - if !bytes.Equal(buf, data) { - t.Errorf("unexpected file contents. Got %v\n", buf) - } - - f, err = os.Open(filepath.Join(dir2, fname2)) - if err != nil { - t.Fatal(err) - } - defer f.Close() - defer os.Remove(f.Name()) - if _, err := f.Read(buf); err != nil { - t.Fatal(err) - } - if !bytes.Equal(buf, data2) { - t.Errorf("unexpected file contents. Got %v\n", buf) - } -} - func TestFreezerCloseSync(t *testing.T) { t.Parallel() f, _ := newFreezerForTesting(t, map[string]bool{"a": true, "b": true}) diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 0a25c5bcdd..bc1d354d10 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -113,12 +113,6 @@ func (t *table) Sync() error { return t.db.Sync() } -// MigrateTable processes the entries in a given table in sequence -// converting them to a new format if they're of an old format. -func (t *table) MigrateTable(kind string, convert convertLegacyFn) error { - return t.db.MigrateTable(kind, convert) -} - // AncientDatadir returns the ancient datadir of the underlying database. func (t *table) AncientDatadir() (string, error) { return t.db.AncientDatadir() diff --git a/ethdb/database.go b/ethdb/database.go index 89c793d0be..c6e76fd2fe 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -124,11 +124,6 @@ type AncientWriter interface { // Sync flushes all in-memory ancient store data to disk. Sync() error - - // MigrateTable processes and migrates entries of a given table to a new format. - // The second argument is a function that takes a raw entry and returns it - // in the newest format. - MigrateTable(string, func([]byte) ([]byte, error)) error } // AncientWriteOp is given to the function argument of ModifyAncients. diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index c8b76eab4a..d0f018cb01 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -110,10 +110,6 @@ func (db *Database) Sync() error { return nil } -func (db *Database) MigrateTable(s string, f func([]byte) ([]byte, error)) error { - panic("not supported") -} - func (db *Database) NewBatch() ethdb.Batch { panic("not supported") }