mirror of https://github.com/ethereum/go-ethereum
core/rawdb, cmd, ethdb, eth: implement freezer tail deletion (#23954)
* core/rawdb, cmd, ethdb, eth: implement freezer tail deletion * core/rawdb: address comments from martin and sina * core/rawdb: fixes cornercase in tail deletion * core/rawdb: separate metadata into a standalone file * core/rawdb: remove unused code * core/rawdb: add random test * core/rawdb: polish code * core/rawdb: fsync meta file before manipulating the index * core/rawdb: fix typo * core/rawdb: address commentspull/24529/head
rjl493456442
3 years ago
committed by
GitHub
parent
8c8a9e5ca1
commit
538a868384
@ -0,0 +1,109 @@ |
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"io" |
||||
"os" |
||||
|
||||
"github.com/ethereum/go-ethereum/log" |
||||
"github.com/ethereum/go-ethereum/rlp" |
||||
) |
||||
|
||||
const freezerVersion = 1 // The initial version tag of freezer table metadata
|
||||
|
||||
// freezerTableMeta wraps all the metadata of the freezer table.
|
||||
type freezerTableMeta struct { |
||||
// Version is the versioning descriptor of the freezer table.
|
||||
Version uint16 |
||||
|
||||
// VirtualTail indicates how many items have been marked as deleted.
|
||||
// Its value is equal to the number of items removed from the table
|
||||
// plus the number of items hidden in the table, so it should never
|
||||
// be lower than the "actual tail".
|
||||
VirtualTail uint64 |
||||
} |
||||
|
||||
// newMetadata initializes the metadata object with the given virtual tail.
|
||||
func newMetadata(tail uint64) *freezerTableMeta { |
||||
return &freezerTableMeta{ |
||||
Version: freezerVersion, |
||||
VirtualTail: tail, |
||||
} |
||||
} |
||||
|
||||
// readMetadata reads the metadata of the freezer table from the
|
||||
// given metadata file.
|
||||
func readMetadata(file *os.File) (*freezerTableMeta, error) { |
||||
_, err := file.Seek(0, io.SeekStart) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
var meta freezerTableMeta |
||||
if err := rlp.Decode(file, &meta); err != nil { |
||||
return nil, err |
||||
} |
||||
return &meta, nil |
||||
} |
||||
|
||||
// writeMetadata writes the metadata of the freezer table into the
|
||||
// given metadata file.
|
||||
func writeMetadata(file *os.File, meta *freezerTableMeta) error { |
||||
_, err := file.Seek(0, io.SeekStart) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return rlp.Encode(file, meta) |
||||
} |
||||
|
||||
// loadMetadata loads the metadata from the given metadata file.
|
||||
// Initializes the metadata file with the given "actual tail" if
|
||||
// it's empty.
|
||||
func loadMetadata(file *os.File, tail uint64) (*freezerTableMeta, error) { |
||||
stat, err := file.Stat() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
// Write the metadata with the given actual tail into metadata file
|
||||
// if it's non-existent. There are two possible scenarios here:
|
||||
// - the freezer table is empty
|
||||
// - the freezer table is legacy
|
||||
// In both cases, write the meta into the file with the actual tail
|
||||
// as the virtual tail.
|
||||
if stat.Size() == 0 { |
||||
m := newMetadata(tail) |
||||
if err := writeMetadata(file, m); err != nil { |
||||
return nil, err |
||||
} |
||||
return m, nil |
||||
} |
||||
m, err := readMetadata(file) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
// Update the virtual tail with the given actual tail if it's even
|
||||
// lower than it. Theoretically it shouldn't happen at all, print
|
||||
// a warning here.
|
||||
if m.VirtualTail < tail { |
||||
log.Warn("Updated virtual tail", "have", m.VirtualTail, "now", tail) |
||||
m.VirtualTail = tail |
||||
if err := writeMetadata(file, m); err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
return m, nil |
||||
} |
||||
@ -0,0 +1,61 @@ |
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"io/ioutil" |
||||
"os" |
||||
"testing" |
||||
) |
||||
|
||||
func TestReadWriteFreezerTableMeta(t *testing.T) { |
||||
f, err := ioutil.TempFile(os.TempDir(), "*") |
||||
if err != nil { |
||||
t.Fatalf("Failed to create file %v", err) |
||||
} |
||||
err = writeMetadata(f, newMetadata(100)) |
||||
if err != nil { |
||||
t.Fatalf("Failed to write metadata %v", err) |
||||
} |
||||
meta, err := readMetadata(f) |
||||
if err != nil { |
||||
t.Fatalf("Failed to read metadata %v", err) |
||||
} |
||||
if meta.Version != freezerVersion { |
||||
t.Fatalf("Unexpected version field") |
||||
} |
||||
if meta.VirtualTail != uint64(100) { |
||||
t.Fatalf("Unexpected virtual tail field") |
||||
} |
||||
} |
||||
|
||||
func TestInitializeFreezerTableMeta(t *testing.T) { |
||||
f, err := ioutil.TempFile(os.TempDir(), "*") |
||||
if err != nil { |
||||
t.Fatalf("Failed to create file %v", err) |
||||
} |
||||
meta, err := loadMetadata(f, uint64(100)) |
||||
if err != nil { |
||||
t.Fatalf("Failed to read metadata %v", err) |
||||
} |
||||
if meta.Version != freezerVersion { |
||||
t.Fatalf("Unexpected version field") |
||||
} |
||||
if meta.VirtualTail != uint64(100) { |
||||
t.Fatalf("Unexpected virtual tail field") |
||||
} |
||||
} |
||||
@ -0,0 +1,120 @@ |
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"io" |
||||
"io/ioutil" |
||||
"os" |
||||
"path/filepath" |
||||
) |
||||
|
||||
// copyFrom copies data from 'srcPath' at offset 'offset' into 'destPath'.
|
||||
// The 'destPath' is created if it doesn't exist, otherwise it is overwritten.
|
||||
// Before the copy is executed, there is a callback can be registered to
|
||||
// manipulate the dest file.
|
||||
// It is perfectly valid to have destPath == srcPath.
|
||||
func copyFrom(srcPath, destPath string, offset uint64, before func(f *os.File) error) error { |
||||
// Create a temp file in the same dir where we want it to wind up
|
||||
f, err := ioutil.TempFile(filepath.Dir(destPath), "*") |
||||
if err != nil { |
||||
return err |
||||
} |
||||
fname := f.Name() |
||||
|
||||
// Clean up the leftover file
|
||||
defer func() { |
||||
if f != nil { |
||||
f.Close() |
||||
} |
||||
os.Remove(fname) |
||||
}() |
||||
// Apply the given function if it's not nil before we copy
|
||||
// the content from the src.
|
||||
if before != nil { |
||||
if err := before(f); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
// Open the source file
|
||||
src, err := os.Open(srcPath) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if _, err = src.Seek(int64(offset), 0); err != nil { |
||||
src.Close() |
||||
return err |
||||
} |
||||
// io.Copy uses 32K buffer internally.
|
||||
_, err = io.Copy(f, src) |
||||
if err != nil { |
||||
src.Close() |
||||
return err |
||||
} |
||||
// Rename the temporary file to the specified dest name.
|
||||
// src may be same as dest, so needs to be closed before
|
||||
// we do the final move.
|
||||
src.Close() |
||||
|
||||
if err := f.Close(); err != nil { |
||||
return err |
||||
} |
||||
f = nil |
||||
|
||||
if err := os.Rename(fname, destPath); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// openFreezerFileForAppend opens a freezer table file and seeks to the end
|
||||
func openFreezerFileForAppend(filename string) (*os.File, error) { |
||||
// Open the file without the O_APPEND flag
|
||||
// because it has differing behaviour during Truncate operations
|
||||
// on different OS's
|
||||
file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0644) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
// Seek to end for append
|
||||
if _, err = file.Seek(0, io.SeekEnd); err != nil { |
||||
return nil, err |
||||
} |
||||
return file, nil |
||||
} |
||||
|
||||
// openFreezerFileForReadOnly opens a freezer table file for read only access
|
||||
func openFreezerFileForReadOnly(filename string) (*os.File, error) { |
||||
return os.OpenFile(filename, os.O_RDONLY, 0644) |
||||
} |
||||
|
||||
// openFreezerFileTruncated opens a freezer table making sure it is truncated
|
||||
func openFreezerFileTruncated(filename string) (*os.File, error) { |
||||
return os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) |
||||
} |
||||
|
||||
// truncateFreezerFile resizes a freezer table file and seeks to the end
|
||||
func truncateFreezerFile(file *os.File, size int64) error { |
||||
if err := file.Truncate(size); err != nil { |
||||
return err |
||||
} |
||||
// Seek to end for append
|
||||
if _, err := file.Seek(0, io.SeekEnd); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,76 @@ |
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"bytes" |
||||
"io/ioutil" |
||||
"os" |
||||
"testing" |
||||
) |
||||
|
||||
func TestCopyFrom(t *testing.T) { |
||||
var ( |
||||
content = []byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8} |
||||
prefix = []byte{0x9, 0xa, 0xb, 0xc, 0xd, 0xf} |
||||
) |
||||
var cases = []struct { |
||||
src, dest string |
||||
offset uint64 |
||||
writePrefix bool |
||||
}{ |
||||
{"foo", "bar", 0, false}, |
||||
{"foo", "bar", 1, false}, |
||||
{"foo", "bar", 8, false}, |
||||
{"foo", "foo", 0, false}, |
||||
{"foo", "foo", 1, false}, |
||||
{"foo", "foo", 8, false}, |
||||
{"foo", "bar", 0, true}, |
||||
{"foo", "bar", 1, true}, |
||||
{"foo", "bar", 8, true}, |
||||
} |
||||
for _, c := range cases { |
||||
ioutil.WriteFile(c.src, content, 0644) |
||||
|
||||
if err := copyFrom(c.src, c.dest, c.offset, func(f *os.File) error { |
||||
if !c.writePrefix { |
||||
return nil |
||||
} |
||||
f.Write(prefix) |
||||
return nil |
||||
}); err != nil { |
||||
os.Remove(c.src) |
||||
t.Fatalf("Failed to copy %v", err) |
||||
} |
||||
|
||||
blob, err := ioutil.ReadFile(c.dest) |
||||
if err != nil { |
||||
os.Remove(c.src) |
||||
os.Remove(c.dest) |
||||
t.Fatalf("Failed to read %v", err) |
||||
} |
||||
want := content[c.offset:] |
||||
if c.writePrefix { |
||||
want = append(prefix, want...) |
||||
} |
||||
if !bytes.Equal(blob, want) { |
||||
t.Fatal("Unexpected value") |
||||
} |
||||
os.Remove(c.src) |
||||
os.Remove(c.dest) |
||||
} |
||||
} |
||||
Loading…
Reference in new issue