mirror of https://github.com/ethereum/go-ethereum
core/rawdb, cmd, ethdb, eth: implement freezer tail deletion (#23954)
* core/rawdb, cmd, ethdb, eth: implement freezer tail deletion * core/rawdb: address comments from martin and sina * core/rawdb: fixes cornercase in tail deletion * core/rawdb: separate metadata into a standalone file * core/rawdb: remove unused code * core/rawdb: add random test * core/rawdb: polish code * core/rawdb: fsync meta file before manipulating the index * core/rawdb: fix typo * core/rawdb: address commentspull/24529/head
rjl493456442
3 years ago
committed by
GitHub
parent
8c8a9e5ca1
commit
538a868384
@ -0,0 +1,109 @@ |
|||||||
|
// Copyright 2022 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
|
||||||
|
|
||||||
|
package rawdb |
||||||
|
|
||||||
|
import ( |
||||||
|
"io" |
||||||
|
"os" |
||||||
|
|
||||||
|
"github.com/ethereum/go-ethereum/log" |
||||||
|
"github.com/ethereum/go-ethereum/rlp" |
||||||
|
) |
||||||
|
|
||||||
|
const freezerVersion = 1 // The initial version tag of freezer table metadata
|
||||||
|
|
||||||
|
// freezerTableMeta wraps all the metadata of the freezer table.
|
||||||
|
type freezerTableMeta struct { |
||||||
|
// Version is the versioning descriptor of the freezer table.
|
||||||
|
Version uint16 |
||||||
|
|
||||||
|
// VirtualTail indicates how many items have been marked as deleted.
|
||||||
|
// Its value is equal to the number of items removed from the table
|
||||||
|
// plus the number of items hidden in the table, so it should never
|
||||||
|
// be lower than the "actual tail".
|
||||||
|
VirtualTail uint64 |
||||||
|
} |
||||||
|
|
||||||
|
// newMetadata initializes the metadata object with the given virtual tail.
|
||||||
|
func newMetadata(tail uint64) *freezerTableMeta { |
||||||
|
return &freezerTableMeta{ |
||||||
|
Version: freezerVersion, |
||||||
|
VirtualTail: tail, |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// readMetadata reads the metadata of the freezer table from the
|
||||||
|
// given metadata file.
|
||||||
|
func readMetadata(file *os.File) (*freezerTableMeta, error) { |
||||||
|
_, err := file.Seek(0, io.SeekStart) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
var meta freezerTableMeta |
||||||
|
if err := rlp.Decode(file, &meta); err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
return &meta, nil |
||||||
|
} |
||||||
|
|
||||||
|
// writeMetadata writes the metadata of the freezer table into the
|
||||||
|
// given metadata file.
|
||||||
|
func writeMetadata(file *os.File, meta *freezerTableMeta) error { |
||||||
|
_, err := file.Seek(0, io.SeekStart) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
return rlp.Encode(file, meta) |
||||||
|
} |
||||||
|
|
||||||
|
// loadMetadata loads the metadata from the given metadata file.
|
||||||
|
// Initializes the metadata file with the given "actual tail" if
|
||||||
|
// it's empty.
|
||||||
|
func loadMetadata(file *os.File, tail uint64) (*freezerTableMeta, error) { |
||||||
|
stat, err := file.Stat() |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
// Write the metadata with the given actual tail into metadata file
|
||||||
|
// if it's non-existent. There are two possible scenarios here:
|
||||||
|
// - the freezer table is empty
|
||||||
|
// - the freezer table is legacy
|
||||||
|
// In both cases, write the meta into the file with the actual tail
|
||||||
|
// as the virtual tail.
|
||||||
|
if stat.Size() == 0 { |
||||||
|
m := newMetadata(tail) |
||||||
|
if err := writeMetadata(file, m); err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
return m, nil |
||||||
|
} |
||||||
|
m, err := readMetadata(file) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
// Update the virtual tail with the given actual tail if it's even
|
||||||
|
// lower than it. Theoretically it shouldn't happen at all, print
|
||||||
|
// a warning here.
|
||||||
|
if m.VirtualTail < tail { |
||||||
|
log.Warn("Updated virtual tail", "have", m.VirtualTail, "now", tail) |
||||||
|
m.VirtualTail = tail |
||||||
|
if err := writeMetadata(file, m); err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
} |
||||||
|
return m, nil |
||||||
|
} |
||||||
@ -0,0 +1,61 @@ |
|||||||
|
// Copyright 2022 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
|
||||||
|
|
||||||
|
package rawdb |
||||||
|
|
||||||
|
import ( |
||||||
|
"io/ioutil" |
||||||
|
"os" |
||||||
|
"testing" |
||||||
|
) |
||||||
|
|
||||||
|
func TestReadWriteFreezerTableMeta(t *testing.T) { |
||||||
|
f, err := ioutil.TempFile(os.TempDir(), "*") |
||||||
|
if err != nil { |
||||||
|
t.Fatalf("Failed to create file %v", err) |
||||||
|
} |
||||||
|
err = writeMetadata(f, newMetadata(100)) |
||||||
|
if err != nil { |
||||||
|
t.Fatalf("Failed to write metadata %v", err) |
||||||
|
} |
||||||
|
meta, err := readMetadata(f) |
||||||
|
if err != nil { |
||||||
|
t.Fatalf("Failed to read metadata %v", err) |
||||||
|
} |
||||||
|
if meta.Version != freezerVersion { |
||||||
|
t.Fatalf("Unexpected version field") |
||||||
|
} |
||||||
|
if meta.VirtualTail != uint64(100) { |
||||||
|
t.Fatalf("Unexpected virtual tail field") |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func TestInitializeFreezerTableMeta(t *testing.T) { |
||||||
|
f, err := ioutil.TempFile(os.TempDir(), "*") |
||||||
|
if err != nil { |
||||||
|
t.Fatalf("Failed to create file %v", err) |
||||||
|
} |
||||||
|
meta, err := loadMetadata(f, uint64(100)) |
||||||
|
if err != nil { |
||||||
|
t.Fatalf("Failed to read metadata %v", err) |
||||||
|
} |
||||||
|
if meta.Version != freezerVersion { |
||||||
|
t.Fatalf("Unexpected version field") |
||||||
|
} |
||||||
|
if meta.VirtualTail != uint64(100) { |
||||||
|
t.Fatalf("Unexpected virtual tail field") |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,120 @@ |
|||||||
|
// Copyright 2022 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package rawdb |
||||||
|
|
||||||
|
import ( |
||||||
|
"io" |
||||||
|
"io/ioutil" |
||||||
|
"os" |
||||||
|
"path/filepath" |
||||||
|
) |
||||||
|
|
||||||
|
// copyFrom copies data from 'srcPath' at offset 'offset' into 'destPath'.
|
||||||
|
// The 'destPath' is created if it doesn't exist, otherwise it is overwritten.
|
||||||
|
// Before the copy is executed, there is a callback can be registered to
|
||||||
|
// manipulate the dest file.
|
||||||
|
// It is perfectly valid to have destPath == srcPath.
|
||||||
|
func copyFrom(srcPath, destPath string, offset uint64, before func(f *os.File) error) error { |
||||||
|
// Create a temp file in the same dir where we want it to wind up
|
||||||
|
f, err := ioutil.TempFile(filepath.Dir(destPath), "*") |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
fname := f.Name() |
||||||
|
|
||||||
|
// Clean up the leftover file
|
||||||
|
defer func() { |
||||||
|
if f != nil { |
||||||
|
f.Close() |
||||||
|
} |
||||||
|
os.Remove(fname) |
||||||
|
}() |
||||||
|
// Apply the given function if it's not nil before we copy
|
||||||
|
// the content from the src.
|
||||||
|
if before != nil { |
||||||
|
if err := before(f); err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
} |
||||||
|
// Open the source file
|
||||||
|
src, err := os.Open(srcPath) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
if _, err = src.Seek(int64(offset), 0); err != nil { |
||||||
|
src.Close() |
||||||
|
return err |
||||||
|
} |
||||||
|
// io.Copy uses 32K buffer internally.
|
||||||
|
_, err = io.Copy(f, src) |
||||||
|
if err != nil { |
||||||
|
src.Close() |
||||||
|
return err |
||||||
|
} |
||||||
|
// Rename the temporary file to the specified dest name.
|
||||||
|
// src may be same as dest, so needs to be closed before
|
||||||
|
// we do the final move.
|
||||||
|
src.Close() |
||||||
|
|
||||||
|
if err := f.Close(); err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
f = nil |
||||||
|
|
||||||
|
if err := os.Rename(fname, destPath); err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
// openFreezerFileForAppend opens a freezer table file and seeks to the end
|
||||||
|
func openFreezerFileForAppend(filename string) (*os.File, error) { |
||||||
|
// Open the file without the O_APPEND flag
|
||||||
|
// because it has differing behaviour during Truncate operations
|
||||||
|
// on different OS's
|
||||||
|
file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0644) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
// Seek to end for append
|
||||||
|
if _, err = file.Seek(0, io.SeekEnd); err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
return file, nil |
||||||
|
} |
||||||
|
|
||||||
|
// openFreezerFileForReadOnly opens a freezer table file for read only access
|
||||||
|
func openFreezerFileForReadOnly(filename string) (*os.File, error) { |
||||||
|
return os.OpenFile(filename, os.O_RDONLY, 0644) |
||||||
|
} |
||||||
|
|
||||||
|
// openFreezerFileTruncated opens a freezer table making sure it is truncated
|
||||||
|
func openFreezerFileTruncated(filename string) (*os.File, error) { |
||||||
|
return os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) |
||||||
|
} |
||||||
|
|
||||||
|
// truncateFreezerFile resizes a freezer table file and seeks to the end
|
||||||
|
func truncateFreezerFile(file *os.File, size int64) error { |
||||||
|
if err := file.Truncate(size); err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
// Seek to end for append
|
||||||
|
if _, err := file.Seek(0, io.SeekEnd); err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
return nil |
||||||
|
} |
||||||
@ -0,0 +1,76 @@ |
|||||||
|
// Copyright 2022 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package rawdb |
||||||
|
|
||||||
|
import ( |
||||||
|
"bytes" |
||||||
|
"io/ioutil" |
||||||
|
"os" |
||||||
|
"testing" |
||||||
|
) |
||||||
|
|
||||||
|
func TestCopyFrom(t *testing.T) { |
||||||
|
var ( |
||||||
|
content = []byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8} |
||||||
|
prefix = []byte{0x9, 0xa, 0xb, 0xc, 0xd, 0xf} |
||||||
|
) |
||||||
|
var cases = []struct { |
||||||
|
src, dest string |
||||||
|
offset uint64 |
||||||
|
writePrefix bool |
||||||
|
}{ |
||||||
|
{"foo", "bar", 0, false}, |
||||||
|
{"foo", "bar", 1, false}, |
||||||
|
{"foo", "bar", 8, false}, |
||||||
|
{"foo", "foo", 0, false}, |
||||||
|
{"foo", "foo", 1, false}, |
||||||
|
{"foo", "foo", 8, false}, |
||||||
|
{"foo", "bar", 0, true}, |
||||||
|
{"foo", "bar", 1, true}, |
||||||
|
{"foo", "bar", 8, true}, |
||||||
|
} |
||||||
|
for _, c := range cases { |
||||||
|
ioutil.WriteFile(c.src, content, 0644) |
||||||
|
|
||||||
|
if err := copyFrom(c.src, c.dest, c.offset, func(f *os.File) error { |
||||||
|
if !c.writePrefix { |
||||||
|
return nil |
||||||
|
} |
||||||
|
f.Write(prefix) |
||||||
|
return nil |
||||||
|
}); err != nil { |
||||||
|
os.Remove(c.src) |
||||||
|
t.Fatalf("Failed to copy %v", err) |
||||||
|
} |
||||||
|
|
||||||
|
blob, err := ioutil.ReadFile(c.dest) |
||||||
|
if err != nil { |
||||||
|
os.Remove(c.src) |
||||||
|
os.Remove(c.dest) |
||||||
|
t.Fatalf("Failed to read %v", err) |
||||||
|
} |
||||||
|
want := content[c.offset:] |
||||||
|
if c.writePrefix { |
||||||
|
want = append(prefix, want...) |
||||||
|
} |
||||||
|
if !bytes.Equal(blob, want) { |
||||||
|
t.Fatal("Unexpected value") |
||||||
|
} |
||||||
|
os.Remove(c.src) |
||||||
|
os.Remove(c.dest) |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue