mirror of https://github.com/ethereum/go-ethereum
Merge pull request #1574 from fjl/fdtrack
fdtrack: hack to track file descriptor usagepull/1587/head
Jeffrey Wilcke
9 years ago
commit
b01f2c29ae
5
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go
generated
vendored
5
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go
generated
vendored
@ -0,0 +1,112 @@ |
||||
// Copyright 2015 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// Package fdtrack logs statistics about open file descriptors.
|
||||
package fdtrack |
||||
|
||||
import ( |
||||
"fmt" |
||||
"net" |
||||
"sort" |
||||
"sync" |
||||
"time" |
||||
|
||||
"github.com/ethereum/go-ethereum/logger/glog" |
||||
) |
||||
|
||||
var ( |
||||
mutex sync.Mutex |
||||
all = make(map[string]int) |
||||
) |
||||
|
||||
func Open(desc string) { |
||||
mutex.Lock() |
||||
all[desc] += 1 |
||||
mutex.Unlock() |
||||
} |
||||
|
||||
func Close(desc string) { |
||||
mutex.Lock() |
||||
defer mutex.Unlock() |
||||
if c, ok := all[desc]; ok { |
||||
if c == 1 { |
||||
delete(all, desc) |
||||
} else { |
||||
all[desc]-- |
||||
} |
||||
} |
||||
} |
||||
|
||||
func WrapListener(desc string, l net.Listener) net.Listener { |
||||
Open(desc) |
||||
return &wrappedListener{l, desc} |
||||
} |
||||
|
||||
type wrappedListener struct { |
||||
net.Listener |
||||
desc string |
||||
} |
||||
|
||||
func (w *wrappedListener) Accept() (net.Conn, error) { |
||||
c, err := w.Listener.Accept() |
||||
if err == nil { |
||||
c = WrapConn(w.desc, c) |
||||
} |
||||
return c, err |
||||
} |
||||
|
||||
func (w *wrappedListener) Close() error { |
||||
err := w.Listener.Close() |
||||
if err == nil { |
||||
Close(w.desc) |
||||
} |
||||
return err |
||||
} |
||||
|
||||
func WrapConn(desc string, conn net.Conn) net.Conn { |
||||
Open(desc) |
||||
return &wrappedConn{conn, desc} |
||||
} |
||||
|
||||
type wrappedConn struct { |
||||
net.Conn |
||||
desc string |
||||
} |
||||
|
||||
func (w *wrappedConn) Close() error { |
||||
err := w.Conn.Close() |
||||
if err == nil { |
||||
Close(w.desc) |
||||
} |
||||
return err |
||||
} |
||||
|
||||
func Start() { |
||||
go func() { |
||||
for range time.Tick(15 * time.Second) { |
||||
mutex.Lock() |
||||
var sum, tracked = 0, []string{} |
||||
for what, n := range all { |
||||
sum += n |
||||
tracked = append(tracked, fmt.Sprintf("%s:%d", what, n)) |
||||
} |
||||
mutex.Unlock() |
||||
used, _ := fdusage() |
||||
sort.Strings(tracked) |
||||
glog.Infof("fd usage %d/%d, tracked %d %v", used, fdlimit(), sum, tracked) |
||||
} |
||||
}() |
||||
} |
||||
@ -0,0 +1,29 @@ |
||||
// Copyright 2015 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// +build !linux,!darwin
|
||||
|
||||
package fdtrack |
||||
|
||||
import "errors" |
||||
|
||||
func fdlimit() int { |
||||
return 0 |
||||
} |
||||
|
||||
func fdusage() (int, error) { |
||||
return 0, errors.New("not implemented") |
||||
} |
||||
@ -0,0 +1,72 @@ |
||||
// Copyright 2015 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// +build darwin
|
||||
|
||||
package fdtrack |
||||
|
||||
import ( |
||||
"os" |
||||
"syscall" |
||||
"unsafe" |
||||
) |
||||
|
||||
// #cgo CFLAGS: -lproc
|
||||
// #include <libproc.h>
|
||||
// #include <stdlib.h>
|
||||
import "C" |
||||
|
||||
func fdlimit() int { |
||||
var nofile syscall.Rlimit |
||||
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &nofile); err != nil { |
||||
return 0 |
||||
} |
||||
return int(nofile.Cur) |
||||
} |
||||
|
||||
func fdusage() (int, error) { |
||||
pid := C.int(os.Getpid()) |
||||
// Query for a rough estimate on the amout of data that
|
||||
// proc_pidinfo will return.
|
||||
rlen, err := C.proc_pidinfo(pid, C.PROC_PIDLISTFDS, 0, nil, 0) |
||||
if rlen <= 0 { |
||||
return 0, err |
||||
} |
||||
// Load the list of file descriptors. We don't actually care about
|
||||
// the content, only about the size. Since the number of fds can
|
||||
// change while we're reading them, the loop enlarges the buffer
|
||||
// until proc_pidinfo says the result fitted.
|
||||
var buf unsafe.Pointer |
||||
defer func() { |
||||
if buf != nil { |
||||
C.free(buf) |
||||
} |
||||
}() |
||||
for buflen := rlen; ; buflen *= 2 { |
||||
buf, err = C.reallocf(buf, C.size_t(buflen)) |
||||
if buf == nil { |
||||
return 0, err |
||||
} |
||||
rlen, err = C.proc_pidinfo(pid, C.PROC_PIDLISTFDS, 0, buf, buflen) |
||||
if rlen <= 0 { |
||||
return 0, err |
||||
} else if rlen == buflen { |
||||
continue |
||||
} |
||||
return int(rlen / C.PROC_PIDLISTFD_SIZE), nil |
||||
} |
||||
panic("unreachable") |
||||
} |
||||
@ -0,0 +1,53 @@ |
||||
// Copyright 2015 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// +build linux
|
||||
|
||||
package fdtrack |
||||
|
||||
import ( |
||||
"io" |
||||
"os" |
||||
"syscall" |
||||
) |
||||
|
||||
func fdlimit() int { |
||||
var nofile syscall.Rlimit |
||||
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &nofile); err != nil { |
||||
return 0 |
||||
} |
||||
return int(nofile.Cur) |
||||
} |
||||
|
||||
func fdusage() (int, error) { |
||||
f, err := os.Open("/proc/self/fd") |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
defer f.Close() |
||||
const batchSize = 100 |
||||
n := 0 |
||||
for { |
||||
list, err := f.Readdirnames(batchSize) |
||||
n += len(list) |
||||
if err == io.EOF { |
||||
break |
||||
} else if err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
return n, nil |
||||
} |
||||
@ -1,182 +0,0 @@ |
||||
// Copyright 2015 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package comms |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"fmt" |
||||
"io" |
||||
"io/ioutil" |
||||
"net" |
||||
"net/http" |
||||
"time" |
||||
|
||||
"github.com/ethereum/go-ethereum/logger" |
||||
"github.com/ethereum/go-ethereum/logger/glog" |
||||
"github.com/ethereum/go-ethereum/rpc/codec" |
||||
"github.com/ethereum/go-ethereum/rpc/shared" |
||||
) |
||||
|
||||
// When https://github.com/golang/go/issues/4674 is implemented this could be replaced
|
||||
type stoppableTCPListener struct { |
||||
*net.TCPListener |
||||
stop chan struct{} // closed when the listener must stop
|
||||
} |
||||
|
||||
func newStoppableTCPListener(addr string) (*stoppableTCPListener, error) { |
||||
wl, err := net.Listen("tcp", addr) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if tcpl, ok := wl.(*net.TCPListener); ok { |
||||
stop := make(chan struct{}) |
||||
return &stoppableTCPListener{tcpl, stop}, nil |
||||
} |
||||
|
||||
return nil, fmt.Errorf("Unable to create TCP listener for RPC service") |
||||
} |
||||
|
||||
// Stop the listener and all accepted and still active connections.
|
||||
func (self *stoppableTCPListener) Stop() { |
||||
close(self.stop) |
||||
} |
||||
|
||||
func (self *stoppableTCPListener) Accept() (net.Conn, error) { |
||||
for { |
||||
self.SetDeadline(time.Now().Add(time.Duration(1 * time.Second))) |
||||
c, err := self.TCPListener.AcceptTCP() |
||||
|
||||
select { |
||||
case <-self.stop: |
||||
if c != nil { // accept timeout
|
||||
c.Close() |
||||
} |
||||
self.TCPListener.Close() |
||||
return nil, listenerStoppedError |
||||
default: |
||||
} |
||||
|
||||
if err != nil { |
||||
if netErr, ok := err.(net.Error); ok && netErr.Timeout() && netErr.Temporary() { |
||||
continue // regular timeout
|
||||
} |
||||
} |
||||
|
||||
return &closableConnection{c, self.stop}, err |
||||
} |
||||
} |
||||
|
||||
type closableConnection struct { |
||||
*net.TCPConn |
||||
closed chan struct{} |
||||
} |
||||
|
||||
func (self *closableConnection) Read(b []byte) (n int, err error) { |
||||
select { |
||||
case <-self.closed: |
||||
self.TCPConn.Close() |
||||
return 0, io.EOF |
||||
default: |
||||
return self.TCPConn.Read(b) |
||||
} |
||||
} |
||||
|
||||
// Wraps the default handler and checks if the RPC service was stopped. In that case it returns an
|
||||
// error indicating that the service was stopped. This will only happen for connections which are
|
||||
// kept open (HTTP keep-alive) when the RPC service was shutdown.
|
||||
func newStoppableHandler(h http.Handler, stop chan struct{}) http.Handler { |
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
||||
select { |
||||
case <-stop: |
||||
w.Header().Set("Content-Type", "application/json") |
||||
err := fmt.Errorf("RPC service stopped") |
||||
response := shared.NewRpcResponse(-1, shared.JsonRpcVersion, nil, err) |
||||
httpSend(w, response) |
||||
default: |
||||
h.ServeHTTP(w, r) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
func httpSend(writer io.Writer, v interface{}) (n int, err error) { |
||||
var payload []byte |
||||
payload, err = json.MarshalIndent(v, "", "\t") |
||||
if err != nil { |
||||
glog.V(logger.Error).Infoln("Error marshalling JSON", err) |
||||
return 0, err |
||||
} |
||||
glog.V(logger.Detail).Infof("Sending payload: %s", payload) |
||||
|
||||
return writer.Write(payload) |
||||
} |
||||
|
||||
func gethHttpHandler(codec codec.Codec, a shared.EthereumApi) http.Handler { |
||||
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { |
||||
w.Header().Set("Content-Type", "application/json") |
||||
|
||||
// Limit request size to resist DoS
|
||||
if req.ContentLength > maxHttpSizeReqLength { |
||||
err := fmt.Errorf("Request too large") |
||||
response := shared.NewRpcErrorResponse(-1, shared.JsonRpcVersion, -32700, err) |
||||
httpSend(w, &response) |
||||
return |
||||
} |
||||
|
||||
defer req.Body.Close() |
||||
payload, err := ioutil.ReadAll(req.Body) |
||||
if err != nil { |
||||
err := fmt.Errorf("Could not read request body") |
||||
response := shared.NewRpcErrorResponse(-1, shared.JsonRpcVersion, -32700, err) |
||||
httpSend(w, &response) |
||||
return |
||||
} |
||||
|
||||
c := codec.New(nil) |
||||
var rpcReq shared.Request |
||||
if err = c.Decode(payload, &rpcReq); err == nil { |
||||
reply, err := a.Execute(&rpcReq) |
||||
res := shared.NewRpcResponse(rpcReq.Id, rpcReq.Jsonrpc, reply, err) |
||||
httpSend(w, &res) |
||||
return |
||||
} |
||||
|
||||
var reqBatch []shared.Request |
||||
if err = c.Decode(payload, &reqBatch); err == nil { |
||||
resBatch := make([]*interface{}, len(reqBatch)) |
||||
resCount := 0 |
||||
|
||||
for i, rpcReq := range reqBatch { |
||||
reply, err := a.Execute(&rpcReq) |
||||
if rpcReq.Id != nil { // this leaves nil entries in the response batch for later removal
|
||||
resBatch[i] = shared.NewRpcResponse(rpcReq.Id, rpcReq.Jsonrpc, reply, err) |
||||
resCount += 1 |
||||
} |
||||
} |
||||
|
||||
// make response omitting nil entries
|
||||
resBatch = resBatch[:resCount] |
||||
httpSend(w, resBatch) |
||||
return |
||||
} |
||||
|
||||
// invalid request
|
||||
err = fmt.Errorf("Could not decode request") |
||||
res := shared.NewRpcErrorResponse(-1, shared.JsonRpcVersion, -32600, err) |
||||
httpSend(w, res) |
||||
}) |
||||
} |
||||
Loading…
Reference in new issue