mirror of https://github.com/ethereum/go-ethereum
metrics, cmd/geth: informational metrics (prometheus, influxdb, opentsb) (#24877)
This chang creates a GaugeInfo metrics type for registering informational (textual) metrics, e.g. geth version number. It also improves the testing for backend-exporters, and uses a shared subpackage in 'internal' to provide sample datasets and ordered registry. Implements #21783 --------- Co-authored-by: Martin Holst Swende <martin@swende.se>pull/28031/head
Jorge
1 year ago
committed by
GitHub
parent
5b159498bb
commit
53f3c2ae65
@ -0,0 +1,144 @@ |
||||
package metrics |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"sync" |
||||
) |
||||
|
||||
// GaugeInfos hold a GaugeInfoValue value that can be set arbitrarily.
|
||||
type GaugeInfo interface { |
||||
Snapshot() GaugeInfo |
||||
Update(GaugeInfoValue) |
||||
Value() GaugeInfoValue |
||||
} |
||||
|
||||
// GaugeInfoValue is a mappng of (string) keys to (string) values
|
||||
type GaugeInfoValue map[string]string |
||||
|
||||
func (val GaugeInfoValue) String() string { |
||||
data, _ := json.Marshal(val) |
||||
return string(data) |
||||
} |
||||
|
||||
// GetOrRegisterGaugeInfo returns an existing GaugeInfo or constructs and registers a
|
||||
// new StandardGaugeInfo.
|
||||
func GetOrRegisterGaugeInfo(name string, r Registry) GaugeInfo { |
||||
if nil == r { |
||||
r = DefaultRegistry |
||||
} |
||||
return r.GetOrRegister(name, NewGaugeInfo()).(GaugeInfo) |
||||
} |
||||
|
||||
// NewGaugeInfo constructs a new StandardGaugeInfo.
|
||||
func NewGaugeInfo() GaugeInfo { |
||||
if !Enabled { |
||||
return NilGaugeInfo{} |
||||
} |
||||
return &StandardGaugeInfo{ |
||||
value: GaugeInfoValue{}, |
||||
} |
||||
} |
||||
|
||||
// NewRegisteredGaugeInfo constructs and registers a new StandardGaugeInfo.
|
||||
func NewRegisteredGaugeInfo(name string, r Registry) GaugeInfo { |
||||
c := NewGaugeInfo() |
||||
if nil == r { |
||||
r = DefaultRegistry |
||||
} |
||||
r.Register(name, c) |
||||
return c |
||||
} |
||||
|
||||
// NewFunctionalGauge constructs a new FunctionalGauge.
|
||||
func NewFunctionalGaugeInfo(f func() GaugeInfoValue) GaugeInfo { |
||||
if !Enabled { |
||||
return NilGaugeInfo{} |
||||
} |
||||
return &FunctionalGaugeInfo{value: f} |
||||
} |
||||
|
||||
// NewRegisteredFunctionalGauge constructs and registers a new StandardGauge.
|
||||
func NewRegisteredFunctionalGaugeInfo(name string, r Registry, f func() GaugeInfoValue) GaugeInfo { |
||||
c := NewFunctionalGaugeInfo(f) |
||||
if nil == r { |
||||
r = DefaultRegistry |
||||
} |
||||
r.Register(name, c) |
||||
return c |
||||
} |
||||
|
||||
// GaugeInfoSnapshot is a read-only copy of another GaugeInfo.
|
||||
type GaugeInfoSnapshot GaugeInfoValue |
||||
|
||||
// Snapshot returns the snapshot.
|
||||
func (g GaugeInfoSnapshot) Snapshot() GaugeInfo { return g } |
||||
|
||||
// Update panics.
|
||||
func (GaugeInfoSnapshot) Update(GaugeInfoValue) { |
||||
panic("Update called on a GaugeInfoSnapshot") |
||||
} |
||||
|
||||
// Value returns the value at the time the snapshot was taken.
|
||||
func (g GaugeInfoSnapshot) Value() GaugeInfoValue { return GaugeInfoValue(g) } |
||||
|
||||
// NilGauge is a no-op Gauge.
|
||||
type NilGaugeInfo struct{} |
||||
|
||||
// Snapshot is a no-op.
|
||||
func (NilGaugeInfo) Snapshot() GaugeInfo { return NilGaugeInfo{} } |
||||
|
||||
// Update is a no-op.
|
||||
func (NilGaugeInfo) Update(v GaugeInfoValue) {} |
||||
|
||||
// Value is a no-op.
|
||||
func (NilGaugeInfo) Value() GaugeInfoValue { return GaugeInfoValue{} } |
||||
|
||||
// StandardGaugeInfo is the standard implementation of a GaugeInfo and uses
|
||||
// sync.Mutex to manage a single string value.
|
||||
type StandardGaugeInfo struct { |
||||
mutex sync.Mutex |
||||
value GaugeInfoValue |
||||
} |
||||
|
||||
// Snapshot returns a read-only copy of the gauge.
|
||||
func (g *StandardGaugeInfo) Snapshot() GaugeInfo { |
||||
return GaugeInfoSnapshot(g.Value()) |
||||
} |
||||
|
||||
// Update updates the gauge's value.
|
||||
func (g *StandardGaugeInfo) Update(v GaugeInfoValue) { |
||||
g.mutex.Lock() |
||||
defer g.mutex.Unlock() |
||||
g.value = v |
||||
} |
||||
|
||||
// Value returns the gauge's current value.
|
||||
func (g *StandardGaugeInfo) Value() GaugeInfoValue { |
||||
g.mutex.Lock() |
||||
defer g.mutex.Unlock() |
||||
return g.value |
||||
} |
||||
|
||||
// FunctionalGaugeInfo returns value from given function
|
||||
type FunctionalGaugeInfo struct { |
||||
value func() GaugeInfoValue |
||||
} |
||||
|
||||
// Value returns the gauge's current value.
|
||||
func (g FunctionalGaugeInfo) Value() GaugeInfoValue { |
||||
return g.value() |
||||
} |
||||
|
||||
// Value returns the gauge's current value in JSON string format
|
||||
func (g FunctionalGaugeInfo) ValueJsonString() string { |
||||
data, _ := json.Marshal(g.value()) |
||||
return string(data) |
||||
} |
||||
|
||||
// Snapshot returns the snapshot.
|
||||
func (g FunctionalGaugeInfo) Snapshot() GaugeInfo { return GaugeInfoSnapshot(g.Value()) } |
||||
|
||||
// Update panics.
|
||||
func (FunctionalGaugeInfo) Update(GaugeInfoValue) { |
||||
panic("Update called on a FunctionalGaugeInfo") |
||||
} |
||||
@ -0,0 +1,75 @@ |
||||
package metrics |
||||
|
||||
import ( |
||||
"strconv" |
||||
"testing" |
||||
) |
||||
|
||||
func TestGaugeInfoJsonString(t *testing.T) { |
||||
g := NewGaugeInfo() |
||||
g.Update(GaugeInfoValue{ |
||||
"chain_id": "5", |
||||
"anotherKey": "any_string_value", |
||||
"third_key": "anything", |
||||
}, |
||||
) |
||||
want := `{"anotherKey":"any_string_value","chain_id":"5","third_key":"anything"}` |
||||
if have := g.Value().String(); have != want { |
||||
t.Errorf("\nhave: %v\nwant: %v\n", have, want) |
||||
} |
||||
} |
||||
|
||||
func TestGaugeInfoSnapshot(t *testing.T) { |
||||
g := NewGaugeInfo() |
||||
g.Update(GaugeInfoValue{"value": "original"}) |
||||
snapshot := g.Snapshot() // Snapshot @chainid 5
|
||||
g.Update(GaugeInfoValue{"value": "updated"}) |
||||
// The 'g' should be updated
|
||||
if have, want := g.Value().String(), `{"value":"updated"}`; have != want { |
||||
t.Errorf("\nhave: %v\nwant: %v\n", have, want) |
||||
} |
||||
// Snapshot should be unupdated
|
||||
if have, want := snapshot.Value().String(), `{"value":"original"}`; have != want { |
||||
t.Errorf("\nhave: %v\nwant: %v\n", have, want) |
||||
} |
||||
} |
||||
|
||||
func TestGetOrRegisterGaugeInfo(t *testing.T) { |
||||
r := NewRegistry() |
||||
NewRegisteredGaugeInfo("foo", r).Update( |
||||
GaugeInfoValue{"chain_id": "5"}) |
||||
g := GetOrRegisterGaugeInfo("foo", r) |
||||
if have, want := g.Value().String(), `{"chain_id":"5"}`; have != want { |
||||
t.Errorf("have\n%v\nwant\n%v\n", have, want) |
||||
} |
||||
} |
||||
|
||||
func TestFunctionalGaugeInfo(t *testing.T) { |
||||
info := GaugeInfoValue{"chain_id": "0"} |
||||
counter := 1 |
||||
// A "functional" gauge invokes the method to obtain the value
|
||||
fg := NewFunctionalGaugeInfo(func() GaugeInfoValue { |
||||
info["chain_id"] = strconv.Itoa(counter) |
||||
counter++ |
||||
return info |
||||
}) |
||||
fg.Value() |
||||
fg.Value() |
||||
if have, want := info["chain_id"], "2"; have != want { |
||||
t.Errorf("have %v want %v", have, want) |
||||
} |
||||
} |
||||
|
||||
func TestGetOrRegisterFunctionalGaugeInfo(t *testing.T) { |
||||
r := NewRegistry() |
||||
NewRegisteredFunctionalGaugeInfo("foo", r, func() GaugeInfoValue { |
||||
return GaugeInfoValue{ |
||||
"chain_id": "5", |
||||
} |
||||
}) |
||||
want := `{"chain_id":"5"}` |
||||
have := GetOrRegisterGaugeInfo("foo", r).Value().String() |
||||
if have != want { |
||||
t.Errorf("have\n%v\nwant\n%v\n", have, want) |
||||
} |
||||
} |
||||
@ -0,0 +1,114 @@ |
||||
// Copyright 2023 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package influxdb |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io" |
||||
"net/http" |
||||
"net/http/httptest" |
||||
"net/url" |
||||
"os" |
||||
"strings" |
||||
"testing" |
||||
|
||||
"github.com/ethereum/go-ethereum/metrics" |
||||
"github.com/ethereum/go-ethereum/metrics/internal" |
||||
influxdb2 "github.com/influxdata/influxdb-client-go/v2" |
||||
) |
||||
|
||||
func TestMain(m *testing.M) { |
||||
metrics.Enabled = true |
||||
os.Exit(m.Run()) |
||||
} |
||||
|
||||
func TestExampleV1(t *testing.T) { |
||||
r := internal.ExampleMetrics() |
||||
var have, want string |
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
||||
haveB, _ := io.ReadAll(r.Body) |
||||
have = string(haveB) |
||||
r.Body.Close() |
||||
})) |
||||
defer ts.Close() |
||||
u, _ := url.Parse(ts.URL) |
||||
rep := &reporter{ |
||||
reg: r, |
||||
url: *u, |
||||
namespace: "goth.", |
||||
} |
||||
if err := rep.makeClient(); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if err := rep.send(978307200); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if wantB, err := os.ReadFile("./testdata/influxdbv1.want"); err != nil { |
||||
t.Fatal(err) |
||||
} else { |
||||
want = string(wantB) |
||||
} |
||||
if have != want { |
||||
t.Errorf("\nhave:\n%v\nwant:\n%v\n", have, want) |
||||
t.Logf("have vs want:\n%v", findFirstDiffPos(have, want)) |
||||
} |
||||
} |
||||
|
||||
func TestExampleV2(t *testing.T) { |
||||
r := internal.ExampleMetrics() |
||||
var have, want string |
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
||||
haveB, _ := io.ReadAll(r.Body) |
||||
have = string(haveB) |
||||
r.Body.Close() |
||||
})) |
||||
defer ts.Close() |
||||
|
||||
rep := &v2Reporter{ |
||||
reg: r, |
||||
endpoint: ts.URL, |
||||
namespace: "goth.", |
||||
} |
||||
rep.client = influxdb2.NewClient(rep.endpoint, rep.token) |
||||
defer rep.client.Close() |
||||
rep.write = rep.client.WriteAPI(rep.organization, rep.bucket) |
||||
|
||||
rep.send(978307200) |
||||
|
||||
if wantB, err := os.ReadFile("./testdata/influxdbv2.want"); err != nil { |
||||
t.Fatal(err) |
||||
} else { |
||||
want = string(wantB) |
||||
} |
||||
if have != want { |
||||
t.Errorf("\nhave:\n%v\nwant:\n%v\n", have, want) |
||||
t.Logf("have vs want:\n %v", findFirstDiffPos(have, want)) |
||||
} |
||||
} |
||||
|
||||
func findFirstDiffPos(a, b string) string { |
||||
yy := strings.Split(b, "\n") |
||||
for i, x := range strings.Split(a, "\n") { |
||||
if i >= len(yy) { |
||||
return fmt.Sprintf("have:%d: %s\nwant:%d: <EOF>", i, x, i) |
||||
} |
||||
if y := yy[i]; x != y { |
||||
return fmt.Sprintf("have:%d: %s\nwant:%d: %s", i, x, i, y) |
||||
} |
||||
} |
||||
return "" |
||||
} |
||||
@ -0,0 +1,9 @@ |
||||
goth.test/counter.count value=12345 978307200000000000 |
||||
goth.test/counter_float64.count value=54321.98 978307200000000000 |
||||
goth.test/gauge.gauge value=23456i 978307200000000000 |
||||
goth.test/gauge_float64.gauge value=34567.89 978307200000000000 |
||||
goth.test/gauge_info.gauge value="{\"arch\":\"amd64\",\"commit\":\"7caa2d8163ae3132c1c2d6978c76610caee2d949\",\"os\":\"linux\",\"protocol_versions\":\"64 65 66\",\"version\":\"1.10.18-unstable\"}" 978307200000000000 |
||||
goth.test/histogram.histogram count=3i,max=3i,mean=2,min=1i,p25=1,p50=2,p75=3,p95=3,p99=3,p999=3,p9999=3,stddev=0.816496580927726,variance=0.6666666666666666 978307200000000000 |
||||
goth.test/meter.meter count=0i,m1=0,m15=0,m5=0,mean=0 978307200000000000 |
||||
goth.test/resetting_timer.span count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12000000i,p95=120000000i,p99=120000000i 978307200000000000 |
||||
goth.test/timer.timer count=6i,m1=0,m15=0,m5=0,max=120000000i,mean=38333333.333333336,meanrate=0,min=20000000i,p50=22500000,p75=48000000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000,stddev=36545253.529775314,variance=1335555555555555.2 978307200000000000 |
||||
@ -0,0 +1,9 @@ |
||||
goth.test/counter.count value=12345 978307200000000000 |
||||
goth.test/counter_float64.count value=54321.98 978307200000000000 |
||||
goth.test/gauge.gauge value=23456i 978307200000000000 |
||||
goth.test/gauge_float64.gauge value=34567.89 978307200000000000 |
||||
goth.test/gauge_info.gauge value="{\"arch\":\"amd64\",\"commit\":\"7caa2d8163ae3132c1c2d6978c76610caee2d949\",\"os\":\"linux\",\"protocol_versions\":\"64 65 66\",\"version\":\"1.10.18-unstable\"}" 978307200000000000 |
||||
goth.test/histogram.histogram count=3i,max=3i,mean=2,min=1i,p25=1,p50=2,p75=3,p95=3,p99=3,p999=3,p9999=3,stddev=0.816496580927726,variance=0.6666666666666666 978307200000000000 |
||||
goth.test/meter.meter count=0i,m1=0,m15=0,m5=0,mean=0 978307200000000000 |
||||
goth.test/resetting_timer.span count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12000000i,p95=120000000i,p99=120000000i 978307200000000000 |
||||
goth.test/timer.timer count=6i,m1=0,m15=0,m5=0,max=120000000i,mean=38333333.333333336,meanrate=0,min=20000000i,p50=22500000,p75=48000000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000,stddev=36545253.529775314,variance=1335555555555555.2 978307200000000000 |
||||
@ -0,0 +1,64 @@ |
||||
// Copyright 2023 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package internal |
||||
|
||||
import ( |
||||
"time" |
||||
|
||||
"github.com/ethereum/go-ethereum/metrics" |
||||
) |
||||
|
||||
// ExampleMetrics returns an ordered registry populated with a sample of metrics.
|
||||
func ExampleMetrics() metrics.Registry { |
||||
var registry = metrics.NewOrderedRegistry() |
||||
|
||||
metrics.NewRegisteredCounterFloat64("test/counter", registry).Inc(12345) |
||||
metrics.NewRegisteredCounterFloat64("test/counter_float64", registry).Inc(54321.98) |
||||
metrics.NewRegisteredGauge("test/gauge", registry).Update(23456) |
||||
metrics.NewRegisteredGaugeFloat64("test/gauge_float64", registry).Update(34567.89) |
||||
metrics.NewRegisteredGaugeInfo("test/gauge_info", registry).Update( |
||||
metrics.GaugeInfoValue{ |
||||
"version": "1.10.18-unstable", |
||||
"arch": "amd64", |
||||
"os": "linux", |
||||
"commit": "7caa2d8163ae3132c1c2d6978c76610caee2d949", |
||||
"protocol_versions": "64 65 66", |
||||
}) |
||||
metrics.NewRegisteredHistogram("test/histogram", registry, metrics.NewSampleSnapshot(3, []int64{1, 2, 3})) |
||||
registry.Register("test/meter", metrics.NewInactiveMeter()) |
||||
{ |
||||
timer := metrics.NewRegisteredResettingTimer("test/resetting_timer", registry) |
||||
timer.Update(10 * time.Millisecond) |
||||
timer.Update(11 * time.Millisecond) |
||||
timer.Update(12 * time.Millisecond) |
||||
timer.Update(120 * time.Millisecond) |
||||
timer.Update(13 * time.Millisecond) |
||||
timer.Update(14 * time.Millisecond) |
||||
} |
||||
{ |
||||
timer := metrics.NewRegisteredTimer("test/timer", registry) |
||||
timer.Update(20 * time.Millisecond) |
||||
timer.Update(21 * time.Millisecond) |
||||
timer.Update(22 * time.Millisecond) |
||||
timer.Update(120 * time.Millisecond) |
||||
timer.Update(23 * time.Millisecond) |
||||
timer.Update(24 * time.Millisecond) |
||||
timer.Stop() |
||||
} |
||||
registry.Register("test/empty_resetting_timer", metrics.NewResettingTimer().Snapshot()) |
||||
return registry |
||||
} |
||||
@ -0,0 +1,48 @@ |
||||
# TYPE test_counter gauge |
||||
test_counter 12345 |
||||
|
||||
# TYPE test_counter_float64 gauge |
||||
test_counter_float64 54321.98 |
||||
|
||||
# TYPE test_gauge gauge |
||||
test_gauge 23456 |
||||
|
||||
# TYPE test_gauge_float64 gauge |
||||
test_gauge_float64 34567.89 |
||||
|
||||
# TYPE test_gauge_info gauge |
||||
test_gauge_info {arch="amd64", commit="7caa2d8163ae3132c1c2d6978c76610caee2d949", os="linux", protocol_versions="64 65 66", version="1.10.18-unstable"} 1 |
||||
|
||||
# TYPE test_histogram_count counter |
||||
test_histogram_count 3 |
||||
|
||||
# TYPE test_histogram summary |
||||
test_histogram {quantile="0.5"} 2 |
||||
test_histogram {quantile="0.75"} 3 |
||||
test_histogram {quantile="0.95"} 3 |
||||
test_histogram {quantile="0.99"} 3 |
||||
test_histogram {quantile="0.999"} 3 |
||||
test_histogram {quantile="0.9999"} 3 |
||||
|
||||
# TYPE test_meter gauge |
||||
test_meter 0 |
||||
|
||||
# TYPE test_resetting_timer_count counter |
||||
test_resetting_timer_count 6 |
||||
|
||||
# TYPE test_resetting_timer summary |
||||
test_resetting_timer {quantile="0.50"} 12000000 |
||||
test_resetting_timer {quantile="0.95"} 120000000 |
||||
test_resetting_timer {quantile="0.99"} 120000000 |
||||
|
||||
# TYPE test_timer_count counter |
||||
test_timer_count 6 |
||||
|
||||
# TYPE test_timer summary |
||||
test_timer {quantile="0.5"} 2.25e+07 |
||||
test_timer {quantile="0.75"} 4.8e+07 |
||||
test_timer {quantile="0.95"} 1.2e+08 |
||||
test_timer {quantile="0.99"} 1.2e+08 |
||||
test_timer {quantile="0.999"} 1.2e+08 |
||||
test_timer {quantile="0.9999"} 1.2e+08 |
||||
|
||||
@ -0,0 +1,23 @@ |
||||
put pre.elite.count 978307200 0 host=hal9000 |
||||
put pre.elite.one-minute 978307200 0.00 host=hal9000 |
||||
put pre.elite.five-minute 978307200 0.00 host=hal9000 |
||||
put pre.elite.fifteen-minute 978307200 0.00 host=hal9000 |
||||
put pre.elite.mean 978307200 0.00 host=hal9000 |
||||
put pre.foo.value 978307200 {"chain_id":"5"} host=hal9000 |
||||
put pre.months.count 978307200 12 host=hal9000 |
||||
put pre.pi.value 978307200 3.140000 host=hal9000 |
||||
put pre.second.count 978307200 1 host=hal9000 |
||||
put pre.second.min 978307200 1000 host=hal9000 |
||||
put pre.second.max 978307200 1000 host=hal9000 |
||||
put pre.second.mean 978307200 1000.00 host=hal9000 |
||||
put pre.second.std-dev 978307200 0.00 host=hal9000 |
||||
put pre.second.50-percentile 978307200 1000.00 host=hal9000 |
||||
put pre.second.75-percentile 978307200 1000.00 host=hal9000 |
||||
put pre.second.95-percentile 978307200 1000.00 host=hal9000 |
||||
put pre.second.99-percentile 978307200 1000.00 host=hal9000 |
||||
put pre.second.999-percentile 978307200 1000.00 host=hal9000 |
||||
put pre.second.one-minute 978307200 0.00 host=hal9000 |
||||
put pre.second.five-minute 978307200 0.00 host=hal9000 |
||||
put pre.second.fifteen-minute 978307200 0.00 host=hal9000 |
||||
put pre.second.mean-rate 978307200 0.00 host=hal9000 |
||||
put pre.tau.count 978307200 1.570000 host=hal9000 |
||||
Loading…
Reference in new issue