From 31015a0bb2f86df0a2bd6b42b509df7266be9992 Mon Sep 17 00:00:00 2001 From: Johan Lindh Date: Thu, 30 Apr 2026 14:01:22 +0200 Subject: [PATCH] feat(runner): add pebble sync switch --- pkg/cmd/run.go | 1 + pkg/runner/run_minimiser_test.go | 17 +++++++++++++++++ pkg/runner/runner.go | 16 +++++++++++++--- 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 pkg/runner/run_minimiser_test.go diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 9a9e221..039ac56 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -34,6 +34,7 @@ func init() { runCmd.Flags().Bool("disable-histogram-sender", false, "do not check for histogram files to upload to core") runCmd.Flags().Bool("disable-mqtt", false, "disable MQTT message sending") runCmd.Flags().Bool("disable-mqtt-filequeue", false, "disable MQTT file based queue") + runCmd.Flags().Bool("pebble-sync", false, "fsync seen-qname pebble writes") runCmd.Flags().String("input-unix", "", "create unix socket for reading dnstap (e.g. /var/lib/unbound/dnstap.sock)") runCmd.Flags().String("input-tcp", "", "create TCP socket for reading dnstap (e.g. '127.0.0.1:53535')") diff --git a/pkg/runner/run_minimiser_test.go b/pkg/runner/run_minimiser_test.go new file mode 100644 index 0000000..a909b61 --- /dev/null +++ b/pkg/runner/run_minimiser_test.go @@ -0,0 +1,17 @@ +package runner + +import ( + "testing" + + "github.com/cockroachdb/pebble" +) + +func TestSeenQnameWriteOptions(t *testing.T) { + if got := seenQnameWriteOptions(config{}); got != pebble.NoSync { + t.Fatalf("default seen-qname write option = %p, want %p", got, pebble.NoSync) + } + + if got := seenQnameWriteOptions(config{PebbleSync: true}); got != pebble.Sync { + t.Fatalf("pebble-sync seen-qname write option = %p, want %p", got, pebble.Sync) + } +} diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 17cb068..5e88266 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -70,6 +70,7 @@ type config struct { DisableHistogramSender bool `mapstructure:"disable-histogram-sender" reload:"true"` DisableMQTT bool `mapstructure:"disable-mqtt"` DisableMQTTFilequeue bool `mapstructure:"disable-mqtt-filequeue"` + PebbleSync bool `mapstructure:"pebble-sync" reload:"true"` InputUnix string `mapstructure:"input-unix" validate:"required_without_all=InputTCP InputTLS,excluded_with=InputTCP InputTLS"` InputTCP string `mapstructure:"input-tcp" validate:"required_without_all=InputUnix InputTLS,excluded_with=InputUnix InputTLS"` InputTLS string `mapstructure:"input-tls" validate:"required_without_all=InputUnix InputTCP,excluded_with=InputUnix InputTCP"` @@ -1048,6 +1049,7 @@ type testConfiger struct { Debug bool DisableHistogramSender bool DisableMQTT bool + PebbleSync bool } func (tc testConfiger) getConfig() (config, error) { @@ -1058,6 +1060,7 @@ func (tc testConfiger) getConfig() (config, error) { Debug: tc.Debug, DisableHistogramSender: tc.DisableHistogramSender, DisableMQTT: tc.DisableMQTT, + PebbleSync: tc.PebbleSync, }, nil } @@ -1820,8 +1823,15 @@ func (wkd *wellKnownDomainsTracker) rotateTracker(edm *dnstapMinimiser, dawgFile return prevWKD, nil } +func seenQnameWriteOptions(conf config) *pebble.WriteOptions { + if conf.PebbleSync { + return pebble.Sync + } + return pebble.NoSync +} + // Check if we have already seen this qname since we started. -func (edm *dnstapMinimiser) qnameSeen(msg *dns.Msg, seenQnameLRU *lru.Cache[string, struct{}], pdb *pebble.DB) bool { +func (edm *dnstapMinimiser) qnameSeen(msg *dns.Msg, seenQnameLRU *lru.Cache[string, struct{}], pdb *pebble.DB, conf config) bool { // NOTE: This looks like it might be a race (calling // Get() followed by separate Add()) but since we want // to keep often looked-up names in the cache we need to @@ -1856,7 +1866,7 @@ func (edm *dnstapMinimiser) qnameSeen(msg *dns.Msg, seenQnameLRU *lru.Cache[stri // If the key does not exist in pebble we insert it if errors.Is(err, pebble.ErrNotFound) { - if err := pdb.Set([]byte(qname), []byte{}, pebble.Sync); err != nil { + if err := pdb.Set([]byte(qname), []byte{}, seenQnameWriteOptions(conf)); err != nil { edm.log.Error("unable to insert key in pebble", "error", err) } return false @@ -2010,7 +2020,7 @@ minimiserLoop: continue } - if !edm.qnameSeen(msg, seenQnameLRU, pdb) { + if !edm.qnameSeen(msg, seenQnameLRU, pdb, conf) { if !startConf.DisableMQTT { newQname := protocols.NewQnameEvent(msg, truncatedTimestamp)