Graceful rolling update

I want to do graceful rolling update, and I notice leader election will be trigged when starting an stopped node, which will cause about 1 second interrupt.
Is this by design and is there anyway I can avoid this?
I expect the graceful rolling update:

leader node give away leader before shutdown (can be done by RequestLeaderTransfer)
former leader node restarted and join cluster without trigger leader election
loop this for all node

Please enable that CheckQuorum option and see whether that solves your problem.
https://github.com/lni/dragonboat/blob/023bafb8e648934a8f16b98ff58d91b743358687/config/config.go#L74
I don't quite understand why such restarted node would trigger an election, once started, it waits for the election timeout first, it is expected to receive heartbeat from the leader node and become a follower.
If the above CheckQuorum option doesn't help, please provide more details including your configurations, more log on what happened on the restarted node.

minimal reproduce code
package main

import (
	"context"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"time"
	"errors"

	"github.com/lni/dragonboat/v3"
	"github.com/lni/dragonboat/v3/config"
	sm "github.com/lni/dragonboat/v3/statemachine"
	"github.com/lni/goutils/syncutil"
)

type exampleStateMachine struct {
	ClusterID uint64
	NodeID    uint64
	Tick      uint64
}

func NewExampleStateMachine(clusterID uint64, nodeID uint64) sm.IStateMachine {
	return &exampleStateMachine{ClusterID: clusterID, NodeID: nodeID}
}
func (it *exampleStateMachine) Update(req []byte) (sm.Result, error) {
	it.Tick++
	return sm.Result{}, nil
}
func (it *exampleStateMachine) Lookup(req interface{}) (interface{}, error) {
	return it.Tick, nil
}
func (it *exampleStateMachine) SaveSnapshot(w io.Writer, _ sm.ISnapshotFileCollection, _ <-chan struct{}) error {
	data := make([]byte, 8)
	binary.LittleEndian.PutUint64(data, it.Tick)
	w.Write(data)
	return nil
}
func (it *exampleStateMachine) RecoverFromSnapshot(r io.Reader, _ []sm.SnapshotFile, _ <-chan struct{}) error {
	data, _ := io.ReadAll(r)
	v := binary.LittleEndian.Uint64(data)
	it.Tick = v
	return nil
}
func (it *exampleStateMachine) Close() error {
	return nil
}

func newNodeHost(nhid uint64, addr string) *dragonboat.NodeHost {
	conf := config.NodeHostConfig{
		NodeHostDir:         fmt.Sprintf("single_nodehost_test_dir_safe_to_delete/%d", nhid),
		AddressByNodeHostID: false,
		RTTMillisecond:      100,
		RaftAddress:         addr,
		Expert: config.ExpertConfig{
			TestNodeHostID: nhid,
		},
	}
	nh, err := dragonboat.NewNodeHost(conf)
	if err != nil {
		panic(err)
	}
	return nh
}

func startShard(nh *dragonboat.NodeHost, clusterID uint64, nodeID uint64, initials map[uint64]string, fn interface{}) {
	conf := config.Config{
		NodeID:              nodeID,
		ClusterID:           clusterID,
		CheckQuorum:         true,
		ElectionRTT:         10,
		HeartbeatRTT:        1,
		SnapshotEntries:     0,
		CompactionOverhead:  0,
		OrderedConfigChange: false,
	}
	var err error
	switch f := fn.(type) {
	case sm.CreateStateMachineFunc:
		if len(initials) > 0 {
			err = nh.StartCluster(initials, false, f, conf)
		} else {
			err = nh.StartCluster(nil, false, f, conf)
		}
	case sm.CreateConcurrentStateMachineFunc:
		if len(initials) > 0 {
			err = nh.StartConcurrentCluster(initials, false, f, conf)
		} else {
			err = nh.StartConcurrentCluster(nil, false, f, conf)
		}
	default:
		panic(fmt.Errorf("unknown fn type: %T", f))
	}
	if err != nil {
		panic(err)
	}
}

func tick(nh *dragonboat.NodeHost, clusterID uint64) *syncutil.Stopper {
	stopper := syncutil.NewStopper()
	stopper.RunWorker(func() {
		t := time.NewTicker(time.Millisecond * 30)
		defer t.Stop()
		nop := nh.GetNoOPSession(clusterID)
		ctx := context.Background()
		for {
			select {
			case <-t.C:
				ctx1, cancel := context.WithTimeout(ctx, time.Second)
				_, err := nh.SyncPropose(ctx1, nop, []byte("tick"))
				cancel()
				if err != nil {
					// log.Fatalln("[FATAL]", fmt.Errorf("node %d shahrd %d tick err: %w", nh.NodeHostConfig().Expert.TestNodeHostID, clusterID, err))
					log.Println("[WARN]", fmt.Errorf("node %d shard %d tick err: %w", nh.NodeHostConfig().Expert.TestNodeHostID, clusterID, err))
				}
			case <-stopper.ShouldStop():
				return
			}
		}
	})
	return stopper
}

func waitReady(nh *dragonboat.NodeHost, clusterID uint64) {
	for {
		ctx, cancel := context.WithTimeout(context.TODO(), time.Second)
		defer cancel()
		if _, err := nh.SyncRead(ctx, clusterID, nil); err != nil {
			if errors.Is(err, dragonboat.ErrClusterNotReady) {
				time.Sleep(time.Second)
				continue
			}
		}
		return
	}
}

func main() {
	clusterID := uint64(0)
	fn := sm.CreateStateMachineFunc(NewExampleStateMachine)
	initials := map[uint64]string{
		1: "<IP_ADDRESS>:3001",
		2: "<IP_ADDRESS>:3002",
		3: "<IP_ADDRESS>:3003",
	}
	nh1 := newNodeHost(1, initials[1])
	nh2 := newNodeHost(2, initials[2])
	nh3 := newNodeHost(3, initials[3])
	startShard(nh1, clusterID, 1, initials, fn)
	startShard(nh2, clusterID, 2, initials, fn)
	startShard(nh3, clusterID, 3, initials, fn)
	if err := nh1.RequestLeaderTransfer(clusterID, 1); err != nil {
		log.Fatalln("[FATAL]", fmt.Errorf("RequestLeaderTransfer err: %w", err))
	}
	waitReady(nh1, clusterID)
	stopper := tick(nh1, clusterID)
	defer stopper.Stop()
	println("phase1")

	nh3.Stop()
	time.Sleep(time.Second)
	println("phase2")

	nh3 = newNodeHost(3, initials[3])
	startShard(nh3, clusterID, 3, nil, fn)
	waitReady(nh3, clusterID)
	println("phase3")

	nh2.Stop()
	time.Sleep(time.Second)
	println("phase4")

	nh2 = newNodeHost(2, initials[2])
	startShard(nh2, clusterID, 2, nil, fn)
	waitReady(nh2, clusterID)
	println("phase5")

	time.Sleep(time.Second*5)
}

log
|| [go run .]
|| 2023-02-14 16:51:15.212588 I | dragonboat: go version: go1.19, windows/amd64
|| 2023-02-14 16:51:15.213146 I | dragonboat: dragonboat version: 3.3.5 (Rel)
|| 2023-02-14 16:51:15.213146 W | dragonboat: unsupported OS/ARCH windows/amd64, don't use for production
|| 2023-02-14 16:51:15.213146 I | config: using default EngineConfig
|| 2023-02-14 16:51:15.213146 I | config: using default LogDBConfig
|| 2023-02-14 16:51:15.213146 I | dragonboat: DeploymentID set to 1
|| 2023-02-14 16:51:15.218534 I | dragonboat: LogDB info received, shard 0, busy false
|| 2023-02-14 16:51:15.222140 I | dragonboat: LogDB info received, shard 1, busy false
|| 2023-02-14 16:51:15.224807 I | dragonboat: LogDB info received, shard 2, busy false
|| 2023-02-14 16:51:15.227983 I | dragonboat: LogDB info received, shard 3, busy false
|| 2023-02-14 16:51:15.230668 I | dragonboat: LogDB info received, shard 4, busy false
|| 2023-02-14 16:51:15.234163 I | dragonboat: LogDB info received, shard 5, busy false
|| 2023-02-14 16:51:15.237202 I | dragonboat: LogDB info received, shard 6, busy false
|| 2023-02-14 16:51:15.240217 I | dragonboat: LogDB info received, shard 7, busy false
|| 2023-02-14 16:51:15.243220 I | dragonboat: LogDB info received, shard 8, busy false
|| 2023-02-14 16:51:15.246549 I | dragonboat: LogDB info received, shard 9, busy false
|| 2023-02-14 16:51:15.248555 I | dragonboat: LogDB info received, shard 10, busy false
|| 2023-02-14 16:51:15.252555 I | dragonboat: LogDB info received, shard 11, busy false
|| 2023-02-14 16:51:15.256415 I | dragonboat: LogDB info received, shard 12, busy false
|| 2023-02-14 16:51:15.259409 I | dragonboat: LogDB info received, shard 13, busy false
|| 2023-02-14 16:51:15.261409 I | dragonboat: LogDB info received, shard 14, busy false
|| 2023-02-14 16:51:15.264409 I | dragonboat: LogDB info received, shard 15, busy false
|| 2023-02-14 16:51:15.265077 I | logdb: using plain logdb
|| 2023-02-14 16:51:15.265077 I | dragonboat: logdb memory limit: 8192 MBytes
|| 2023-02-14 16:51:15.265077 I | dragonboat: NodeHost ID: nhid-1
|| 2023-02-14 16:51:15.265077 I | dragonboat: using regular node registry
|| 2023-02-14 16:51:15.265077 I | dragonboat: filesystem error injection mode enabled: false
|| 2023-02-14 16:51:15.266122 I | transport: transport type: go-tcp-transport
|| 2023-02-14 16:51:15.269541 I | dragonboat: transport type: go-tcp-transport
|| 2023-02-14 16:51:15.269541 I | dragonboat: logdb type: sharded-pebble
|| 2023-02-14 16:51:15.269541 I | dragonboat: nodehost address: <IP_ADDRESS>:3001
|| 2023-02-14 16:51:15.269541 I | dragonboat: go version: go1.19, windows/amd64
|| 2023-02-14 16:51:15.269541 I | dragonboat: dragonboat version: 3.3.5 (Rel)
|| 2023-02-14 16:51:15.269541 W | dragonboat: unsupported OS/ARCH windows/amd64, don't use for production
|| 2023-02-14 16:51:15.269541 I | config: using default EngineConfig
|| 2023-02-14 16:51:15.269541 I | config: using default LogDBConfig
|| 2023-02-14 16:51:15.269541 I | dragonboat: DeploymentID set to 1
|| 2023-02-14 16:51:15.275077 I | dragonboat: LogDB info received, shard 0, busy false
|| 2023-02-14 16:51:15.278075 I | dragonboat: LogDB info received, shard 1, busy false
|| 2023-02-14 16:51:15.281073 I | dragonboat: LogDB info received, shard 2, busy false
|| 2023-02-14 16:51:15.284563 I | dragonboat: LogDB info received, shard 3, busy false
|| 2023-02-14 16:51:15.288325 I | dragonboat: LogDB info received, shard 4, busy false
|| 2023-02-14 16:51:15.291489 I | dragonboat: LogDB info received, shard 5, busy false
|| 2023-02-14 16:51:15.294622 I | dragonboat: LogDB info received, shard 6, busy false
|| 2023-02-14 16:51:15.297301 I | dragonboat: LogDB info received, shard 7, busy false
|| 2023-02-14 16:51:15.301001 I | dragonboat: LogDB info received, shard 8, busy false
|| 2023-02-14 16:51:15.304093 I | dragonboat: LogDB info received, shard 9, busy false
|| 2023-02-14 16:51:15.306996 I | dragonboat: LogDB info received, shard 10, busy false
|| 2023-02-14 16:51:15.310227 I | dragonboat: LogDB info received, shard 11, busy false
|| 2023-02-14 16:51:15.312938 I | dragonboat: LogDB info received, shard 12, busy false
|| 2023-02-14 16:51:15.315887 I | dragonboat: LogDB info received, shard 13, busy false
|| 2023-02-14 16:51:15.319392 I | dragonboat: LogDB info received, shard 14, busy false
|| 2023-02-14 16:51:15.322454 I | logdb: using plain logdb
|| 2023-02-14 16:51:15.322454 I | dragonboat: LogDB info received, shard 15, busy false
|| 2023-02-14 16:51:15.322454 I | dragonboat: logdb memory limit: 8192 MBytes
|| 2023-02-14 16:51:15.322454 I | dragonboat: NodeHost ID: nhid-2
|| 2023-02-14 16:51:15.322454 I | dragonboat: using regular node registry
|| 2023-02-14 16:51:15.322454 I | dragonboat: filesystem error injection mode enabled: false
|| 2023-02-14 16:51:15.322454 I | transport: transport type: go-tcp-transport
|| 2023-02-14 16:51:15.322454 I | dragonboat: transport type: go-tcp-transport
|| 2023-02-14 16:51:15.322454 I | dragonboat: logdb type: sharded-pebble
|| 2023-02-14 16:51:15.322454 I | dragonboat: nodehost address: <IP_ADDRESS>:3002
|| 2023-02-14 16:51:15.322454 I | dragonboat: go version: go1.19, windows/amd64
|| 2023-02-14 16:51:15.322454 I | dragonboat: dragonboat version: 3.3.5 (Rel)
|| 2023-02-14 16:51:15.322454 W | dragonboat: unsupported OS/ARCH windows/amd64, don't use for production
|| 2023-02-14 16:51:15.322454 I | config: using default EngineConfig
|| 2023-02-14 16:51:15.322454 I | config: using default LogDBConfig
|| 2023-02-14 16:51:15.322454 I | dragonboat: DeploymentID set to 1
|| 2023-02-14 16:51:15.327183 I | dragonboat: LogDB info received, shard 0, busy false
|| 2023-02-14 16:51:15.329256 I | dragonboat: LogDB info received, shard 1, busy false
|| 2023-02-14 16:51:15.332258 I | dragonboat: LogDB info received, shard 2, busy false
|| 2023-02-14 16:51:15.335255 I | dragonboat: LogDB info received, shard 3, busy false
|| 2023-02-14 16:51:15.338713 I | dragonboat: LogDB info received, shard 4, busy false
|| 2023-02-14 16:51:15.342038 I | dragonboat: LogDB info received, shard 5, busy false
|| 2023-02-14 16:51:15.345234 I | dragonboat: LogDB info received, shard 6, busy false
|| 2023-02-14 16:51:15.347824 I | dragonboat: LogDB info received, shard 7, busy false
|| 2023-02-14 16:51:15.351558 I | dragonboat: LogDB info received, shard 8, busy false
|| 2023-02-14 16:51:15.354732 I | dragonboat: LogDB info received, shard 9, busy false
|| 2023-02-14 16:51:15.357619 I | dragonboat: LogDB info received, shard 10, busy false
|| 2023-02-14 16:51:15.360751 I | dragonboat: LogDB info received, shard 11, busy false
|| 2023-02-14 16:51:15.363739 I | dragonboat: LogDB info received, shard 12, busy false
|| 2023-02-14 16:51:15.366866 I | dragonboat: LogDB info received, shard 13, busy false
|| 2023-02-14 16:51:15.370934 I | dragonboat: LogDB info received, shard 14, busy false
|| 2023-02-14 16:51:15.373427 I | dragonboat: LogDB info received, shard 15, busy false
|| 2023-02-14 16:51:15.373427 I | logdb: using plain logdb
|| 2023-02-14 16:51:15.373427 I | dragonboat: logdb memory limit: 8192 MBytes
|| 2023-02-14 16:51:15.373427 I | dragonboat: NodeHost ID: nhid-3
|| 2023-02-14 16:51:15.373427 I | dragonboat: using regular node registry
|| 2023-02-14 16:51:15.373427 I | dragonboat: filesystem error injection mode enabled: false
|| 2023-02-14 16:51:15.374973 I | transport: transport type: go-tcp-transport
|| 2023-02-14 16:51:15.374973 I | dragonboat: transport type: go-tcp-transport
|| 2023-02-14 16:51:15.374973 I | dragonboat: logdb type: sharded-pebble
|| 2023-02-14 16:51:15.374973 I | dragonboat: nodehost address: <IP_ADDRESS>:3003
|| 2023-02-14 16:51:15.374973 I | dragonboat: [00000:00001] replaying raft logs
|| 2023-02-14 16:51:15.376003 I | dragonboat: [00000:00001] has logdb entries size 1330 commit 1330 term 13
|| 2023-02-14 16:51:15.376003 I | raft: [00000:00001] created, initial: true, new: false
|| 2023-02-14 16:51:15.376003 I | raft: [00000:00001] raft log rate limit enabled: false, 0
|| 2023-02-14 16:51:15.376003 I | raft: [f:1,l:1330,t:13,c:1330,a:0] [00000:00001] t13 became follower
|| 2023-02-14 16:51:15.376003 I | rsm: [00000:00001] no snapshot available during launch
|| 2023-02-14 16:51:15.376003 I | dragonboat: [00000:00001] initialized using <00000:00001:0>
|| 2023-02-14 16:51:15.376003 I | dragonboat: [00000:00001] initial index set to 0
|| 2023-02-14 16:51:15.376003 I | dragonboat: [00000:00002] replaying raft logs
|| 2023-02-14 16:51:15.376003 I | dragonboat: [00000:00002] has logdb entries size 1330 commit 1330 term 13
|| 2023-02-14 16:51:15.376003 I | raft: [00000:00002] created, initial: true, new: false
|| 2023-02-14 16:51:15.376003 I | raft: [00000:00002] raft log rate limit enabled: false, 0
|| 2023-02-14 16:51:15.376003 I | raft: [f:1,l:1330,t:13,c:1330,a:0] [00000:00002] t13 became follower
|| 2023-02-14 16:51:15.376003 I | rsm: [00000:00002] no snapshot available during launch
|| 2023-02-14 16:51:15.376003 I | dragonboat: [00000:00002] initialized using <00000:00002:0>
|| 2023-02-14 16:51:15.376003 I | dragonboat: [00000:00002] initial index set to 0
|| 2023-02-14 16:51:15.377999 I | dragonboat: [00000:00003] replaying raft logs
|| 2023-02-14 16:51:15.377999 I | dragonboat: [00000:00003] has logdb entries size 1330 commit 1330 term 13
|| 2023-02-14 16:51:15.377999 I | raft: [00000:00003] created, initial: true, new: false
|| 2023-02-14 16:51:15.377999 I | raft: [00000:00003] raft log rate limit enabled: false, 0
|| 2023-02-14 16:51:15.377999 I | raft: [f:1,l:1330,t:13,c:1330,a:0] [00000:00003] t13 became follower
|| 2023-02-14 16:51:15.377999 I | rsm: [00000:00003] no snapshot available during launch
|| 2023-02-14 16:51:15.377999 I | dragonboat: [00000:00003] initialized using <00000:00003:0>
|| 2023-02-14 16:51:15.377999 I | dragonboat: [00000:00003] initial index set to 0
|| 2023-02-14 16:51:15.378999 W | raft: [f:1,l:1330,t:13,c:1330,a:0] [00000:00001] t13 dropped ReadIndex, no leader
|| 2023-02-14 16:51:15.378999 W | raft: [f:1,l:1330,t:13,c:1330,a:0] [00000:00001] t13 dropped LeaderTransfer, no leader
|| 2023-02-14 16:51:15.378999 I | rsm: [00000:00001] applied ADD ccid 0 (1), n00001 (<IP_ADDRESS>:3001)
|| 2023-02-14 16:51:15.378999 I | rsm: [00000:00001] applied ADD ccid 0 (2), n00002 (<IP_ADDRESS>:3002)
|| 2023-02-14 16:51:15.378999 I | rsm: [00000:00001] applied ADD ccid 0 (3), n00003 (<IP_ADDRESS>:3003)
|| 2023-02-14 16:51:15.436895 I | rsm: [00000:00002] applied ADD ccid 0 (1), n00001 (<IP_ADDRESS>:3001)
|| 2023-02-14 16:51:15.436895 I | rsm: [00000:00002] applied ADD ccid 0 (2), n00002 (<IP_ADDRESS>:3002)
|| 2023-02-14 16:51:15.436895 I | rsm: [00000:00002] applied ADD ccid 0 (3), n00003 (<IP_ADDRESS>:3003)
|| 2023-02-14 16:51:15.484188 I | rsm: [00000:00003] applied ADD ccid 0 (1), n00001 (<IP_ADDRESS>:3001)
|| 2023-02-14 16:51:15.484680 I | rsm: [00000:00003] applied ADD ccid 0 (2), n00002 (<IP_ADDRESS>:3002)
|| 2023-02-14 16:51:15.484686 I | rsm: [00000:00003] applied ADD ccid 0 (3), n00003 (<IP_ADDRESS>:3003)
|| 2023-02-14 16:51:16.384139 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00001] t13 dropped ReadIndex, no leader
|| 2023-02-14 16:51:16.584278 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00003] t14 became candidate
|| 2023-02-14 16:51:16.584278 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00003] t14 received RequestVoteResp from n00003
|| 2023-02-14 16:51:16.584278 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00003] t14 sent RequestVote to n00001
|| 2023-02-14 16:51:16.584278 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00003] t14 sent RequestVote to n00002
|| 2023-02-14 16:51:16.593644 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00001] t13 received RequestVote with higher term (14) from n00003
|| 2023-02-14 16:51:16.593644 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00001] t13 become followerKE after receiving higher term from n00003
|| 2023-02-14 16:51:16.593644 I | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00001] t14 became follower
|| 2023-02-14 16:51:16.593644 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00001] t14 cast vote from n00003 index 1330 term 14, log term: 13
|| 2023-02-14 16:51:16.593644 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00002] t13 received RequestVote with higher term (14) from n00003
|| 2023-02-14 16:51:16.593644 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00002] t13 become followerKE after receiving higher term from n00003
|| 2023-02-14 16:51:16.593644 I | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00002] t14 became follower
|| 2023-02-14 16:51:16.593644 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00002] t14 cast vote from n00003 index 1330 term 14, log term: 13
|| 2023-02-14 16:51:16.595686 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00003] t14 received RequestVoteResp from n00001
|| 2023-02-14 16:51:16.595686 W | raft: [f:1,l:1330,t:13,c:1330,a:1330] [00000:00003] t14 received 2 votes and 0 rejections, quorum is 2
|| 2023-02-14 16:51:16.595686 I | raft: [f:1,l:1331,t:14,c:1330,a:1330] [00000:00003] t14 became leader
|| phase1
|| 2023-02-14 16:51:17.422183 E | transport: send batch failed, target <IP_ADDRESS>:3003 (write tcp <IP_ADDRESS>:49747-><IP_ADDRESS>:3003: wsasend: An existing connection was forcibly closed by the remote host.), 1
|| 2023-02-14 16:51:17.422183 W | transport: breaker <IP_ADDRESS>:3001 to <IP_ADDRESS>:3003 failed, connect and process failed: write tcp <IP_ADDRESS>:49747-><IP_ADDRESS>:3003: wsasend: An existing connection was forcibly closed by the remote host.
|| 2023-02-14 16:51:17.422183 W | transport: <IP_ADDRESS>:3003 became unreachable, affected 1 nodes
|| phase2
|| 2023-02-14 16:51:18.413507 I | dragonboat: go version: go1.19, windows/amd64
|| 2023-02-14 16:51:18.413507 I | dragonboat: dragonboat version: 3.3.5 (Rel)
|| 2023-02-14 16:51:18.413507 W | dragonboat: unsupported OS/ARCH windows/amd64, don't use for production
|| 2023-02-14 16:51:18.413507 I | config: using default EngineConfig
|| 2023-02-14 16:51:18.413507 I | config: using default LogDBConfig
|| 2023-02-14 16:51:18.413507 I | dragonboat: DeploymentID set to 1
|| 2023-02-14 16:51:18.425109 I | [WARN] node 1 shard 0 tick err: timeout
|| 2023-02-14 16:51:18.425109 I | dragonboat: LogDB info received, shard 0, busy false
|| 2023-02-14 16:51:18.427746 I | dragonboat: LogDB info received, shard 1, busy false
|| 2023-02-14 16:51:18.430809 I | dragonboat: LogDB info received, shard 2, busy false
|| 2023-02-14 16:51:18.433810 I | dragonboat: LogDB info received, shard 3, busy false
|| 2023-02-14 16:51:18.436876 I | dragonboat: LogDB info received, shard 4, busy false
|| 2023-02-14 16:51:18.440034 I | dragonboat: LogDB info received, shard 5, busy false
|| 2023-02-14 16:51:18.442639 I | dragonboat: LogDB info received, shard 6, busy false
|| 2023-02-14 16:51:18.445815 I | dragonboat: LogDB info received, shard 7, busy false
|| 2023-02-14 16:51:18.448922 I | dragonboat: LogDB info received, shard 8, busy false
|| 2023-02-14 16:51:18.452620 I | dragonboat: LogDB info received, shard 9, busy false
|| 2023-02-14 16:51:18.456197 I | dragonboat: LogDB info received, shard 10, busy false
|| 2023-02-14 16:51:18.458931 I | dragonboat: LogDB info received, shard 11, busy false
|| 2023-02-14 16:51:18.461512 I | dragonboat: LogDB info received, shard 12, busy false
|| 2023-02-14 16:51:18.464511 I | dragonboat: LogDB info received, shard 13, busy false
|| 2023-02-14 16:51:18.469297 I | dragonboat: LogDB info received, shard 14, busy false
|| 2023-02-14 16:51:18.472697 I | dragonboat: LogDB info received, shard 15, busy false
|| 2023-02-14 16:51:18.472697 I | logdb: using plain logdb
|| 2023-02-14 16:51:18.473195 I | dragonboat: logdb memory limit: 8192 MBytes
|| 2023-02-14 16:51:18.473195 I | dragonboat: NodeHost ID: nhid-3
|| 2023-02-14 16:51:18.473195 I | dragonboat: using regular node registry
|| 2023-02-14 16:51:18.473195 I | dragonboat: filesystem error injection mode enabled: false
|| 2023-02-14 16:51:18.473696 I | transport: transport type: go-tcp-transport
|| 2023-02-14 16:51:18.473696 I | dragonboat: transport type: go-tcp-transport
|| 2023-02-14 16:51:18.473696 I | dragonboat: logdb type: sharded-pebble
|| 2023-02-14 16:51:18.473696 I | dragonboat: nodehost address: <IP_ADDRESS>:3003
|| 2023-02-14 16:51:18.474196 I | dragonboat: [00000:00003] replaying raft logs
|| 2023-02-14 16:51:18.474695 I | dragonboat: [00000:00003] has logdb entries size 1331 commit 1331 term 14
|| 2023-02-14 16:51:18.474695 I | raft: [00000:00003] created, initial: true, new: false
|| 2023-02-14 16:51:18.474695 I | raft: [00000:00003] raft log rate limit enabled: false, 0
|| 2023-02-14 16:51:18.474695 I | raft: [f:1,l:1331,t:14,c:1331,a:0] [00000:00003] t14 became follower
|| 2023-02-14 16:51:18.474695 I | rsm: [00000:00003] no snapshot available during launch
|| 2023-02-14 16:51:18.474695 I | dragonboat: [00000:00003] initialized using <00000:00003:0>
|| 2023-02-14 16:51:18.474695 I | dragonboat: [00000:00003] initial index set to 0
|| 2023-02-14 16:51:18.584054 I | rsm: [00000:00003] applied ADD ccid 0 (1), n00001 (<IP_ADDRESS>:3001)
|| 2023-02-14 16:51:18.584134 I | rsm: [00000:00003] applied ADD ccid 0 (2), n00002 (<IP_ADDRESS>:3002)
|| 2023-02-14 16:51:18.584134 I | rsm: [00000:00003] applied ADD ccid 0 (3), n00003 (<IP_ADDRESS>:3003)
|| 2023-02-14 16:51:18.939717 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00003] t14 dropped proposal, no leader
|| 2023-02-14 16:51:19.081221 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00001] t15 became candidate
|| 2023-02-14 16:51:19.081221 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00001] t15 received RequestVoteResp from n00001
|| 2023-02-14 16:51:19.081221 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00001] t15 sent RequestVote to n00002
|| 2023-02-14 16:51:19.081221 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00001] t15 sent RequestVote to n00003
|| 2023-02-14 16:51:19.083553 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00003] t14 received RequestVote with higher term (15) from n00001
|| 2023-02-14 16:51:19.083553 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00003] t14 become followerKE after receiving higher term from n00001
|| 2023-02-14 16:51:19.084053 I | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00003] t15 became follower
|| 2023-02-14 16:51:19.084053 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00003] t15 cast vote from n00001 index 1331 term 15, log term: 14
|| 2023-02-14 16:51:19.085718 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00002] t14 received RequestVote with higher term (15) from n00001
|| 2023-02-14 16:51:19.085718 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00002] t14 become followerKE after receiving higher term from n00001
|| 2023-02-14 16:51:19.085718 I | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00002] t15 became follower
|| 2023-02-14 16:51:19.085718 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00002] t15 cast vote from n00001 index 1331 term 15, log term: 14
|| 2023-02-14 16:51:19.086394 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00001] t15 received RequestVoteResp from n00003
|| 2023-02-14 16:51:19.086394 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00001] t15 received 2 votes and 0 rejections, quorum is 2
|| 2023-02-14 16:51:19.086394 I | raft: [f:1,l:1332,t:15,c:1331,a:1331] [00000:00001] t15 became leader
|| 2023-02-14 16:51:19.436801 I | [WARN] node 1 shard 0 tick err: timeout
|| phase3
|| 2023-02-14 16:51:19.492018 E | transport: send batch failed, target <IP_ADDRESS>:3002 (write tcp <IP_ADDRESS>:49751-><IP_ADDRESS>:3002: wsasend: An existing connection was forcibly closed by the remote host.), 1
|| 2023-02-14 16:51:19.492018 W | transport: breaker <IP_ADDRESS>:3001 to <IP_ADDRESS>:3002 failed, connect and process failed: write tcp <IP_ADDRESS>:49751-><IP_ADDRESS>:3002: wsasend: An existing connection was forcibly closed by the remote host.
|| 2023-02-14 16:51:19.492018 W | transport: <IP_ADDRESS>:3002 became unreachable, affected 1 nodes
|| phase4
|| 2023-02-14 16:51:20.509298 I | dragonboat: go version: go1.19, windows/amd64
|| 2023-02-14 16:51:20.509298 I | dragonboat: dragonboat version: 3.3.5 (Rel)
|| 2023-02-14 16:51:20.509298 W | dragonboat: unsupported OS/ARCH windows/amd64, don't use for production
|| 2023-02-14 16:51:20.509298 I | config: using default EngineConfig
|| 2023-02-14 16:51:20.509298 I | config: using default LogDBConfig
|| 2023-02-14 16:51:20.509298 I | dragonboat: DeploymentID set to 1
|| 2023-02-14 16:51:20.514325 I | dragonboat: LogDB info received, shard 0, busy false
|| 2023-02-14 16:51:20.518341 I | dragonboat: LogDB info received, shard 1, busy false
|| 2023-02-14 16:51:20.520319 I | dragonboat: LogDB info received, shard 2, busy false
|| 2023-02-14 16:51:20.523469 I | dragonboat: LogDB info received, shard 3, busy false
|| 2023-02-14 16:51:20.527125 I | dragonboat: LogDB info received, shard 4, busy false
|| 2023-02-14 16:51:20.529123 I | dragonboat: LogDB info received, shard 5, busy false
|| 2023-02-14 16:51:20.532139 I | dragonboat: LogDB info received, shard 6, busy false
|| 2023-02-14 16:51:20.536017 I | dragonboat: LogDB info received, shard 7, busy false
|| 2023-02-14 16:51:20.539014 I | dragonboat: LogDB info received, shard 8, busy false
|| 2023-02-14 16:51:20.542241 I | dragonboat: LogDB info received, shard 9, busy false
|| 2023-02-14 16:51:20.545859 I | dragonboat: LogDB info received, shard 10, busy false
|| 2023-02-14 16:51:20.548434 I | dragonboat: LogDB info received, shard 11, busy false
|| 2023-02-14 16:51:20.551543 I | dragonboat: LogDB info received, shard 12, busy false
|| 2023-02-14 16:51:20.554125 I | dragonboat: LogDB info received, shard 13, busy false
|| 2023-02-14 16:51:20.557211 I | dragonboat: LogDB info received, shard 14, busy false
|| 2023-02-14 16:51:20.560315 I | logdb: using plain logdb
|| 2023-02-14 16:51:20.560315 I | dragonboat: LogDB info received, shard 15, busy false
|| 2023-02-14 16:51:20.560315 I | dragonboat: logdb memory limit: 8192 MBytes
|| 2023-02-14 16:51:20.560315 I | dragonboat: NodeHost ID: nhid-2
|| 2023-02-14 16:51:20.560315 I | dragonboat: using regular node registry
|| 2023-02-14 16:51:20.560315 I | dragonboat: filesystem error injection mode enabled: false
|| 2023-02-14 16:51:20.560833 I | transport: transport type: go-tcp-transport
|| 2023-02-14 16:51:20.560833 I | dragonboat: transport type: go-tcp-transport
|| 2023-02-14 16:51:20.560833 I | dragonboat: logdb type: sharded-pebble
|| 2023-02-14 16:51:20.560833 I | dragonboat: nodehost address: <IP_ADDRESS>:3002
|| 2023-02-14 16:51:20.561361 I | dragonboat: [00000:00002] replaying raft logs
|| 2023-02-14 16:51:20.561913 I | dragonboat: [00000:00002] has logdb entries size 1334 commit 1334 term 15
|| 2023-02-14 16:51:20.561913 I | raft: [00000:00002] created, initial: true, new: false
|| 2023-02-14 16:51:20.561913 I | raft: [00000:00002] raft log rate limit enabled: false, 0
|| 2023-02-14 16:51:20.561913 I | raft: [f:1,l:1334,t:15,c:1334,a:0] [00000:00002] t15 became follower
|| 2023-02-14 16:51:20.561913 I | rsm: [00000:00002] no snapshot available during launch
|| 2023-02-14 16:51:20.561913 I | dragonboat: [00000:00002] initialized using <00000:00002:0>
|| 2023-02-14 16:51:20.561913 I | dragonboat: [00000:00002] initial index set to 0
|| 2023-02-14 16:51:20.664065 I | rsm: [00000:00002] applied ADD ccid 0 (1), n00001 (<IP_ADDRESS>:3001)
|| 2023-02-14 16:51:20.664065 I | rsm: [00000:00002] applied ADD ccid 0 (2), n00002 (<IP_ADDRESS>:3002)
|| 2023-02-14 16:51:20.664065 I | rsm: [00000:00002] applied ADD ccid 0 (3), n00003 (<IP_ADDRESS>:3003)
|| phase5
|| [Finished in 13 seconds]

no leader show between phase2 and phase3, and tick timeouted switch means interrupt.
2023-02-14 16:51:18.939717 W | raft: [f:1,l:1331,t:14,c:1331,a:1331] [00000:00003] t14 dropped proposal, no leader

Minimal reproduce code, dragonboat verision: v3.6.6
package main

import (
	"context"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"time"
	"errors"

	"github.com/lni/dragonboat/v3"
	"github.com/lni/dragonboat/v3/config"
	sm "github.com/lni/dragonboat/v3/statemachine"
	"github.com/lni/goutils/syncutil"
)

type exampleStateMachine struct {
	ClusterID uint64
	NodeID    uint64
	Tick      uint64
}

func NewExampleStateMachine(clusterID uint64, nodeID uint64) sm.IStateMachine {
	return &exampleStateMachine{ClusterID: clusterID, NodeID: nodeID}
}
func (it *exampleStateMachine) Update(req []byte) (sm.Result, error) {
	it.Tick++
	return sm.Result{}, nil
}
func (it *exampleStateMachine) Lookup(req interface{}) (interface{}, error) {
	return it.Tick, nil
}
func (it *exampleStateMachine) SaveSnapshot(w io.Writer, _ sm.ISnapshotFileCollection, _ <-chan struct{}) error {
	data := make([]byte, 8)
	binary.LittleEndian.PutUint64(data, it.Tick)
	w.Write(data)
	return nil
}
func (it *exampleStateMachine) RecoverFromSnapshot(r io.Reader, _ []sm.SnapshotFile, _ <-chan struct{}) error {
	data, _ := io.ReadAll(r)
	v := binary.LittleEndian.Uint64(data)
	it.Tick = v
	return nil
}
func (it *exampleStateMachine) Close() error {
	return nil
}

func newNodeHost(nhid uint64, addr string) *dragonboat.NodeHost {
	conf := config.NodeHostConfig{
		NodeHostDir:         fmt.Sprintf("single_nodehost_test_dir_safe_to_delete/%d", nhid),
		AddressByNodeHostID: false,
		RTTMillisecond:      100,
		RaftAddress:         addr,
		Expert: config.ExpertConfig{
			TestNodeHostID: nhid,
		},
	}
	nh, err := dragonboat.NewNodeHost(conf)
	if err != nil {
		panic(err)
	}
	return nh
}

func startShard(nh *dragonboat.NodeHost, clusterID uint64, nodeID uint64, initials map[uint64]string, fn interface{}) {
	conf := config.Config{
		NodeID:              nodeID,
		ClusterID:           clusterID,
		CheckQuorum:         true,
		ElectionRTT:         10,
		HeartbeatRTT:        1,
		SnapshotEntries:     0,
		CompactionOverhead:  0,
		OrderedConfigChange: false,
	}
	var err error
	switch f := fn.(type) {
	case sm.CreateStateMachineFunc:
		if len(initials) > 0 {
			err = nh.StartCluster(initials, false, f, conf)
		} else {
			err = nh.StartCluster(nil, false, f, conf)
		}
	case sm.CreateConcurrentStateMachineFunc:
		if len(initials) > 0 {
			err = nh.StartConcurrentCluster(initials, false, f, conf)
		} else {
			err = nh.StartConcurrentCluster(nil, false, f, conf)
		}
	default:
		panic(fmt.Errorf("unknown fn type: %T", f))
	}
	if err != nil {
		panic(err)
	}
}

func tick(nh *dragonboat.NodeHost, clusterID uint64) *syncutil.Stopper {
	stopper := syncutil.NewStopper()
	stopper.RunWorker(func() {
		t := time.NewTicker(time.Millisecond * 30)
		defer t.Stop()
		nop := nh.GetNoOPSession(clusterID)
		ctx := context.Background()
		for {
			select {
			case <-t.C:
				ctx1, cancel := context.WithTimeout(ctx, time.Second)
				_, err := nh.SyncPropose(ctx1, nop, []byte("tick"))
				cancel()
				if err != nil {
					// log.Fatalln("[FATAL]", fmt.Errorf("node %d shahrd %d tick err: %w", nh.NodeHostConfig().Expert.TestNodeHostID, clusterID, err))
					log.Println("[WARN]", fmt.Errorf("node %d shard %d tick err: %w", nh.NodeHostConfig().Expert.TestNodeHostID, clusterID, err))
				}
			case <-stopper.ShouldStop():
				return
			}
		}
	})
	return stopper
}

func waitReady(nh *dragonboat.NodeHost, clusterID uint64) {
	for {
		ctx, cancel := context.WithTimeout(context.TODO(), time.Second)
		defer cancel()
		if _, err := nh.SyncRead(ctx, clusterID, nil); err != nil {
			if errors.Is(err, dragonboat.ErrClusterNotReady) {
				time.Sleep(time.Second)
				continue
			}
		}
		return
	}
}

func main() {
	clusterID := uint64(0)
	fn := sm.CreateStateMachineFunc(NewExampleStateMachine)
	initials := map[uint64]string{
		1: "<IP_ADDRESS>:3001",
		2: "<IP_ADDRESS>:3002",
		3: "<IP_ADDRESS>:3003",
	}
	nh1 := newNodeHost(1, initials[1])
	nh2 := newNodeHost(2, initials[2])
	nh3 := newNodeHost(3, initials[3])
	startShard(nh1, clusterID, 1, initials, fn)
	startShard(nh2, clusterID, 2, initials, fn)
	startShard(nh3, clusterID, 3, initials, fn)
	if err := nh1.RequestLeaderTransfer(clusterID, 1); err != nil {
		log.Fatalln("[FATAL]", fmt.Errorf("RequestLeaderTransfer err: %w", err))
	}
	waitReady(nh1, clusterID)
	stopper := tick(nh1, clusterID)
	defer stopper.Stop()
	println("phase1")

	nh3.Stop()
	time.Sleep(time.Second*3)
	println("phase2")

	nh3 = newNodeHost(3, initials[3])
	startShard(nh3, clusterID, 3, nil, fn)
	waitReady(nh3, clusterID)
	println("phase3")

	nh2.Stop()
	time.Sleep(time.Second*3)
	println("phase4")

	nh2 = newNodeHost(2, initials[2])
	startShard(nh2, clusterID, 2, nil, fn)
	waitReady(nh2, clusterID)
	println("phase5")

	stopper.Stop()
	stopper = tick(nh3, clusterID)

	if err := nh1.RequestLeaderTransfer(clusterID, 3); err != nil {
		log.Fatalln("[FATAL]", fmt.Errorf("RequestLeaderTransfer err: %w", err))
	}
	println("phase6")

	nh1.Stop()
	time.Sleep(time.Second*3)
	println("phase7")

	nh1 = newNodeHost(1, initials[1])
	startShard(nh1, clusterID, 1, nil, fn)
	waitReady(nh1, clusterID)
	println("phase8")

	time.Sleep(time.Second*5)
}

log
|| [go run .]
|| 2023-02-14 17:17:41.958516 I | dragonboat: go version: go1.19, windows/amd64
|| 2023-02-14 17:17:41.958516 I | dragonboat: dragonboat version: 3.3.5 (Rel)
|| 2023-02-14 17:17:41.958516 W | dragonboat: unsupported OS/ARCH windows/amd64, don't use for production
|| 2023-02-14 17:17:41.959032 I | config: using default EngineConfig
|| 2023-02-14 17:17:41.959032 I | config: using default LogDBConfig
|| 2023-02-14 17:17:41.959032 I | dragonboat: DeploymentID set to 1
|| 2023-02-14 17:17:41.964515 I | dragonboat: LogDB info received, shard 0, busy false
|| 2023-02-14 17:17:41.967515 I | dragonboat: LogDB info received, shard 1, busy false
|| 2023-02-14 17:17:41.970516 I | dragonboat: LogDB info received, shard 2, busy false
|| 2023-02-14 17:17:41.973515 I | dragonboat: LogDB info received, shard 3, busy false
|| 2023-02-14 17:17:41.976535 I | dragonboat: LogDB info received, shard 4, busy false
|| 2023-02-14 17:17:41.980042 I | dragonboat: LogDB info received, shard 5, busy false
|| 2023-02-14 17:17:41.983041 I | dragonboat: LogDB info received, shard 6, busy false
|| 2023-02-14 17:17:41.986134 I | dragonboat: LogDB info received, shard 7, busy false
|| 2023-02-14 17:17:41.989649 I | dragonboat: LogDB info received, shard 8, busy false
|| 2023-02-14 17:17:41.992649 I | dragonboat: LogDB info received, shard 9, busy false
|| 2023-02-14 17:17:41.996373 I | dragonboat: LogDB info received, shard 10, busy false
|| 2023-02-14 17:17:41.998869 I | dragonboat: LogDB info received, shard 11, busy false
|| 2023-02-14 17:17:42.002190 I | dragonboat: LogDB info received, shard 12, busy false
|| 2023-02-14 17:17:42.005686 I | dragonboat: LogDB info received, shard 13, busy false
|| 2023-02-14 17:17:42.008186 I | dragonboat: LogDB info received, shard 14, busy false
|| 2023-02-14 17:17:42.011185 I | dragonboat: LogDB info received, shard 15, busy false
|| 2023-02-14 17:17:42.011185 I | logdb: using plain logdb
|| 2023-02-14 17:17:42.011185 I | dragonboat: logdb memory limit: 8192 MBytes
|| 2023-02-14 17:17:42.011185 I | dragonboat: NodeHost ID: nhid-1
|| 2023-02-14 17:17:42.011185 I | dragonboat: using regular node registry
|| 2023-02-14 17:17:42.011185 I | dragonboat: filesystem error injection mode enabled: false
|| 2023-02-14 17:17:42.012849 I | transport: transport type: go-tcp-transport
|| 2023-02-14 17:17:42.015470 I | dragonboat: transport type: go-tcp-transport
|| 2023-02-14 17:17:42.015470 I | dragonboat: logdb type: sharded-pebble
|| 2023-02-14 17:17:42.015470 I | dragonboat: nodehost address: <IP_ADDRESS>:3001
|| 2023-02-14 17:17:42.015470 I | dragonboat: go version: go1.19, windows/amd64
|| 2023-02-14 17:17:42.015470 I | dragonboat: dragonboat version: 3.3.5 (Rel)
|| 2023-02-14 17:17:42.015470 W | dragonboat: unsupported OS/ARCH windows/amd64, don't use for production
|| 2023-02-14 17:17:42.015470 I | config: using default EngineConfig
|| 2023-02-14 17:17:42.015470 I | config: using default LogDBConfig
|| 2023-02-14 17:17:42.015470 I | dragonboat: DeploymentID set to 1
|| 2023-02-14 17:17:42.020646 I | dragonboat: LogDB info received, shard 0, busy false
|| 2023-02-14 17:17:42.023233 I | dragonboat: LogDB info received, shard 1, busy false
|| 2023-02-14 17:17:42.026861 I | dragonboat: LogDB info received, shard 2, busy false
|| 2023-02-14 17:17:42.029408 I | dragonboat: LogDB info received, shard 3, busy false
|| 2023-02-14 17:17:42.032512 I | dragonboat: LogDB info received, shard 4, busy false
|| 2023-02-14 17:17:42.036604 I | dragonboat: LogDB info received, shard 5, busy false
|| 2023-02-14 17:17:42.039054 I | dragonboat: LogDB info received, shard 6, busy false
|| 2023-02-14 17:17:42.042054 I | dragonboat: LogDB info received, shard 7, busy false
|| 2023-02-14 17:17:42.045745 I | dragonboat: LogDB info received, shard 8, busy false
|| 2023-02-14 17:17:42.048933 I | dragonboat: LogDB info received, shard 9, busy false
|| 2023-02-14 17:17:42.052546 I | dragonboat: LogDB info received, shard 10, busy false
|| 2023-02-14 17:17:42.055937 I | dragonboat: LogDB info received, shard 11, busy false
|| 2023-02-14 17:17:42.059433 I | dragonboat: LogDB info received, shard 12, busy false
|| 2023-02-14 17:17:42.062565 I | dragonboat: LogDB info received, shard 13, busy false
|| 2023-02-14 17:17:42.066266 I | dragonboat: LogDB info received, shard 14, busy false
|| 2023-02-14 17:17:42.069384 I | logdb: using plain logdb
|| 2023-02-14 17:17:42.069384 I | dragonboat: LogDB info received, shard 15, busy false
|| 2023-02-14 17:17:42.069896 I | dragonboat: logdb memory limit: 8192 MBytes
|| 2023-02-14 17:17:42.069896 I | dragonboat: NodeHost ID: nhid-2
|| 2023-02-14 17:17:42.069896 I | dragonboat: using regular node registry
|| 2023-02-14 17:17:42.069896 I | dragonboat: filesystem error injection mode enabled: false
|| 2023-02-14 17:17:42.070973 I | transport: transport type: go-tcp-transport
|| 2023-02-14 17:17:42.071479 I | dragonboat: transport type: go-tcp-transport
|| 2023-02-14 17:17:42.071479 I | dragonboat: logdb type: sharded-pebble
|| 2023-02-14 17:17:42.071479 I | dragonboat: nodehost address: <IP_ADDRESS>:3002
|| 2023-02-14 17:17:42.071479 I | dragonboat: go version: go1.19, windows/amd64
|| 2023-02-14 17:17:42.071479 I | dragonboat: dragonboat version: 3.3.5 (Rel)
|| 2023-02-14 17:17:42.071479 W | dragonboat: unsupported OS/ARCH windows/amd64, don't use for production
|| 2023-02-14 17:17:42.071479 I | config: using default EngineConfig
|| 2023-02-14 17:17:42.071479 I | config: using default LogDBConfig
|| 2023-02-14 17:17:42.071558 I | dragonboat: DeploymentID set to 1
|| 2023-02-14 17:17:42.076060 I | dragonboat: LogDB info received, shard 0, busy false
|| 2023-02-14 17:17:42.079203 I | dragonboat: LogDB info received, shard 1, busy false
|| 2023-02-14 17:17:42.082219 I | dragonboat: LogDB info received, shard 2, busy false
|| 2023-02-14 17:17:42.085220 I | dragonboat: LogDB info received, shard 3, busy false
|| 2023-02-14 17:17:42.087897 I | dragonboat: LogDB info received, shard 4, busy false
|| 2023-02-14 17:17:42.090896 I | dragonboat: LogDB info received, shard 5, busy false
|| 2023-02-14 17:17:42.093894 I | dragonboat: LogDB info received, shard 6, busy false
|| 2023-02-14 17:17:42.097009 I | dragonboat: LogDB info received, shard 7, busy false
|| 2023-02-14 17:17:42.100196 I | dragonboat: LogDB info received, shard 8, busy false
|| 2023-02-14 17:17:42.103345 I | dragonboat: LogDB info received, shard 9, busy false
|| 2023-02-14 17:17:42.105987 I | dragonboat: LogDB info received, shard 10, busy false
|| 2023-02-14 17:17:42.108622 I | dragonboat: LogDB info received, shard 11, busy false
|| 2023-02-14 17:17:42.111842 I | dragonboat: LogDB info received, shard 12, busy false
|| 2023-02-14 17:17:42.114451 I | dragonboat: LogDB info received, shard 13, busy false
|| 2023-02-14 17:17:42.117568 I | dragonboat: LogDB info received, shard 14, busy false
|| 2023-02-14 17:17:42.120759 I | dragonboat: LogDB info received, shard 15, busy false
|| 2023-02-14 17:17:42.120759 I | logdb: using plain logdb
|| 2023-02-14 17:17:42.121272 I | dragonboat: logdb memory limit: 8192 MBytes
|| 2023-02-14 17:17:42.121272 I | dragonboat: NodeHost ID: nhid-3
|| 2023-02-14 17:17:42.121272 I | dragonboat: using regular node registry
|| 2023-02-14 17:17:42.121272 I | dragonboat: filesystem error injection mode enabled: false
|| 2023-02-14 17:17:42.122064 I | transport: transport type: go-tcp-transport
|| 2023-02-14 17:17:42.122580 I | dragonboat: transport type: go-tcp-transport
|| 2023-02-14 17:17:42.122580 I | dragonboat: logdb type: sharded-pebble
|| 2023-02-14 17:17:42.122580 I | dragonboat: nodehost address: <IP_ADDRESS>:3003
|| 2023-02-14 17:17:42.123139 I | dragonboat: [00000:00001] replaying raft logs
|| 2023-02-14 17:17:42.123139 I | dragonboat: [00000:00001] has logdb entries size 1602 commit 1602 term 12
|| 2023-02-14 17:17:42.123139 I | raft: [00000:00001] created, initial: true, new: false
|| 2023-02-14 17:17:42.123139 I | raft: [00000:00001] raft log rate limit enabled: false, 0
|| 2023-02-14 17:17:42.123139 I | raft: [f:1,l:1602,t:12,c:1602,a:0] [00000:00001] t12 became follower
|| 2023-02-14 17:17:42.123688 I | rsm: [00000:00001] no snapshot available during launch
|| 2023-02-14 17:17:42.123688 I | dragonboat: [00000:00001] initialized using <00000:00001:0>
|| 2023-02-14 17:17:42.123688 I | dragonboat: [00000:00001] initial index set to 0
|| 2023-02-14 17:17:42.124208 I | dragonboat: [00000:00002] replaying raft logs
|| 2023-02-14 17:17:42.124208 I | dragonboat: [00000:00002] has logdb entries size 1602 commit 1602 term 12
|| 2023-02-14 17:17:42.124208 I | raft: [00000:00002] created, initial: true, new: false
|| 2023-02-14 17:17:42.124208 I | raft: [00000:00002] raft log rate limit enabled: false, 0
|| 2023-02-14 17:17:42.124208 I | raft: [f:1,l:1602,t:12,c:1602,a:0] [00000:00002] t12 became follower
|| 2023-02-14 17:17:42.124208 I | rsm: [00000:00002] no snapshot available during launch
|| 2023-02-14 17:17:42.124208 I | dragonboat: [00000:00002] initialized using <00000:00002:0>
|| 2023-02-14 17:17:42.124208 I | dragonboat: [00000:00002] initial index set to 0
|| 2023-02-14 17:17:42.124720 I | dragonboat: [00000:00003] replaying raft logs
|| 2023-02-14 17:17:42.124720 I | dragonboat: [00000:00003] has logdb entries size 1602 commit 1602 term 12
|| 2023-02-14 17:17:42.124720 I | raft: [00000:00003] created, initial: true, new: false
|| 2023-02-14 17:17:42.124720 I | raft: [00000:00003] raft log rate limit enabled: false, 0
|| 2023-02-14 17:17:42.124720 I | raft: [f:1,l:1602,t:12,c:1602,a:0] [00000:00003] t12 became follower
|| 2023-02-14 17:17:42.124720 I | rsm: [00000:00003] no snapshot available during launch
|| 2023-02-14 17:17:42.124720 I | dragonboat: [00000:00003] initialized using <00000:00003:0>
|| 2023-02-14 17:17:42.124720 I | dragonboat: [00000:00003] initial index set to 0
|| 2023-02-14 17:17:42.124720 W | raft: [f:1,l:1602,t:12,c:1602,a:0] [00000:00001] t12 dropped ReadIndex, no leader
|| 2023-02-14 17:17:42.124720 W | raft: [f:1,l:1602,t:12,c:1602,a:0] [00000:00001] t12 dropped LeaderTransfer, no leader
|| 2023-02-14 17:17:42.125233 I | rsm: [00000:00001] applied ADD ccid 0 (1), n00001 (<IP_ADDRESS>:3001)
|| 2023-02-14 17:17:42.125233 I | rsm: [00000:00001] applied ADD ccid 0 (2), n00002 (<IP_ADDRESS>:3002)
|| 2023-02-14 17:17:42.125233 I | rsm: [00000:00001] applied ADD ccid 0 (3), n00003 (<IP_ADDRESS>:3003)
|| 2023-02-14 17:17:42.173643 I | rsm: [00000:00002] applied ADD ccid 0 (1), n00001 (<IP_ADDRESS>:3001)
|| 2023-02-14 17:17:42.173643 I | rsm: [00000:00002] applied ADD ccid 0 (2), n00002 (<IP_ADDRESS>:3002)
|| 2023-02-14 17:17:42.173909 I | rsm: [00000:00002] applied ADD ccid 0 (3), n00003 (<IP_ADDRESS>:3003)
|| 2023-02-14 17:17:42.233179 I | rsm: [00000:00003] applied ADD ccid 0 (1), n00001 (<IP_ADDRESS>:3001)
|| 2023-02-14 17:17:42.233179 I | rsm: [00000:00003] applied ADD ccid 0 (2), n00002 (<IP_ADDRESS>:3002)
|| 2023-02-14 17:17:42.233179 I | rsm: [00000:00003] applied ADD ccid 0 (3), n00003 (<IP_ADDRESS>:3003)
|| 2023-02-14 17:17:43.135162 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00001] t12 dropped ReadIndex, no leader
|| 2023-02-14 17:17:43.181232 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00002] t13 became candidate
|| 2023-02-14 17:17:43.181232 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00002] t13 received RequestVoteResp from n00002
|| 2023-02-14 17:17:43.181232 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00002] t13 sent RequestVote to n00003
|| 2023-02-14 17:17:43.181758 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00002] t13 sent RequestVote to n00001
|| 2023-02-14 17:17:43.189452 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00001] t12 received RequestVote with higher term (13) from n00002
|| 2023-02-14 17:17:43.189452 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00001] t12 become followerKE after receiving higher term from n00002
|| 2023-02-14 17:17:43.189452 I | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00001] t13 became follower
|| 2023-02-14 17:17:43.190450 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00001] t13 cast vote from n00002 index 1602 term 13, log term: 12
|| 2023-02-14 17:17:43.191450 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00003] t12 received RequestVote with higher term (13) from n00002
|| 2023-02-14 17:17:43.191450 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00003] t12 become followerKE after receiving higher term from n00002
|| 2023-02-14 17:17:43.192449 I | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00003] t13 became follower
|| 2023-02-14 17:17:43.192449 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00003] t13 cast vote from n00002 index 1602 term 13, log term: 12
|| 2023-02-14 17:17:43.196047 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00002] t13 received RequestVoteResp from n00001
|| 2023-02-14 17:17:43.196047 W | raft: [f:1,l:1602,t:12,c:1602,a:1602] [00000:00002] t13 received 2 votes and 0 rejections, quorum is 2
|| 2023-02-14 17:17:43.196047 I | raft: [f:1,l:1603,t:13,c:1602,a:1602] [00000:00002] t13 became leader
|| phase1
|| 2023-02-14 17:17:44.177114 E | transport: send batch failed, target <IP_ADDRESS>:3003 (write tcp <IP_ADDRESS>:52308-><IP_ADDRESS>:3003: wsasend: An existing connection was forcibly closed by the remote host.), 1
|| 2023-02-14 17:17:44.177114 W | transport: breaker <IP_ADDRESS>:3002 to <IP_ADDRESS>:3003 failed, connect and process failed: write tcp <IP_ADDRESS>:52308-><IP_ADDRESS>:3003: wsasend: An existing connection was forcibly closed by the remote host.
|| 2023-02-14 17:17:44.177114 W | transport: <IP_ADDRESS>:3003 became unreachable, affected 1 nodes
|| 2023-02-14 17:17:46.826915 E | transport: Nodehost <IP_ADDRESS>:3002 failed to get a connection to <IP_ADDRESS>:3003, dial tcp <IP_ADDRESS>:3003: connectex: No connection could be made because the target machine actively refused it.
|| 2023-02-14 17:17:46.826915 W | transport: breaker <IP_ADDRESS>:3002 to <IP_ADDRESS>:3003 failed, connect and process failed: dial tcp <IP_ADDRESS>:3003: connectex: No connection could be made because the target machine actively refused it.
|| 2023-02-14 17:17:46.826915 W | transport: <IP_ADDRESS>:3003 became unreachable, affected 0 nodes
|| phase2
|| 2023-02-14 17:17:47.169227 I | dragonboat: go version: go1.19, windows/amd64
|| 2023-02-14 17:17:47.169227 I | dragonboat: dragonboat version: 3.3.5 (Rel)
|| 2023-02-14 17:17:47.169227 W | dragonboat: unsupported OS/ARCH windows/amd64, don't use for production
|| 2023-02-14 17:17:47.169227 I | config: using default EngineConfig
|| 2023-02-14 17:17:47.169227 I | config: using default LogDBConfig
|| 2023-02-14 17:17:47.169472 I | dragonboat: DeploymentID set to 1
|| 2023-02-14 17:17:47.178334 I | dragonboat: LogDB info received, shard 0, busy false
|| 2023-02-14 17:17:47.181350 I | dragonboat: LogDB info received, shard 1, busy false
|| 2023-02-14 17:17:47.185958 I | dragonboat: LogDB info received, shard 2, busy false
|| 2023-02-14 17:17:47.189458 I | dragonboat: LogDB info received, shard 3, busy false
|| 2023-02-14 17:17:47.193457 I | dragonboat: LogDB info received, shard 4, busy false
|| 2023-02-14 17:17:47.196957 I | dragonboat: LogDB info received, shard 5, busy false
|| 2023-02-14 17:17:47.200404 I | dragonboat: LogDB info received, shard 6, busy false
|| 2023-02-14 17:17:47.204289 I | dragonboat: LogDB info received, shard 7, busy false
|| 2023-02-14 17:17:47.207289 I | dragonboat: LogDB info received, shard 8, busy false
|| 2023-02-14 17:17:47.210808 I | dragonboat: LogDB info received, shard 9, busy false
|| 2023-02-14 17:17:47.213791 I | dragonboat: LogDB info received, shard 10, busy false
|| 2023-02-14 17:17:47.216723 I | dragonboat: LogDB info received, shard 11, busy false
|| 2023-02-14 17:17:47.219718 I | dragonboat: LogDB info received, shard 12, busy false
|| 2023-02-14 17:17:47.223395 I | dragonboat: LogDB info received, shard 13, busy false
|| 2023-02-14 17:17:47.226391 I | dragonboat: LogDB info received, shard 14, busy false
|| 2023-02-14 17:17:47.229406 I | dragonboat: LogDB info received, shard 15, busy false
|| 2023-02-14 17:17:47.229406 I | logdb: using plain logdb
|| 2023-02-14 17:17:47.229406 I | dragonboat: logdb memory limit: 8192 MBytes
|| 2023-02-14 17:17:47.229406 I | dragonboat: NodeHost ID: nhid-3
|| 2023-02-14 17:17:47.229406 I | dragonboat: using regular node registry
|| 2023-02-14 17:17:47.229406 I | dragonboat: filesystem error injection mode enabled: false
|| 2023-02-14 17:17:47.229406 I | transport: transport type: go-tcp-transport
|| 2023-02-14 17:17:47.229406 I | dragonboat: transport type: go-tcp-transport
|| 2023-02-14 17:17:47.229406 I | dragonboat: logdb type: sharded-pebble
|| 2023-02-14 17:17:47.229406 I | dragonboat: nodehost address: <IP_ADDRESS>:3003
|| 2023-02-14 17:17:47.230376 I | dragonboat: [00000:00003] replaying raft logs
|| 2023-02-14 17:17:47.230376 I | dragonboat: [00000:00003] has logdb entries size 1603 commit 1603 term 13
|| 2023-02-14 17:17:47.230376 I | raft: [00000:00003] created, initial: true, new: false
|| 2023-02-14 17:17:47.230376 I | raft: [00000:00003] raft log rate limit enabled: false, 0
|| 2023-02-14 17:17:47.230376 I | raft: [f:1,l:1603,t:13,c:1603,a:0] [00000:00003] t13 became follower
|| 2023-02-14 17:17:47.231377 I | rsm: [00000:00003] no snapshot available during launch
|| 2023-02-14 17:17:47.231377 I | dragonboat: [00000:00003] initialized using <00000:00003:0>
|| 2023-02-14 17:17:47.231459 I | dragonboat: [00000:00003] initial index set to 0
|| 2023-02-14 17:17:47.340019 I | rsm: [00000:00003] applied ADD ccid 0 (1), n00001 (<IP_ADDRESS>:3001)
|| 2023-02-14 17:17:47.340019 I | rsm: [00000:00003] applied ADD ccid 0 (2), n00002 (<IP_ADDRESS>:3002)
|| 2023-02-14 17:17:47.340019 I | rsm: [00000:00003] applied ADD ccid 0 (3), n00003 (<IP_ADDRESS>:3003)
|| 2023-02-14 17:17:48.241145 W | raft: [f:1,l:1603,t:13,c:1603,a:1603] [00000:00003] t13 dropped ReadIndex, no leader
|| phase3
|| 2023-02-14 17:17:49.278337 E | transport: send batch failed, target <IP_ADDRESS>:3002 (write tcp <IP_ADDRESS>:52310-><IP_ADDRESS>:3002: wsasend: An existing connection was forcibly closed by the remote host.), 1
|| 2023-02-14 17:17:49.278337 W | transport: breaker <IP_ADDRESS>:3001 to <IP_ADDRESS>:3002 failed, connect and process failed: write tcp <IP_ADDRESS>:52310-><IP_ADDRESS>:3002: wsasend: An existing connection was forcibly closed by the remote host.
|| 2023-02-14 17:17:49.278337 W | transport: <IP_ADDRESS>:3002 became unreachable, affected 1 nodes
|| 2023-02-14 17:17:50.289025 I | [WARN] node 1 shard 0 tick err: timeout
|| 2023-02-14 17:17:51.129979 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00003] t14 became candidate
|| 2023-02-14 17:17:51.129979 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00003] t14 received RequestVoteResp from n00003
|| 2023-02-14 17:17:51.129979 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00003] t14 sent RequestVote to n00002
|| 2023-02-14 17:17:51.129979 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00003] t14 sent RequestVote to n00001
|| 2023-02-14 17:17:51.130111 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 became candidate
|| 2023-02-14 17:17:51.130111 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 received RequestVoteResp from n00001
|| 2023-02-14 17:17:51.130111 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 sent RequestVote to n00002
|| 2023-02-14 17:17:51.130111 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 sent RequestVote to n00003
|| 2023-02-14 17:17:51.131138 E | transport: send batch failed, target <IP_ADDRESS>:3002 (write tcp <IP_ADDRESS>:52319-><IP_ADDRESS>:3002: wsasend: An existing connection was forcibly closed by the remote host.), 1
|| 2023-02-14 17:17:51.131138 W | transport: breaker <IP_ADDRESS>:3003 to <IP_ADDRESS>:3002 failed, connect and process failed: write tcp <IP_ADDRESS>:52319-><IP_ADDRESS>:3002: wsasend: An existing connection was forcibly closed by the remote host.
|| 2023-02-14 17:17:51.131138 W | transport: <IP_ADDRESS>:3002 became unreachable, affected 1 nodes
|| 2023-02-14 17:17:51.132179 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 rejected vote n00003 index1773 term14,logterm13,grantfalse,utdtrue
|| 2023-02-14 17:17:51.132678 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00003] t14 rejected vote n00001 index1773 term14,logterm13,grantfalse,utdtrue
|| 2023-02-14 17:17:51.132678 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00003] t14 received RequestVoteResp rejection from n00001
|| 2023-02-14 17:17:51.132678 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00003] t14 received 1 votes and 1 rejections, quorum is 2
|| 2023-02-14 17:17:51.132678 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 received RequestVoteResp rejection from n00003
|| 2023-02-14 17:17:51.132678 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 received 1 votes and 1 rejections, quorum is 2
|| 2023-02-14 17:17:51.301186 I | [WARN] node 1 shard 0 tick err: timeout
|| 2023-02-14 17:17:51.301186 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.301186 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.316982 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.316982 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.347969 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.347969 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.379060 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.379060 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.410740 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.410740 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.442249 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.442249 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.473248 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.473248 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.504398 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.504398 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.534938 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.535329 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.566274 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.566274 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.597141 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.597141 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.627855 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.627855 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.658238 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.658238 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.689227 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.689227 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.720035 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.720035 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.736070 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.736070 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.781256 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.781256 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.796270 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.796270 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.827174 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.827174 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.858269 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.858269 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.889278 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.889278 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.920060 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.920060 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.951175 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.951175 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:51.982435 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:51.982435 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:52.013788 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:52.013788 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:52.044462 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:52.044462 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:52.075132 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:52.075132 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:52.106424 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:52.106424 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:52.136909 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:52.136909 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:52.168862 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:52.169017 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:52.200652 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:52.200652 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:52.231472 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:52.231515 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:52.247104 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:52.247104 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| phase4
|| 2023-02-14 17:17:52.262667 I | dragonboat: go version: go1.19, windows/amd64
|| 2023-02-14 17:17:52.262667 I | dragonboat: dragonboat version: 3.3.5 (Rel)
|| 2023-02-14 17:17:52.262667 W | dragonboat: unsupported OS/ARCH windows/amd64, don't use for production
|| 2023-02-14 17:17:52.262667 I | config: using default EngineConfig
|| 2023-02-14 17:17:52.262667 I | config: using default LogDBConfig
|| 2023-02-14 17:17:52.262667 I | dragonboat: DeploymentID set to 1
|| 2023-02-14 17:17:52.274229 I | dragonboat: LogDB info received, shard 0, busy false
|| 2023-02-14 17:17:52.276866 I | dragonboat: LogDB info received, shard 1, busy false
|| 2023-02-14 17:17:52.276866 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:52.277376 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:52.279739 I | dragonboat: LogDB info received, shard 2, busy false
|| 2023-02-14 17:17:52.282500 I | dragonboat: LogDB info received, shard 3, busy false
|| 2023-02-14 17:17:52.285111 I | dragonboat: LogDB info received, shard 4, busy false
|| 2023-02-14 17:17:52.288676 I | dragonboat: LogDB info received, shard 5, busy false
|| 2023-02-14 17:17:52.291677 I | dragonboat: LogDB info received, shard 6, busy false
|| 2023-02-14 17:17:52.295431 I | dragonboat: LogDB info received, shard 7, busy false
|| 2023-02-14 17:17:52.299127 I | dragonboat: LogDB info received, shard 8, busy false
|| 2023-02-14 17:17:52.302487 I | dragonboat: LogDB info received, shard 9, busy false
|| 2023-02-14 17:17:52.306194 I | dragonboat: LogDB info received, shard 10, busy false
|| 2023-02-14 17:17:52.306194 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:52.306194 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:52.309415 I | dragonboat: LogDB info received, shard 11, busy false
|| 2023-02-14 17:17:52.313151 I | dragonboat: LogDB info received, shard 12, busy false
|| 2023-02-14 17:17:52.316489 I | dragonboat: LogDB info received, shard 13, busy false
|| 2023-02-14 17:17:52.319708 I | dragonboat: LogDB info received, shard 14, busy false
|| 2023-02-14 17:17:52.322982 I | logdb: using plain logdb
|| 2023-02-14 17:17:52.322982 I | dragonboat: LogDB info received, shard 15, busy false
|| 2023-02-14 17:17:52.323501 I | dragonboat: logdb memory limit: 8192 MBytes
|| 2023-02-14 17:17:52.323501 I | dragonboat: NodeHost ID: nhid-2
|| 2023-02-14 17:17:52.323552 I | dragonboat: using regular node registry
|| 2023-02-14 17:17:52.323552 I | dragonboat: filesystem error injection mode enabled: false
|| 2023-02-14 17:17:52.323752 I | transport: transport type: go-tcp-transport
|| 2023-02-14 17:17:52.323752 I | dragonboat: transport type: go-tcp-transport
|| 2023-02-14 17:17:52.323752 I | dragonboat: logdb type: sharded-pebble
|| 2023-02-14 17:17:52.323752 I | dragonboat: nodehost address: <IP_ADDRESS>:3002
|| 2023-02-14 17:17:52.324280 I | dragonboat: [00000:00002] replaying raft logs
|| 2023-02-14 17:17:52.324794 I | dragonboat: [00000:00002] has logdb entries size 1773 commit 1773 term 13
|| 2023-02-14 17:17:52.324794 I | raft: [00000:00002] created, initial: true, new: false
|| 2023-02-14 17:17:52.324794 I | raft: [00000:00002] raft log rate limit enabled: false, 0
|| 2023-02-14 17:17:52.324794 I | raft: [f:1,l:1773,t:13,c:1773,a:0] [00000:00002] t13 became follower
|| 2023-02-14 17:17:52.324794 I | rsm: [00000:00002] no snapshot available during launch
|| 2023-02-14 17:17:52.324794 I | dragonboat: [00000:00002] initialized using <00000:00002:0>
|| 2023-02-14 17:17:52.324794 I | dragonboat: [00000:00002] initial index set to 0
|| 2023-02-14 17:17:52.339541 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 dropped proposal, no leader
|| 2023-02-14 17:17:52.339541 I | [WARN] node 1 shard 0 tick err: request dropped as the cluster is not ready
|| 2023-02-14 17:17:52.340739 W | raft: [f:1,l:1773,t:13,c:1773,a:0] [00000:00002] t13 dropped proposal, no leader
|| 2023-02-14 17:17:52.340739 W | raft: [f:1,l:1773,t:13,c:1773,a:0] [00000:00002] t13 received RequestVote with higher term (14) from n00001
|| 2023-02-14 17:17:52.340739 W | raft: [f:1,l:1773,t:13,c:1773,a:0] [00000:00002] t13 become followerKE after receiving higher term from n00001
|| 2023-02-14 17:17:52.340739 I | raft: [f:1,l:1773,t:13,c:1773,a:0] [00000:00002] t14 became follower
|| 2023-02-14 17:17:52.340739 W | raft: [f:1,l:1773,t:13,c:1773,a:0] [00000:00002] t14 cast vote from n00001 index 1773 term 14, log term: 13
|| 2023-02-14 17:17:52.341333 I | rsm: [00000:00002] applied ADD ccid 0 (1), n00001 (<IP_ADDRESS>:3001)
|| 2023-02-14 17:17:52.341333 I | rsm: [00000:00002] applied ADD ccid 0 (2), n00002 (<IP_ADDRESS>:3002)
|| 2023-02-14 17:17:52.341333 I | rsm: [00000:00002] applied ADD ccid 0 (3), n00003 (<IP_ADDRESS>:3003)
|| 2023-02-14 17:17:52.344656 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 received RequestVoteResp from n00002
|| 2023-02-14 17:17:52.344656 W | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00001] t14 received 2 votes and 1 rejections, quorum is 2
|| 2023-02-14 17:17:52.344656 I | raft: [f:1,l:1774,t:14,c:1773,a:1773] [00000:00001] t14 became leader
|| 2023-02-14 17:17:52.344760 I | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00003] t14 became follower
|| phase5
|| phase6
|| 2023-02-14 17:17:53.358028 E | transport: send batch failed, target <IP_ADDRESS>:3001 (write tcp <IP_ADDRESS>:52330-><IP_ADDRESS>:3001: wsasend: An existing connection was forcibly closed by the remote host.), 1
|| 2023-02-14 17:17:53.358028 W | transport: breaker <IP_ADDRESS>:3003 to <IP_ADDRESS>:3001 failed, connect and process failed: write tcp <IP_ADDRESS>:52330-><IP_ADDRESS>:3001: wsasend: An existing connection was forcibly closed by the remote host.
|| 2023-02-14 17:17:53.358028 W | transport: <IP_ADDRESS>:3001 became unreachable, affected 1 nodes
|| 2023-02-14 17:17:54.368471 I | [WARN] node 3 shard 0 tick err: timeout
|| 2023-02-14 17:17:54.429976 W | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00002] t15 became candidate
|| 2023-02-14 17:17:54.429976 W | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00002] t15 received RequestVoteResp from n00002
|| 2023-02-14 17:17:54.429976 W | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00002] t15 sent RequestVote to n00003
|| 2023-02-14 17:17:54.429976 W | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00002] t15 sent RequestVote to n00001
|| 2023-02-14 17:17:54.436908 E | transport: send batch failed, target <IP_ADDRESS>:3001 (write tcp <IP_ADDRESS>:52332-><IP_ADDRESS>:3001: wsasend: An existing connection was forcibly closed by the remote host.), 1
|| 2023-02-14 17:17:54.436908 W | transport: breaker <IP_ADDRESS>:3002 to <IP_ADDRESS>:3001 failed, connect and process failed: write tcp <IP_ADDRESS>:52332-><IP_ADDRESS>:3001: wsasend: An existing connection was forcibly closed by the remote host.
|| 2023-02-14 17:17:54.436908 W | transport: <IP_ADDRESS>:3001 became unreachable, affected 1 nodes
|| 2023-02-14 17:17:54.438487 W | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00003] t14 received RequestVote with higher term (15) from n00002
|| 2023-02-14 17:17:54.438487 W | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00003] t14 become followerKE after receiving higher term from n00002
|| 2023-02-14 17:17:54.438487 I | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00003] t15 became follower
|| 2023-02-14 17:17:54.438487 W | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00003] t15 cast vote from n00002 index 1807 term 15, log term: 14
|| 2023-02-14 17:17:54.441201 W | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00002] t15 received RequestVoteResp from n00003
|| 2023-02-14 17:17:54.441201 W | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00002] t15 received 2 votes and 0 rejections, quorum is 2
|| 2023-02-14 17:17:54.441201 I | raft: [f:1,l:1808,t:15,c:1807,a:1807] [00000:00002] t15 became leader
|| 2023-02-14 17:17:55.374277 I | [WARN] node 3 shard 0 tick err: timeout
|| phase7
|| 2023-02-14 17:17:56.350925 I | dragonboat: go version: go1.19, windows/amd64
|| 2023-02-14 17:17:56.350925 I | dragonboat: dragonboat version: 3.3.5 (Rel)
|| 2023-02-14 17:17:56.350925 W | dragonboat: unsupported OS/ARCH windows/amd64, don't use for production
|| 2023-02-14 17:17:56.351246 I | config: using default EngineConfig
|| 2023-02-14 17:17:56.351246 I | config: using default LogDBConfig
|| 2023-02-14 17:17:56.351352 I | dragonboat: DeploymentID set to 1
|| 2023-02-14 17:17:56.362717 I | dragonboat: LogDB info received, shard 0, busy false
|| 2023-02-14 17:17:56.367712 I | dragonboat: LogDB info received, shard 0, busy false
|| 2023-02-14 17:17:56.369271 I | dragonboat: LogDB info received, shard 1, busy false
|| 2023-02-14 17:17:56.372607 I | dragonboat: LogDB info received, shard 2, busy false
|| 2023-02-14 17:17:56.376247 I | dragonboat: LogDB info received, shard 3, busy false
|| 2023-02-14 17:17:56.379276 I | dragonboat: LogDB info received, shard 4, busy false
|| 2023-02-14 17:17:56.382486 I | dragonboat: LogDB info received, shard 5, busy false
|| 2023-02-14 17:17:56.385099 I | dragonboat: LogDB info received, shard 6, busy false
|| 2023-02-14 17:17:56.388554 I | dragonboat: LogDB info received, shard 7, busy false
|| 2023-02-14 17:17:56.391690 I | dragonboat: LogDB info received, shard 8, busy false
|| 2023-02-14 17:17:56.394827 I | dragonboat: LogDB info received, shard 9, busy false
|| 2023-02-14 17:17:56.398190 I | dragonboat: LogDB info received, shard 10, busy false
|| 2023-02-14 17:17:56.400910 I | dragonboat: LogDB info received, shard 11, busy false
|| 2023-02-14 17:17:56.403893 I | dragonboat: LogDB info received, shard 12, busy false
|| 2023-02-14 17:17:56.406893 I | dragonboat: LogDB info received, shard 13, busy false
|| 2023-02-14 17:17:56.409894 I | dragonboat: LogDB info received, shard 14, busy false
|| 2023-02-14 17:17:56.412893 I | logdb: using plain logdb
|| 2023-02-14 17:17:56.412893 I | dragonboat: LogDB info received, shard 15, busy false
|| 2023-02-14 17:17:56.412893 I | dragonboat: logdb memory limit: 8192 MBytes
|| 2023-02-14 17:17:56.412893 I | dragonboat: NodeHost ID: nhid-1
|| 2023-02-14 17:17:56.412893 I | dragonboat: using regular node registry
|| 2023-02-14 17:17:56.412893 I | dragonboat: filesystem error injection mode enabled: false
|| 2023-02-14 17:17:56.414042 E | transport: Nodehost <IP_ADDRESS>:3003 failed to get a connection to <IP_ADDRESS>:3001, dial tcp <IP_ADDRESS>:3001: connectex: No connection could be made because the target machine actively refused it.
|| 2023-02-14 17:17:56.414042 W | transport: breaker <IP_ADDRESS>:3003 to <IP_ADDRESS>:3001 failed, connect and process failed: dial tcp <IP_ADDRESS>:3001: connectex: No connection could be made because the target machine actively refused it.
|| 2023-02-14 17:17:56.414120 W | transport: <IP_ADDRESS>:3001 became unreachable, affected 0 nodes
|| 2023-02-14 17:17:56.414120 I | transport: transport type: go-tcp-transport
|| 2023-02-14 17:17:56.414120 I | dragonboat: transport type: go-tcp-transport
|| 2023-02-14 17:17:56.414120 I | dragonboat: logdb type: sharded-pebble
|| 2023-02-14 17:17:56.414120 I | dragonboat: nodehost address: <IP_ADDRESS>:3001
|| 2023-02-14 17:17:56.414120 I | dragonboat: [00000:00001] replaying raft logs
|| 2023-02-14 17:17:56.415156 I | dragonboat: [00000:00001] has logdb entries size 1807 commit 1807 term 14
|| 2023-02-14 17:17:56.415156 I | raft: [00000:00001] created, initial: true, new: false
|| 2023-02-14 17:17:56.415156 I | raft: [00000:00001] raft log rate limit enabled: false, 0
|| 2023-02-14 17:17:56.415156 I | raft: [f:1,l:1807,t:14,c:1807,a:0] [00000:00001] t14 became follower
|| 2023-02-14 17:17:56.415156 I | rsm: [00000:00001] no snapshot available during launch
|| 2023-02-14 17:17:56.415156 I | dragonboat: [00000:00001] initialized using <00000:00001:0>
|| 2023-02-14 17:17:56.415156 I | dragonboat: [00000:00001] initial index set to 0
|| 2023-02-14 17:17:56.523627 I | rsm: [00000:00001] applied ADD ccid 0 (1), n00001 (<IP_ADDRESS>:3001)
|| 2023-02-14 17:17:56.523627 I | rsm: [00000:00001] applied ADD ccid 0 (2), n00002 (<IP_ADDRESS>:3002)
|| 2023-02-14 17:17:56.523627 I | rsm: [00000:00001] applied ADD ccid 0 (3), n00003 (<IP_ADDRESS>:3003)
|| 2023-02-14 17:17:56.881657 W | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00001] t14 received Heartbeat with higher term (15) from n00002
|| 2023-02-14 17:17:56.881657 W | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00001] t14 become follower after receiving higher term from n00002
|| 2023-02-14 17:17:56.882685 I | raft: [f:1,l:1807,t:14,c:1807,a:1807] [00000:00001] t15 became follower
|| phase8
|| [Finished in 22 seconds with code 1]

You can see multiple tick timeout and no leader election after phase1, which is not expected.

Please provide more descriptions on what your test program is trying to do, what is expected and what actually happened.

@hulucc Your logs indicate that the leader change request was dropped because the cluster was not yet stable.
|| 2023-02-14 16:51:15.378999 W | raft: [f:1,l:1330,t:13,c:1330,a:0] [00000:00001] t13 dropped LeaderTransfer, no leader

... so when you Stop nh3 in phase1 you think you're stopping a follower but you're actually stopping the leader.
|| 2023-02-14 16:51:16.595686 I | raft: [f:1,l:1331,t:14,c:1330,a:1330] [00000:00003] t14 became leader
|| phase1

[00000:00003] in the logs means clusterID 0, nodeID 3. This can help debug such issues.
Were you expecting the nodehost to automatically request a leader change for all active clusters of which the host is a leader prior to returning from Stop?

@hulucc Your logs indicate that the first leader change request was dropped because the cluster was not yet stable.
|| 2023-02-14 16:51:15.378999 W | raft: [f:1,l:1330,t:13,c:1330,a:0] [00000:00001] t13 dropped LeaderTransfer, no leader

... so when your code Stops nh3 in phase1 your code appears to intend to stop a follower but it's actually stopping the leader.
|| 2023-02-14 16:51:16.595686 I | raft: [f:1,l:1331,t:14,c:1330,a:1330] [00000:00003] t14 became leader
|| phase1

[00000:00003] t14 became leader in the logs means clusterID 0, nodeID 3. This can help debug such issues.
Similarly at the beginning of Phase 6 your code appears to expect node 3 to be the leader but the leader is actually node 1.
|| 2023-02-14 17:17:52.344656 I | raft: [f:1,l:1774,t:14,c:1773,a:1773] [00000:00001] t14 became leader
|| 2023-02-14 17:17:52.344760 I | raft: [f:1,l:1773,t:13,c:1773,a:1773] [00000:00003] t14 became follower
|| phase5
|| phase6

RequestLeaderTransfer makes a request to transfer the leadership of the specified Raft cluster to the target node identified by targetNodeID. It returns an error if the request fails to be started. There is no guarantee that such request can be fulfilled, i.e. the leadership transfer can still fail after a successful return of the RequestLeaderTransfer method.

Were you expecting RequestLeaderTransfer to be synchronous? Or were you expecting the nodehost to automatically request a synchronous leader change for all active clusters of which the host is a leader prior to returning from Stop? Or both?

Yes, actually I was not realized that RequestLeaderTransfer is async. After make RequestLeaderTransfer synchronous, there is no problem with graceful rolling update. Here is fixed code for any one may need it.
package main

import (
	"context"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"time"
	"errors"

	"github.com/lni/dragonboat/v3"
	"github.com/lni/dragonboat/v3/config"
	sm "github.com/lni/dragonboat/v3/statemachine"
	"github.com/lni/goutils/syncutil"
)

type exampleStateMachine struct {
	ClusterID uint64
	NodeID    uint64
	Tick      uint64
}

func NewExampleStateMachine(clusterID uint64, nodeID uint64) sm.IStateMachine {
	return &exampleStateMachine{ClusterID: clusterID, NodeID: nodeID}
}
func (it *exampleStateMachine) Update(req []byte) (sm.Result, error) {
	it.Tick++
	return sm.Result{}, nil
}
func (it *exampleStateMachine) Lookup(req interface{}) (interface{}, error) {
	return it.Tick, nil
}
func (it *exampleStateMachine) SaveSnapshot(w io.Writer, _ sm.ISnapshotFileCollection, _ <-chan struct{}) error {
	data := make([]byte, 8)
	binary.LittleEndian.PutUint64(data, it.Tick)
	w.Write(data)
	return nil
}
func (it *exampleStateMachine) RecoverFromSnapshot(r io.Reader, _ []sm.SnapshotFile, _ <-chan struct{}) error {
	data, _ := io.ReadAll(r)
	v := binary.LittleEndian.Uint64(data)
	it.Tick = v
	return nil
}
func (it *exampleStateMachine) Close() error {
	return nil
}

func newNodeHost(nhid uint64, addr string) *dragonboat.NodeHost {
	conf := config.NodeHostConfig{
		NodeHostDir:         fmt.Sprintf("single_nodehost_test_dir_safe_to_delete/%d", nhid),
		AddressByNodeHostID: false,
		RTTMillisecond:      100,
		RaftAddress:         addr,
		Expert: config.ExpertConfig{
			TestNodeHostID: nhid,
		},
	}
	nh, err := dragonboat.NewNodeHost(conf)
	if err != nil {
		panic(err)
	}
	return nh
}

func startShard(nh *dragonboat.NodeHost, clusterID uint64, nodeID uint64, initials map[uint64]string, fn interface{}) {
	conf := config.Config{
		NodeID:              nodeID,
		ClusterID:           clusterID,
		CheckQuorum:         true,
		ElectionRTT:         10,
		HeartbeatRTT:        1,
		SnapshotEntries:     0,
		CompactionOverhead:  0,
		OrderedConfigChange: false,
	}
	var err error
	switch f := fn.(type) {
	case sm.CreateStateMachineFunc:
		if len(initials) > 0 {
			err = nh.StartCluster(initials, false, f, conf)
		} else {
			err = nh.StartCluster(nil, false, f, conf)
		}
	case sm.CreateConcurrentStateMachineFunc:
		if len(initials) > 0 {
			err = nh.StartConcurrentCluster(initials, false, f, conf)
		} else {
			err = nh.StartConcurrentCluster(nil, false, f, conf)
		}
	default:
		panic(fmt.Errorf("unknown fn type: %T", f))
	}
	if err != nil {
		panic(err)
	}
}

func tick(nh *dragonboat.NodeHost, clusterID uint64) *syncutil.Stopper {
	stopper := syncutil.NewStopper()
	stopper.RunWorker(func() {
		t := time.NewTicker(time.Millisecond * 30)
		defer t.Stop()
		nop := nh.GetNoOPSession(clusterID)
		ctx := context.Background()
		for {
			select {
			case <-t.C:
				ctx1, cancel := context.WithTimeout(ctx, time.Second)
				_, err := nh.SyncPropose(ctx1, nop, []byte("tick"))
				cancel()
				if err != nil {
					// log.Fatalln("[FATAL]", fmt.Errorf("node %d shahrd %d tick err: %w", nh.NodeHostConfig().Expert.TestNodeHostID, clusterID, err))
					log.Println("[WARN]", fmt.Errorf("node %d shard %d tick err: %w", nh.NodeHostConfig().Expert.TestNodeHostID, clusterID, err))
				}
			case <-stopper.ShouldStop():
				return
			}
		}
	})
	return stopper
}

func waitReady(nh *dragonboat.NodeHost, clusterID uint64) {
	for {
		ctx, cancel := context.WithTimeout(context.TODO(), time.Second)
		defer cancel()
		if _, err := nh.SyncRead(ctx, clusterID, nil); err != nil {
			if errors.Is(err, dragonboat.ErrClusterNotReady) {
				time.Sleep(time.Second)
				continue
			}
		}
		return
	}
}

func waitLeader(nh *dragonboat.NodeHost, clusterID uint64) {
	for {
		leaderNodeID, valid, err := nh.GetLeaderID(clusterID)
		if !valid || err != nil {
			time.Sleep(time.Second)
			continue
		}
		if nh.HasNodeInfo(clusterID, leaderNodeID) {
			return
		}
	}
}

func main() {
	clusterID := uint64(0)
	fn := sm.CreateStateMachineFunc(NewExampleStateMachine)
	initials := map[uint64]string{
		1: "<IP_ADDRESS>:3001",
		2: "<IP_ADDRESS>:3002",
		3: "<IP_ADDRESS>:3003",
	}
	nh1 := newNodeHost(1, initials[1])
	nh2 := newNodeHost(2, initials[2])
	nh3 := newNodeHost(3, initials[3])
	startShard(nh1, clusterID, 1, initials, fn)
	startShard(nh2, clusterID, 2, initials, fn)
	startShard(nh3, clusterID, 3, initials, fn)
	waitReady(nh1, clusterID)
	if err := nh1.RequestLeaderTransfer(clusterID, 1); err != nil {
		log.Fatalln("[FATAL]", fmt.Errorf("RequestLeaderTransfer err: %w", err))
	}
	waitLeader(nh1, clusterID)
	stopper := tick(nh1, clusterID)
	defer stopper.Stop()
	println("phase1")

	nh3.Stop()
	time.Sleep(time.Second*3)
	println("phase2")

	nh3 = newNodeHost(3, initials[3])
	startShard(nh3, clusterID, 3, nil, fn)
	waitReady(nh3, clusterID)
	println("phase3")

	nh2.Stop()
	time.Sleep(time.Second*3)
	println("phase4")

	nh2 = newNodeHost(2, initials[2])
	startShard(nh2, clusterID, 2, nil, fn)
	waitReady(nh2, clusterID)
	println("phase5")

	stopper.Stop()
	stopper = tick(nh3, clusterID)

	if err := nh3.RequestLeaderTransfer(clusterID, 3); err != nil {
		log.Fatalln("[FATAL]", fmt.Errorf("RequestLeaderTransfer err: %w", err))
	}
	waitLeader(nh3, clusterID)
	println("phase6")

	nh1.Stop()
	time.Sleep(time.Second*3)
	println("phase7")

	nh1 = newNodeHost(1, initials[1])
	startShard(nh1, clusterID, 1, nil, fn)
	waitReady(nh1, clusterID)
	println("phase8")

	time.Sleep(time.Second*5)
}


Segmentation fault, when using with numpy and itertools

I tried that... I don't know, if it's a nice idea, but with normal dicts, that works fine.
import addict
b = addict.Dict({'t':{'o':2}})
import numpy
numpy.array([b,b,b,b])

I get: Process finished with exit code 139 (interrupted by signal 11: SIGSEGV)

I've narrowed your example down to this Numpy bug: https://github.com/numpy/numpy/pull/13273
But what do you mean by "and itertools"?

Ah, pretty well.
We could edit the thread, the occurence of itertools is a rudiment, from a previous version of this issue. Sorry, my fault.

Hey guys, did https://github.com/numpy/numpy/pull/13273 fix this for you?

Closing due to staleness.


⚠ TorrentLeech has degraded performance

In 39433eb, TorrentLeech ($TL) experienced degraded performance:

HTTP code: 200
Response time: 3348 ms

Resolved: TorrentLeech performance has improved in ce9de45 after 6 minutes.


Fix the problem of using '+' and '+=' operators to concatenate strings a in a loop.

The method is building a String using concatenation in a loop.
In each iteration, the String is converted to a StringBuilder, appended to, and converted back to a String.
This can lead to a cost quadratic in the number of iterations, as the growing string is recopied in each iteration.
Better performance can be obtained by using a StringBuilder explicitly.
http://findbugs.sourceforge.net/bugDescriptions.html#SBSC_USE_STRINGBUFFER_CONCATENATION

Thanks @BruceKuiLiu the PR has been merged. Can you close this?

@oscerd  Thanks.


F#2266 mdm lineage redesign

Description
MDM Lineage screen redesign. Displaying the column's relationship in the Lineage screen.
MDM 리니지 화면 재기획. 리니지 화면 내 컬럼 연결 관계 표출
In existing lineage planning, we will separate the relationship UIs between the tables and the columns.
기존 리니지 기획에서, 테이블간의 관계와 컬럼간의 관계 화면 영역을 분리할 예정
Related Issue :
#2266
How Has This Been Tested?
[Management]-[Explore Data]-[Metadata] and [Management]-[Explore Data]-[Lineage]
is new features.
Test them according to the planning document.
Need additional checks?
Types of changes

[ ] Bug fix (non-breaking change which fixes an issue)
[x] New feature (non-breaking change which adds functionality)
[ ] Breaking change (fix or feature that would cause existing functionality to change)

Checklist:

[ ] I have read the CONTRIBUTING document.
[ ] My code follows the code style of this project.
[ ] My change requires a change to the documentation.
[ ] I have added tests to cover my changes.

Additional Context

run build
deploy to 1

run build
deploy to 1

run build
deploy to 1

@ksparknot @AnnieHwang Node count and alignment have been implemented.


Fix FFI errors, and add script for running TPC-H

Closes https://github.com/apache/datafusion-ray/issues/10
Changes in this PR:

Update to datafusion 42 / arrow-rs 53.1.0
Update datafusion.proto
Add benchmark script
Remove old performance results because they are no longer valid

I can now run TPC-H and see tasks distributed across both nodes in my Ray cluster.

I will go ahead and merge this since the current main branch is broken within this FFI fix


weblogin parameter

It would be helpful if the -useweblogin parameter discussion mentioned that this needed when Multi Factor authentication is required.

Document Details
⚠ Do not edit this section. It is required for docs.microsoft.com ➟ GitHub issue linking.

ID: 7f910a5c-5883-6394-19a3-4cf1670c431f
Version Independent ID: 4451fe4c-df86-34a8-0db0-46a47e70b5a5
Content: Connect-PnPOnline (sharepoint-pnp)
Content Source: sharepoint/sharepoint-ps/sharepoint-pnp/Connect-PnPOnline.md
Service: unspecified
GitHub Login: @techwriter40
Microsoft Alias: kirks

@DeanGross Thanks for your comment. We are looking into this matter and will let you know what we can do about it.

Will be fixed in June 2018 content refresh for PnP PowerShell cmdlets. Thanks, @DeanGross for reporting, appreciated.


Take screenshots of FB attachments for VR frames

Screenshot the renderpass's framebuffer color attachments that were used
in a command buffer that marks the end of a VR frame.

CI gfxreconstruct build queued with queue ID 9356.

CI gfxreconstruct build # 3003 running.

CI gfxreconstruct build # 3003 passed.


fix(iter): A "at" time earlier same day would panic

The change added in #15 would cause a panic with the following code:
#[test]
fn earlier_same_day_start() {
    let start = datetime!(2024-12-24 8:00);
    let mut iter = start.every(Interval::from_days(1)).at(time!(07:00)).into_iter();

    let dt = iter.next().unwrap();
    let duration = dt.duration_since(&start);
    assert_eq!(duration, DAY - HOUR);
}

This change does resolve that, but I am not sure how to check if it is working correctly with different timezones, or if it needs to do that.

Moving it from build to next should be fine, even for the timezone aspect. I think having it be in build was mostly a leftover.


Move the webgpu CMPLT hack to one place

The high-level problem is that we can't define globals with boolean buffers in webgpu. These hacks were first put in both lazy and mlops in #2888 and #2887 to cast the CMPLT output to float. This made it really hard to manage all the issues it caused with the UOps spec PR #2800.
these changes:

ensure CMPLT is always a bool, we don't need the mlops bool cast
remove the workaround in lazy asserts for webgpu, CMPLT is still a bool in wgsl.

yeah it's fine. this ready to merge? i will do a cleanup here / webgpu when i add CMPEQ


Explain page content in notification banner guide

Fixes #1411.
This PR updates our notification banner guidance to:

give examples of what we mean by 'page content'
specify what we mean when we say the notification banner should be the same width as the page content
include a link to our layout guidance

We've made these changes because a user told us the previous wording was ambiguous.

@vanitabarrett Thanks for reviewing!


Find another Energymeter

Hi,
first the SE-RGMTR-1D-240C-A was found and i got the sensor values. A day later no values were available and another sensor pop up
SE-MTR-3Y-400V-A (and this one is really installed)

The integration can only report what the inverter tells it. Maybe something rebooted or got a firmware update. I would probably not worry about this.

Had the same issue... only a remove of the integration and reconfigure helped to get rid of the 2nd meter.

@WillCodeForCats do you have any hints to stop getting another meter in the config? This lets my Integration to not work anymore :-(

Can you post all of your device attributes?
Go to: Developer Tools -> States
Search for: solaredge*device

Had the same issue yesterday, all of a sudden an extra meter showed up and I had to delete and reconfigure the integration.
These are my devices:

And happened again today. These are the devices when it created the second meter

What is the real model label on the meter?

I need to check. I am not at home currently, so will not be able to get that information until next week.

Is there only one meter on the RS485 bus ore more than one meter?

Only one meter. Nothing else connected to the modbus.

Ok, I was able to find which meter is connected. It is the SE-MTR-3Y-400V-A
So my problem is exactly the same as from the OP. Initially it detects the meter with a wrong identification and after some time (unknown how much time; it can be good for weeks or the issue shows up every day like this weekend) it "finds" the correct identification and we end up with the 2 meters in the integration.

The integration only reports the data the inverter tells it. The question is why is the inverter reporting one model number then changing it later.
You could try looking at the inverter configuration and make sure it only has one meter in configuration and it's the correct one. The meter setting screens for a SetApp inverter are here:
https://github.com/WillCodeForCats/solaredge-modbus-multi/wiki/Configuration#meter-settings-on-an-inverter

Also, it is possible for a hidden meter to be inside the inverter plus an external meter for import/export or consumption monitoring. The internal meter is usually set up as revenue grade production for higher accuracy than the inverter's self reporting.

I used Setapp in the past to check all the configuration options and it only has one meter configured (the right one). So maybe it is something with this "hidden meter"....the question is...what do I do with this now? Do I wait till the second meter shows up again and use that in my configuration and ignore the "unavailable" one...?
I used to use the original integration and I never had this issue at all...

If there is a revenue grade meter inside the inverter (that's what SE-RGMTR means: "revenue grade" meter), you would probably configure it as a "production" meter in setapp. You would need to take the covers off the inverter and anything else related to look inside to see if it came with a built in meter.
The other integration just polls registers and ignores device information. It doesn't try to detect anything either. Since the model randomly changing will conflict with the auto-detection feature, I would have to consider what options I could come up with to disable auto-detect and ignore device information.

I will check what is inside when I get back home, but as I said... will take another week at least. I will see what happens if it detects the second meter again and if I can just use that in my configuration and ignore the other one. I will update here if I find out more.

One other interesting fact....after I deleted and reinstalled the integration it now shows my meter as the correct one right away. I do not have that  SE-RGMTR now. Maybe....just maybe it detected the right meter as the first one now. I'll keep an eye on it.

It just seems strange to me that the inverter would randomly change what device information it reports. I would think it has to get that data from somewhere.

Yes, I also do not really understand it and I would like to get to the bottom of it. Since I am not the only one running into this issue, there must be something going on.

Next Time the issue will occur, i'll try to add logs.
Also in my case - only 1 Meter ist installed, but sometimes the Addon gets a 2nd Meter.

Hello,
I have same issue as describe, this is happening to me not since the last update but since the previous one. At I didnt update the inverters manually, if they were updated the update was OTA but I dont really know because I dont have tracking of the firmware versions over time.
This is issue, as mentioned before, happens randomly and it solves by just but deleting the integration and adding it again. I have two inverters and one battery. Within the detected devices, the integration doesnt show the model of my meter, only shows WattNode in the Manufacturer column

in my case it was after a reboot of an Update

after delete and reconfigure it is ok:

Device detection is only done once when the integration is loaded, either at startup or manual reload.
So far everyone with the problem has an SEnK-RWS model inverter (I have meters but a different model inverter).
Inverter firmware updates would be shown with a change in the solaredge_i1_version sensor.

In my case i have the latest FW installed.

I've thought about this for a bit and I think the best option to keep auto detection and still work if the meter identification reported by the inverter changes for unknown reasons is to not rely on the meter serial number and model number. These are currently used for the entity registry Unique ID in Home Assistant. When the Unique ID changes because the model number changed, the issue of seeing a "new" meter and new entities will happen.
Since this is not being reported as a problem with the inverter's device information (otherwise you would see the same thing with a new/different inverter being detected) I think it's still safe to rely on the inverter's device information as a base for the unique id.

Just as another point of data the meter model change has never happened to me, but I opened my meter box and mine is labeled as a SE-RGMTR-3Y-277V-A but it's reported as SE-RGMTR-3Y-208V-A in Home Assistant.

Could be a Solution to detect the meter Inlay by adding the Integration not by every bootup? Or what do you think could be to fix this issue?

The fix will be to rely on different values that are stable for the unique id.

@FranciscoBigWings What model inverter do you have (from sensor.solaredge_i1_device)?

i1 is SE5000H-RW000BNV4
i2 is SE5000H-RWS00BNO4
Francisco F. Mayoralas

On 17 Aug 2022, at 16:33, Seth @.***> wrote:

@FranciscoBigWings What model inverter do you have (from sensor.solaredge_i1_device)?
—
Reply to this email directly, view it on GitHub, or unsubscribe.
You are receiving this because you were mentioned.

PR #83 is my solution to the issue of duplicate meters.

PR #83 is my solution to the issue of duplicate meters.

Great. Thank you a lot for your dedication. I will update within the next days and test it in time. If it happens again I will update you

PR #83 is based on PR #81 so it also contains a bunch of updates to modbus error handling.

I have Release v2.2.0-pre.1 for this issue. Please read the release notes since this is a breaking change.

Thanks for your effort. Just installed it and everything seems to work fine so far. I will update if I see any problems.


rattler_conda_types::Version::to_string() does not preserve case

When uploading packages with rattler-build, we use different fields from parsed IndexJson to upload packages to anaconda.org. If we attempt upload a package with version 1.2.3_Beta1, it will fail because the original version case is lost after parsing and version.to_string() will return 1.2.3_beta1. Subsequently a release 1.2.3_beta1 is created but anaconda.org rejects the package upload as the versions from the IndexJson and the release are not the same. While it is true that 1.2.3_Beta1 == 1.2.3_beta1 when comparing versions, rattler should return the version with the original version when calling .to_string().
The following unit-test should pass:
#[test]
fn preserve_case() {
    let v = "1.2.3-Beta1";
    let version = Version::from_str(v).unwrap();
    assert_eq!(version.to_string(), v);
}

We have VersionWithSource for that use case. Does that help?

Yeah, missed it. Works perfectly for this use-case


Bug: Squeak hangs when I cursor is moved while holding Ctrl

I can reproduce this issue on Squeak5.2 and Squeak5.3alpha: Open Squeak, install the latest master of Window Acrobatics and move your cursor while holding Ctrl pressed. On my machine (Win 10), this leads to an immediate hang of Squeak.
Currently, I am not able to terminate the hanging process by Alt + ., but using Squeak5.2, @CodeZeilen has been able to locate the problem in the key handling mechanisms of WA.
I am going to provide a pull request that turns the problem off.

The new PR fixes the issue for now. However, the proper solution would be to add a WA-specific object as a capture filter to the current world. Something along the lines of:
Project current world addKeyboardCaptureFilter: (...)