diff --git a/p2p/server.go b/p2p/server.go
index 5424b54632..16768f920f 100644
--- a/p2p/server.go
+++ b/p2p/server.go
@@ -422,13 +422,13 @@ func (srv *Server) dialLoop() {
 		}
 		// Request a dial slot to prevent CPU exhaustion
 		<-slots
-		defer func() { slots <- struct{}{} }()
 
 		dialing[dest.ID] = true
 		srv.peerWG.Add(1)
 		go func() {
 			srv.dialNode(dest)
 			dialed <- dest
+			slots <- struct{}{}
 		}()
 	}