diff --git a/libnetwork/agent.go b/libnetwork/agent.go
index 3265add3cb..499e2d5221 100644
--- a/libnetwork/agent.go
+++ b/libnetwork/agent.go
@@ -183,17 +183,20 @@ func (c *controller) agentSetup() error {
 				}
 				return false
 			})
-
-			if c.agent != nil {
-				close(c.agentInitDone)
-			}
 		}
 	}
+
 	if remoteAddr != "" {
 		if err := c.agentJoin(remoteAddr); err != nil {
 			logrus.Errorf("Error in agentJoin : %v", err)
+			return nil
 		}
 	}
+
+	if c.agent != nil {
+		close(c.agentInitDone)
+	}
+
 	return nil
 }