Improve zookeeper client reconnect logic. UI: Flot widget - Fixed incorrect indi…

…vidual tooltip content.

@@ -52,6 +52,8 @@ import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.util.List;
 import java.util.NoSuchElementException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.stream.Collectors;
 
 import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED;
@@ -96,11 +98,13 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
     @Lazy
     private ClusterRoutingService routingService;
 
+    private ExecutorService reconnectExecutorService;
+
     private CuratorFramework client;
     private PathChildrenCache cache;
     private String nodePath;
 
-    private volatile boolean stopped = false;
+    private volatile boolean stopped = true;
 
     @PostConstruct
     public void init() {
@@ -110,9 +114,15 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
         Assert.notNull(zkConnectionTimeout, MiscUtils.missingProperty("zk.connection_timeout_ms"));
         Assert.notNull(zkSessionTimeout, MiscUtils.missingProperty("zk.session_timeout_ms"));
 
+        reconnectExecutorService = Executors.newSingleThreadExecutor();
+
         log.info("Initializing discovery service using ZK connect string: {}", zkUrl);
 
         zkNodesDir = zkDir + "/nodes";
+        initZkClient();
+    }
+
+    private void initZkClient() {
         try {
             client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval));
             client.start();
@@ -120,6 +130,8 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
             cache = new PathChildrenCache(client, zkNodesDir, true);
             cache.getListenable().addListener(this);
             cache.start();
+            stopped = false;
+            log.info("ZK client connected");
         } catch (Exception e) {
             log.error("Failed to connect to ZK: {}", e.getMessage(), e);
             CloseableUtils.closeQuietly(cache);
@@ -128,12 +140,20 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
         }
     }
 
-    @PreDestroy
-    public void destroy() {
+    private void destroyZkClient() {
         stopped = true;
-        unpublishCurrentServer();
+        try {
+            unpublishCurrentServer();
+        } catch (Exception e) {}
         CloseableUtils.closeQuietly(cache);
         CloseableUtils.closeQuietly(client);
+        log.info("ZK client disconnected");
+    }
+
+    @PreDestroy
+    public void destroy() {
+        destroyZkClient();
+        reconnectExecutorService.shutdownNow();
         log.info("Stopped discovery service");
     }
 
@@ -180,20 +200,21 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
         return (client, newState) -> {
             log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState);
             if (newState == ConnectionState.LOST) {
-                reconnect();
+                reconnectExecutorService.submit(this::reconnect);
             }
         };
     }
 
-    private boolean reconnectInProgress = false;
+    private volatile boolean reconnectInProgress = false;
 
     private synchronized void reconnect() {
         if (!reconnectInProgress) {
             reconnectInProgress = true;
             try {
-                client.blockUntilConnected();
+                destroyZkClient();
+                initZkClient();
                 publishCurrentServer();
-            } catch (InterruptedException e) {
+            } catch (Exception e) {
                 log.error("Failed to reconnect to ZK: {}", e.getMessage(), e);
             } finally {
                 reconnectInProgress = false;

@@ -50,7 +50,7 @@
         <commons-validator.version>1.5.0</commons-validator.version>
         <commons-io.version>2.5</commons-io.version>
         <commons-csv.version>1.4</commons-csv.version>
-        <jackson.version>2.9.7</jackson.version>
+        <jackson.version>2.9.8</jackson.version>
         <json-schema-validator.version>2.2.6</json-schema-validator.version>
         <scala.version>2.11</scala.version>
         <akka.version>2.4.2</akka.version>
@@ -59,7 +59,7 @@
         <velocity.version>1.7</velocity.version>
         <velocity-tools.version>2.0</velocity-tools.version>
         <mail.version>1.4.3</mail.version>
-        <curator.version>4.0.1</curator.version>
+        <curator.version>4.2.0</curator.version>
         <protobuf.version>3.6.1</protobuf.version>
         <grpc.version>1.16.1</grpc.version>
         <lombok.version>1.16.18</lombok.version>

@@ -137,9 +137,11 @@ export default class TbFlot {
                 });
                 content += dateDiv.prop('outerHTML');
                 if (tbFlot.ctx.tooltipIndividual) {
-                    var seriesHoverInfo = hoverInfo.seriesHover[seriesIndex];
-                    if (seriesHoverInfo) {
-                        content += seriesInfoDivFromInfo(seriesHoverInfo, seriesIndex);
+                    var found = hoverInfo.seriesHover.filter((seriesHover) => {
+                        return seriesHover.index === seriesIndex;
+                    });
+                    if (found && found.length) {
+                        content += seriesInfoDivFromInfo(found[0], seriesIndex);
                     }
                 } else {
                     var seriesDiv = $('<div></div>');
@@ -161,7 +163,7 @@ export default class TbFlot {
                             if (i == hoverInfo.seriesHover.length) {
                                 break;
                             }
-                            seriesHoverInfo = hoverInfo.seriesHover[i];
+                            var seriesHoverInfo = hoverInfo.seriesHover[i];
                             columnContent += seriesInfoDivFromInfo(seriesHoverInfo, seriesIndex);
                         }
                         columnDiv.html(columnContent);