简柏林 / thingskit · Commits

Commit f07d7441b2e79655a8ccec116b5683d42e5382c3

Authored by Andrii Shvaika
1 parent b8de64f3

Change of the partition routing strategy

Inline Side-by-side
Showing 3 changed files with 42 additions and 54 deletions
... ... @@ -621,8 +621,7 @@ queue:
621 621 notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
622 622 js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
623 623 partitions:
624   - hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}"
625   - virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}"
  624 + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256
626 625 transport_api:
627 626 requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}"
628 627 responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}"
... ...
application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java renamed from application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java
... ... @@ -26,14 +26,14 @@ import org.springframework.context.ApplicationEventPublisher;
26 26 import org.springframework.test.util.ReflectionTestUtils;
27 27 import org.thingsboard.server.common.data.id.DeviceId;
28 28 import org.thingsboard.server.common.data.id.TenantId;
29   -import org.thingsboard.server.queue.discovery.ConsistentHashPartitionService;
  29 +import org.thingsboard.server.common.msg.queue.ServiceQueue;
  30 +import org.thingsboard.server.queue.discovery.HashPartitionService;
30 31 import org.thingsboard.server.common.msg.queue.ServiceType;
31 32 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
32 33 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
33 34 import org.thingsboard.server.gen.transport.TransportProtos;
34 35 import org.thingsboard.server.queue.discovery.TenantRoutingInfoService;
35 36 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
36   -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
37 37
38 38 import java.util.ArrayList;
39 39 import java.util.Collections;
... ... @@ -48,18 +48,18 @@ import static org.mockito.Mockito.when;
48 48
49 49 @Slf4j
50 50 @RunWith(MockitoJUnitRunner.class)
51   -public class ConsistentHashParitionServiceTest {
  51 +public class HashPartitionServiceTest {
52 52
53 53 public static final int ITERATIONS = 1000000;
54   - private ConsistentHashPartitionService clusterRoutingService;
  54 + public static final int SERVER_COUNT = 3;
  55 + private HashPartitionService clusterRoutingService;
55 56
56 57 private TbServiceInfoProvider discoveryService;
57 58 private TenantRoutingInfoService routingInfoService;
58 59 private ApplicationEventPublisher applicationEventPublisher;
59 60 private TbQueueRuleEngineSettings ruleEngineSettings;
60 61
61   - private String hashFunctionName = "murmur3_128";
62   - private Integer virtualNodesSize = 16;
  62 + private String hashFunctionName = "sha256";
63 63
64 64
65 65 @Before
... ... @@ -68,25 +68,28 @@ public class ConsistentHashParitionServiceTest {
68 68 applicationEventPublisher = mock(ApplicationEventPublisher.class);
69 69 routingInfoService = mock(TenantRoutingInfoService.class);
70 70 ruleEngineSettings = mock(TbQueueRuleEngineSettings.class);
71   - clusterRoutingService = new ConsistentHashPartitionService(discoveryService,
  71 + clusterRoutingService = new HashPartitionService(discoveryService,
72 72 routingInfoService,
73 73 applicationEventPublisher,
74 74 ruleEngineSettings
75 75 );
76 76 when(ruleEngineSettings.getQueues()).thenReturn(Collections.emptyList());
77 77 ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core");
78   - ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3);
  78 + ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 10);
79 79 ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName);
80   - ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize);
81 80 TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder()
82   - .setServiceId("100.96.1.1")
  81 + .setServiceId("tb-core-0")
  82 + .setTenantIdMSB(TenantId.NULL_UUID.getMostSignificantBits())
  83 + .setTenantIdLSB(TenantId.NULL_UUID.getLeastSignificantBits())
83 84 .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name()))
84 85 .build();
85 86 // when(discoveryService.getServiceInfo()).thenReturn(currentServer);
86 87 List<TransportProtos.ServiceInfo> otherServers = new ArrayList<>();
87   - for (int i = 1; i < 30; i++) {
  88 + for (int i = 1; i < SERVER_COUNT; i++) {
88 89 otherServers.add(TransportProtos.ServiceInfo.newBuilder()
89   - .setServiceId("100.96." + i * 2 + "." + i)
  90 + .setServiceId("tb-rule-" + i)
  91 + .setTenantIdMSB(TenantId.NULL_UUID.getMostSignificantBits())
  92 + .setTenantIdLSB(TenantId.NULL_UUID.getLeastSignificantBits())
90 93 .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name()))
91 94 .build());
92 95 }
... ... @@ -116,12 +119,11 @@ public class ConsistentHashParitionServiceTest {
116 119 long end = System.currentTimeMillis();
117 120 double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue());
118 121 double diffPercent = (diff / ITERATIONS) * 100.0;
119   - System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", diffPercent) + "%)");
  122 + System.out.println("Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", diffPercent) + "%)");
120 123 Assert.assertTrue(diffPercent < 0.5);
121 124 for (Map.Entry<Integer, Integer> entry : data) {
122 125 System.out.println(entry.getKey() + ": " + entry.getValue());
123 126 }
124   -
125 127 }
126 128
127 129 }
... ...
common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java renamed from common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.discovery;
18 18 import com.google.common.hash.HashCode;
19 19 import com.google.common.hash.HashFunction;
20 20 import com.google.common.hash.Hashing;
  21 +import lombok.Getter;
21 22 import lombok.extern.slf4j.Slf4j;
22 23 import org.springframework.beans.factory.annotation.Value;
23 24 import org.springframework.context.ApplicationEventPublisher;
... ... @@ -35,6 +36,7 @@ import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
35 36 import javax.annotation.PostConstruct;
36 37 import java.nio.charset.StandardCharsets;
37 38 import java.util.ArrayList;
  39 +import java.util.Comparator;
38 40 import java.util.HashMap;
39 41 import java.util.HashSet;
40 42 import java.util.List;
... ... @@ -48,7 +50,7 @@ import java.util.stream.Collectors;
48 50
49 51 @Service
50 52 @Slf4j
51   -public class ConsistentHashPartitionService implements PartitionService {
  53 +public class HashPartitionService implements PartitionService {
52 54
53 55 @Value("${queue.core.topic}")
54 56 private String coreTopic;
... ... @@ -56,8 +58,6 @@ public class ConsistentHashPartitionService implements PartitionService {
56 58 private Integer corePartitions;
57 59 @Value("${queue.partitions.hash_function_name:murmur3_128}")
58 60 private String hashFunctionName;
59   - @Value("${queue.partitions.virtual_nodes_size:16}")
60   - private Integer virtualNodesSize;
61 61
62 62 private final ApplicationEventPublisher applicationEventPublisher;
63 63 private final TbServiceInfoProvider serviceInfoProvider;
... ... @@ -76,10 +76,10 @@ public class ConsistentHashPartitionService implements PartitionService {
76 76
77 77 private HashFunction hashFunction;
78 78
79   - public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider,
80   - TenantRoutingInfoService tenantRoutingInfoService,
81   - ApplicationEventPublisher applicationEventPublisher,
82   - TbQueueRuleEngineSettings tbQueueRuleEngineSettings) {
  79 + public HashPartitionService(TbServiceInfoProvider serviceInfoProvider,
  80 + TenantRoutingInfoService tenantRoutingInfoService,
  81 + ApplicationEventPublisher applicationEventPublisher,
  82 + TbQueueRuleEngineSettings tbQueueRuleEngineSettings) {
83 83 this.serviceInfoProvider = serviceInfoProvider;
84 84 this.tenantRoutingInfoService = tenantRoutingInfoService;
85 85 this.applicationEventPublisher = applicationEventPublisher;
... ... @@ -128,20 +128,22 @@ public class ConsistentHashPartitionService implements PartitionService {
128 128 public void recalculatePartitions(ServiceInfo currentService, List<ServiceInfo> otherServices) {
129 129 logServiceInfo(currentService);
130 130 otherServices.forEach(this::logServiceInfo);
131   - Map<ServiceQueueKey, ConsistentHashCircle<ServiceInfo>> circles = new HashMap<>();
132   - addNode(circles, currentService);
  131 + Map<ServiceQueueKey, List<ServiceInfo>> queueServicesMap = new HashMap<>();
  132 + addNode(queueServicesMap, currentService);
133 133 for (ServiceInfo other : otherServices) {
134   - addNode(circles, other);
  134 + addNode(queueServicesMap, other);
135 135 }
  136 + queueServicesMap.values().forEach(list -> list.sort((a, b) -> a.getServiceId().compareTo(b.getServiceId())));
  137 +
136 138 ConcurrentMap<ServiceQueueKey, List<Integer>> oldPartitions = myPartitions;
137 139 TenantId myIsolatedOrSystemTenantId = getSystemOrIsolatedTenantId(currentService);
138 140 myPartitions = new ConcurrentHashMap<>();
139   - partitionSizes.forEach((type, size) -> {
140   - ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(type, myIsolatedOrSystemTenantId);
  141 + partitionSizes.forEach((serviceQueue, size) -> {
  142 + ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(serviceQueue, myIsolatedOrSystemTenantId);
141 143 for (int i = 0; i < size; i++) {
142   - ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceQueueKey), i);
  144 + ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(myServiceQueueKey), i);
143 145 if (currentService.equals(serviceInfo)) {
144   - ServiceQueueKey serviceQueueKey = new ServiceQueueKey(type, getSystemOrIsolatedTenantId(serviceInfo));
  146 + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(serviceQueue, getSystemOrIsolatedTenantId(serviceInfo));
145 147 myPartitions.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(i);
146 148 }
147 149 }
... ... @@ -293,7 +295,7 @@ public class ConsistentHashPartitionService implements PartitionService {
293 295 return new TenantId(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB()));
294 296 }
295 297
296   - private void addNode(Map<ServiceQueueKey, ConsistentHashCircle<ServiceInfo>> circles, ServiceInfo instance) {
  298 + private void addNode(Map<ServiceQueueKey, List<ServiceInfo>> queueServiceList, ServiceInfo instance) {
297 299 TenantId tenantId = getSystemOrIsolatedTenantId(instance);
298 300 for (String serviceTypeStr : instance.getServiceTypesList()) {
299 301 ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase());
... ... @@ -302,34 +304,20 @@ public class ConsistentHashPartitionService implements PartitionService {
302 304 ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), tenantId);
303 305 partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getPartitions());
304 306 partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getTopic());
305   - for (int i = 0; i < virtualNodesSize; i++) {
306   - circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance);
307   - }
  307 + queueServiceList.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(instance);
308 308 }
309 309 } else {
310 310 ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), tenantId);
311   - for (int i = 0; i < virtualNodesSize; i++) {
312   - circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance);
313   - }
  311 + queueServiceList.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(instance);
314 312 }
315 313 }
316 314 }
317 315
318   - private ServiceInfo resolveByPartitionIdx(ConsistentHashCircle<ServiceInfo> circle, Integer partitionIdx) {
319   - if (circle == null || circle.isEmpty()) {
  316 + private ServiceInfo resolveByPartitionIdx(List<ServiceInfo> servers, Integer partitionIdx) {
  317 + if (servers == null || servers.isEmpty()) {
320 318 return null;
321 319 }
322   - Long hash = hashFunction.newHasher().putInt(partitionIdx).hash().asLong();
323   - if (!circle.containsKey(hash)) {
324   - ConcurrentNavigableMap<Long, ServiceInfo> tailMap = circle.tailMap(hash);
325   - hash = tailMap.isEmpty() ?
326   - circle.firstKey() : tailMap.firstKey();
327   - }
328   - return circle.get(hash);
329   - }
330   -
331   - private HashCode hash(ServiceInfo instance, int i) {
332   - return hashFunction.newHasher().putString(instance.getServiceId(), StandardCharsets.UTF_8).putInt(i).hash();
  320 + return servers.get(partitionIdx % servers.size());
333 321 }
334 322
335 323 public static HashFunction forName(String name) {
... ... @@ -338,12 +326,11 @@ public class ConsistentHashPartitionService implements PartitionService {
338 326 return Hashing.murmur3_32();
339 327 case "murmur3_128":
340 328 return Hashing.murmur3_128();
341   - case "crc32":
342   - return Hashing.crc32();
343   - case "md5":
344   - return Hashing.md5();
  329 + case "sha256":
  330 + return Hashing.sha256();
345 331 default:
346 332 throw new IllegalArgumentException("Can't find hash function with name " + name);
347 333 }
348 334 }
  335 +
349 336 }
... ...