Showing
13 changed files
with
1 additions
and
584 deletions
@@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.id.TenantId; | @@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.id.TenantId; | ||
26 | import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; | 26 | import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; |
27 | import org.thingsboard.server.common.msg.TbMsg; | 27 | import org.thingsboard.server.common.msg.TbMsg; |
28 | import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; | 28 | import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; |
29 | -import org.thingsboard.server.service.queue.MsgQueueService; | ||
30 | 29 | ||
31 | import javax.annotation.Nullable; | 30 | import javax.annotation.Nullable; |
32 | import java.util.function.Consumer; | 31 | import java.util.function.Consumer; |
application/src/main/java/org/thingsboard/server/service/queue/DefaultMsgQueueService.java
deleted
100644 → 0
1 | -/** | ||
2 | - * Copyright © 2016-2018 The Thingsboard Authors | ||
3 | - * | ||
4 | - * Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | - * you may not use this file except in compliance with the License. | ||
6 | - * You may obtain a copy of the License at | ||
7 | - * | ||
8 | - * http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | - * | ||
10 | - * Unless required by applicable law or agreed to in writing, software | ||
11 | - * distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | - * See the License for the specific language governing permissions and | ||
14 | - * limitations under the License. | ||
15 | - */ | ||
16 | -package org.thingsboard.server.service.queue; | ||
17 | - | ||
18 | -import com.google.common.util.concurrent.Futures; | ||
19 | -import com.google.common.util.concurrent.ListenableFuture; | ||
20 | -import lombok.extern.slf4j.Slf4j; | ||
21 | -import org.springframework.beans.factory.annotation.Autowired; | ||
22 | -import org.springframework.beans.factory.annotation.Value; | ||
23 | -import org.springframework.stereotype.Service; | ||
24 | -import org.thingsboard.server.common.data.id.TenantId; | ||
25 | -import org.thingsboard.server.common.msg.TbMsg; | ||
26 | -import org.thingsboard.server.common.transport.quota.tenant.TenantQuotaService; | ||
27 | -import org.thingsboard.server.dao.queue.MsgQueue; | ||
28 | - | ||
29 | -import javax.annotation.PostConstruct; | ||
30 | -import javax.annotation.PreDestroy; | ||
31 | -import java.util.Map; | ||
32 | -import java.util.UUID; | ||
33 | -import java.util.concurrent.ConcurrentHashMap; | ||
34 | -import java.util.concurrent.Executors; | ||
35 | -import java.util.concurrent.ScheduledExecutorService; | ||
36 | -import java.util.concurrent.TimeUnit; | ||
37 | -import java.util.concurrent.atomic.AtomicLong; | ||
38 | - | ||
39 | -@Service | ||
40 | -@Slf4j | ||
41 | -public class DefaultMsgQueueService implements MsgQueueService { | ||
42 | - | ||
43 | - @Value("${actors.rule.queue.max_size}") | ||
44 | - private long queueMaxSize; | ||
45 | - | ||
46 | - @Value("${actors.rule.queue.cleanup_period}") | ||
47 | - private long queueCleanUpPeriod; | ||
48 | - | ||
49 | - @Autowired | ||
50 | - private MsgQueue msgQueue; | ||
51 | - | ||
52 | - @Autowired(required = false) | ||
53 | - private TenantQuotaService quotaService; | ||
54 | - | ||
55 | - private ScheduledExecutorService cleanupExecutor; | ||
56 | - | ||
57 | - private Map<TenantId, AtomicLong> pendingCountPerTenant = new ConcurrentHashMap<>(); | ||
58 | - | ||
59 | - @PostConstruct | ||
60 | - public void init() { | ||
61 | - if (queueCleanUpPeriod > 0) { | ||
62 | - cleanupExecutor = Executors.newSingleThreadScheduledExecutor(); | ||
63 | - cleanupExecutor.scheduleAtFixedRate(() -> cleanup(), | ||
64 | - queueCleanUpPeriod, queueCleanUpPeriod, TimeUnit.SECONDS); | ||
65 | - } | ||
66 | - } | ||
67 | - | ||
68 | - @PreDestroy | ||
69 | - public void stop() { | ||
70 | - if (cleanupExecutor != null) { | ||
71 | - cleanupExecutor.shutdownNow(); | ||
72 | - } | ||
73 | - } | ||
74 | - | ||
75 | - @Override | ||
76 | - public ListenableFuture<Void> put(TenantId tenantId, TbMsg msg, UUID nodeId, long clusterPartition) { | ||
77 | - if(quotaService != null && quotaService.isQuotaExceeded(tenantId.getId().toString())) { | ||
78 | - log.warn("Tenant TbMsg Quota exceeded for [{}:{}] . Reject", tenantId.getId()); | ||
79 | - return Futures.immediateFailedFuture(new RuntimeException("Tenant TbMsg Quota exceeded")); | ||
80 | - } | ||
81 | - | ||
82 | - AtomicLong pendingMsgCount = pendingCountPerTenant.computeIfAbsent(tenantId, key -> new AtomicLong()); | ||
83 | - if (pendingMsgCount.incrementAndGet() < queueMaxSize) { | ||
84 | - return msgQueue.put(tenantId, msg, nodeId, clusterPartition); | ||
85 | - } else { | ||
86 | - pendingMsgCount.decrementAndGet(); | ||
87 | - return Futures.immediateFailedFuture(new RuntimeException("Message queue is full!")); | ||
88 | - } | ||
89 | - } | ||
90 | - | ||
91 | - @Override | ||
92 | - public ListenableFuture<Void> ack(TenantId tenantId, TbMsg msg, UUID nodeId, long clusterPartition) { | ||
93 | - ListenableFuture<Void> result = msgQueue.ack(tenantId, msg, nodeId, clusterPartition); | ||
94 | - AtomicLong pendingMsgCount = pendingCountPerTenant.computeIfAbsent(tenantId, key -> new AtomicLong()); | ||
95 | - pendingMsgCount.decrementAndGet(); | ||
96 | - return result; | ||
97 | - } | ||
98 | - | ||
99 | - @Override | ||
100 | - public Iterable<TbMsg> findUnprocessed(TenantId tenantId, UUID nodeId, long clusterPartition) { | ||
101 | - return msgQueue.findUnprocessed(tenantId, nodeId, clusterPartition); | ||
102 | - } | ||
103 | - | ||
104 | - private void cleanup() { | ||
105 | - pendingCountPerTenant.forEach((tenantId, pendingMsgCount) -> { | ||
106 | - pendingMsgCount.set(0); | ||
107 | - msgQueue.cleanUp(tenantId); | ||
108 | - }); | ||
109 | - } | ||
110 | - | ||
111 | -} |
application/src/main/java/org/thingsboard/server/service/queue/MsgQueueService.java
deleted
100644 → 0
1 | -/** | ||
2 | - * Copyright © 2016-2018 The Thingsboard Authors | ||
3 | - * | ||
4 | - * Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | - * you may not use this file except in compliance with the License. | ||
6 | - * You may obtain a copy of the License at | ||
7 | - * | ||
8 | - * http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | - * | ||
10 | - * Unless required by applicable law or agreed to in writing, software | ||
11 | - * distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | - * See the License for the specific language governing permissions and | ||
14 | - * limitations under the License. | ||
15 | - */ | ||
16 | -package org.thingsboard.server.service.queue; | ||
17 | - | ||
18 | -import com.google.common.util.concurrent.ListenableFuture; | ||
19 | -import org.thingsboard.server.common.data.id.TenantId; | ||
20 | -import org.thingsboard.server.common.msg.TbMsg; | ||
21 | - | ||
22 | -import java.util.UUID; | ||
23 | - | ||
24 | -public interface MsgQueueService { | ||
25 | - | ||
26 | - ListenableFuture<Void> put(TenantId tenantId, TbMsg msg, UUID nodeId, long clusterPartition); | ||
27 | - | ||
28 | - ListenableFuture<Void> ack(TenantId tenantId, TbMsg msg, UUID nodeId, long clusterPartition); | ||
29 | - | ||
30 | - Iterable<TbMsg> findUnprocessed(TenantId tenantId, UUID nodeId, long clusterPartition); | ||
31 | - | ||
32 | -} |
@@ -182,12 +182,6 @@ cassandra: | @@ -182,12 +182,6 @@ cassandra: | ||
182 | permit_max_wait_time: "${PERMIT_MAX_WAIT_TIME:120000}" | 182 | permit_max_wait_time: "${PERMIT_MAX_WAIT_TIME:120000}" |
183 | rate_limit_print_interval_ms: "${CASSANDRA_QUERY_RATE_LIMIT_PRINT_MS:30000}" | 183 | rate_limit_print_interval_ms: "${CASSANDRA_QUERY_RATE_LIMIT_PRINT_MS:30000}" |
184 | 184 | ||
185 | - queue: | ||
186 | - msg.ttl: 604800 # 7 days | ||
187 | - ack.ttl: 604800 # 7 days | ||
188 | - partitions.ttl: 604800 # 7 days | ||
189 | - partitioning: "HOURS" | ||
190 | - | ||
191 | # SQL configuration parameters | 185 | # SQL configuration parameters |
192 | sql: | 186 | sql: |
193 | # Specify executor service type used to perform timeseries insert tasks: SINGLE FIXED CACHED | 187 | # Specify executor service type used to perform timeseries insert tasks: SINGLE FIXED CACHED |
@@ -221,13 +215,6 @@ actors: | @@ -221,13 +215,6 @@ actors: | ||
221 | node: | 215 | node: |
222 | # Errors for particular actor are persisted once per specified amount of milliseconds | 216 | # Errors for particular actor are persisted once per specified amount of milliseconds |
223 | error_persist_frequency: "${ACTORS_RULE_NODE_ERROR_FREQUENCY:3000}" | 217 | error_persist_frequency: "${ACTORS_RULE_NODE_ERROR_FREQUENCY:3000}" |
224 | - queue: | ||
225 | - # Message queue type | ||
226 | - type: "${ACTORS_RULE_QUEUE_TYPE:memory}" | ||
227 | - # Message queue maximum size (per tenant) | ||
228 | - max_size: "${ACTORS_RULE_QUEUE_MAX_SIZE:100}" | ||
229 | - # Message queue cleanup period in seconds | ||
230 | - cleanup_period: "${ACTORS_RULE_QUEUE_CLEANUP_PERIOD:3600}" | ||
231 | statistics: | 218 | statistics: |
232 | # Enable/disable actor statistics | 219 | # Enable/disable actor statistics |
233 | enabled: "${ACTORS_STATISTICS_ENABLED:true}" | 220 | enabled: "${ACTORS_STATISTICS_ENABLED:true}" |
@@ -27,9 +27,7 @@ import org.thingsboard.server.common.data.page.TimePageData; | @@ -27,9 +27,7 @@ import org.thingsboard.server.common.data.page.TimePageData; | ||
27 | import org.thingsboard.server.common.data.page.TimePageLink; | 27 | import org.thingsboard.server.common.data.page.TimePageLink; |
28 | import org.thingsboard.server.common.data.rule.RuleChain; | 28 | import org.thingsboard.server.common.data.rule.RuleChain; |
29 | import org.thingsboard.server.common.data.rule.RuleChainMetaData; | 29 | import org.thingsboard.server.common.data.rule.RuleChainMetaData; |
30 | -import org.thingsboard.server.dao.queue.MsgQueue; | ||
31 | import org.thingsboard.server.dao.rule.RuleChainService; | 30 | import org.thingsboard.server.dao.rule.RuleChainService; |
32 | -import org.thingsboard.server.service.queue.MsgQueueService; | ||
33 | 31 | ||
34 | import java.io.IOException; | 32 | import java.io.IOException; |
35 | import java.util.function.Predicate; | 33 | import java.util.function.Predicate; |
@@ -42,9 +40,6 @@ public class AbstractRuleEngineControllerTest extends AbstractControllerTest { | @@ -42,9 +40,6 @@ public class AbstractRuleEngineControllerTest extends AbstractControllerTest { | ||
42 | @Autowired | 40 | @Autowired |
43 | protected RuleChainService ruleChainService; | 41 | protected RuleChainService ruleChainService; |
44 | 42 | ||
45 | - @Autowired | ||
46 | - protected MsgQueueService msgQueueService; | ||
47 | - | ||
48 | protected RuleChain saveRuleChain(RuleChain ruleChain) throws Exception { | 43 | protected RuleChain saveRuleChain(RuleChain ruleChain) throws Exception { |
49 | return doPost("/api/ruleChain", ruleChain, RuleChain.class); | 44 | return doPost("/api/ruleChain", ruleChain, RuleChain.class); |
50 | } | 45 | } |
@@ -191,9 +191,6 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule | @@ -191,9 +191,6 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule | ||
191 | 191 | ||
192 | Assert.assertEquals("serverAttributeValue1", getMetadata(outEvent).get("ss_serverAttributeKey1").asText()); | 192 | Assert.assertEquals("serverAttributeValue1", getMetadata(outEvent).get("ss_serverAttributeKey1").asText()); |
193 | Assert.assertEquals("serverAttributeValue2", getMetadata(outEvent).get("ss_serverAttributeKey2").asText()); | 193 | Assert.assertEquals("serverAttributeValue2", getMetadata(outEvent).get("ss_serverAttributeKey2").asText()); |
194 | - | ||
195 | - List<TbMsg> unAckMsgList = Lists.newArrayList(msgQueueService.findUnprocessed(savedTenant.getId(), ruleChain.getId().getId(), 0L)); | ||
196 | - Assert.assertEquals(0, unAckMsgList.size()); | ||
197 | } | 194 | } |
198 | 195 | ||
199 | @Test | 196 | @Test |
@@ -311,12 +308,6 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule | @@ -311,12 +308,6 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule | ||
311 | 308 | ||
312 | Assert.assertEquals("serverAttributeValue1", getMetadata(outEvent).get("ss_serverAttributeKey1").asText()); | 309 | Assert.assertEquals("serverAttributeValue1", getMetadata(outEvent).get("ss_serverAttributeKey1").asText()); |
313 | Assert.assertEquals("serverAttributeValue2", getMetadata(outEvent).get("ss_serverAttributeKey2").asText()); | 310 | Assert.assertEquals("serverAttributeValue2", getMetadata(outEvent).get("ss_serverAttributeKey2").asText()); |
314 | - | ||
315 | - List<TbMsg> unAckMsgList = Lists.newArrayList(msgQueueService.findUnprocessed(savedTenant.getId(), rootRuleChain.getId().getId(), 0L)); | ||
316 | - Assert.assertEquals(0, unAckMsgList.size()); | ||
317 | - | ||
318 | - unAckMsgList = Lists.newArrayList(msgQueueService.findUnprocessed(savedTenant.getId(), secondaryRuleChain.getId().getId(), 0L)); | ||
319 | - Assert.assertEquals(0, unAckMsgList.size()); | ||
320 | } | 311 | } |
321 | 312 | ||
322 | } | 313 | } |
@@ -162,73 +162,4 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac | @@ -162,73 +162,4 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac | ||
162 | Assert.assertEquals("serverAttributeValue", getMetadata(outEvent).get("ss_serverAttributeKey").asText()); | 162 | Assert.assertEquals("serverAttributeValue", getMetadata(outEvent).get("ss_serverAttributeKey").asText()); |
163 | } | 163 | } |
164 | 164 | ||
165 | - @Test | ||
166 | - public void testRuleChainWithOneRuleAndMsgFromQueue() throws Exception { | ||
167 | - // Creating Rule Chain | ||
168 | - RuleChain ruleChain = new RuleChain(); | ||
169 | - ruleChain.setName("Simple Rule Chain"); | ||
170 | - ruleChain.setTenantId(savedTenant.getId()); | ||
171 | - ruleChain.setRoot(true); | ||
172 | - ruleChain.setDebugMode(true); | ||
173 | - ruleChain = saveRuleChain(ruleChain); | ||
174 | - Assert.assertNull(ruleChain.getFirstRuleNodeId()); | ||
175 | - | ||
176 | - // Saving the device | ||
177 | - Device device = new Device(); | ||
178 | - device.setName("My device"); | ||
179 | - device.setType("default"); | ||
180 | - device = doPost("/api/device", device, Device.class); | ||
181 | - | ||
182 | - attributesService.save(device.getId(), DataConstants.SERVER_SCOPE, | ||
183 | - Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry("serverAttributeKey", "serverAttributeValue"), System.currentTimeMillis()))); | ||
184 | - | ||
185 | - // Pushing Message to the system | ||
186 | - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), | ||
187 | - "CUSTOM", | ||
188 | - device.getId(), | ||
189 | - new TbMsgMetaData(), | ||
190 | - "{}", | ||
191 | - ruleChain.getId(), null, 0L); | ||
192 | - msgQueueService.put(device.getTenantId(), tbMsg, ruleChain.getId().getId(), 0L); | ||
193 | - | ||
194 | - Thread.sleep(1000); | ||
195 | - | ||
196 | - RuleChainMetaData metaData = new RuleChainMetaData(); | ||
197 | - metaData.setRuleChainId(ruleChain.getId()); | ||
198 | - | ||
199 | - RuleNode ruleNode = new RuleNode(); | ||
200 | - ruleNode.setName("Simple Rule Node"); | ||
201 | - ruleNode.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); | ||
202 | - ruleNode.setDebugMode(true); | ||
203 | - TbGetAttributesNodeConfiguration configuration = new TbGetAttributesNodeConfiguration(); | ||
204 | - configuration.setServerAttributeNames(Collections.singletonList("serverAttributeKey")); | ||
205 | - ruleNode.setConfiguration(mapper.valueToTree(configuration)); | ||
206 | - | ||
207 | - metaData.setNodes(Collections.singletonList(ruleNode)); | ||
208 | - metaData.setFirstNodeIndex(0); | ||
209 | - | ||
210 | - metaData = saveRuleChainMetaData(metaData); | ||
211 | - Assert.assertNotNull(metaData); | ||
212 | - | ||
213 | - ruleChain = getRuleChain(ruleChain.getId()); | ||
214 | - Assert.assertNotNull(ruleChain.getFirstRuleNodeId()); | ||
215 | - | ||
216 | - Thread.sleep(3000); | ||
217 | - | ||
218 | - TimePageData<Event> eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); | ||
219 | - List<Event> events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); | ||
220 | - | ||
221 | - Assert.assertEquals(2, events.size()); | ||
222 | - | ||
223 | - Event inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); | ||
224 | - Assert.assertEquals(ruleChain.getFirstRuleNodeId(), inEvent.getEntityId()); | ||
225 | - Assert.assertEquals(device.getId().getId().toString(), inEvent.getBody().get("entityId").asText()); | ||
226 | - | ||
227 | - Event outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); | ||
228 | - Assert.assertEquals(ruleChain.getFirstRuleNodeId(), outEvent.getEntityId()); | ||
229 | - Assert.assertEquals(device.getId().getId().toString(), outEvent.getBody().get("entityId").asText()); | ||
230 | - | ||
231 | - Assert.assertEquals("serverAttributeValue", getMetadata(outEvent).get("ss_serverAttributeKey").asText()); | ||
232 | - } | ||
233 | - | ||
234 | } | 165 | } |
@@ -57,7 +57,7 @@ public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest { | @@ -57,7 +57,7 @@ public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest { | ||
57 | @Test | 57 | @Test |
58 | public void testGetAttributes() throws Exception { | 58 | public void testGetAttributes() throws Exception { |
59 | doGetAsync("/api/v1/" + "WRONG_TOKEN" + "/attributes?clientKeys=keyA,keyB,keyC").andExpect(status().isUnauthorized()); | 59 | doGetAsync("/api/v1/" + "WRONG_TOKEN" + "/attributes?clientKeys=keyA,keyB,keyC").andExpect(status().isUnauthorized()); |
60 | - doGetAsync("/api/v1/" + deviceCredentials.getCredentialsId() + "/attributes?clientKeys=keyA,keyB,keyC").andExpect(status().isNotFound()); | 60 | + doGetAsync("/api/v1/" + deviceCredentials.getCredentialsId() + "/attributes?clientKeys=keyA,keyB,keyC").andExpect(status().isOk()); |
61 | 61 | ||
62 | Map<String, String> attrMap = new HashMap<>(); | 62 | Map<String, String> attrMap = new HashMap<>(); |
63 | attrMap.put("keyA", "valueA"); | 63 | attrMap.put("keyA", "valueA"); |
dao/src/main/java/org/thingsboard/server/dao/queue/MsgQueue.java
deleted
100644 → 0
1 | -/** | ||
2 | - * Copyright © 2016-2018 The Thingsboard Authors | ||
3 | - * | ||
4 | - * Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | - * you may not use this file except in compliance with the License. | ||
6 | - * You may obtain a copy of the License at | ||
7 | - * | ||
8 | - * http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | - * | ||
10 | - * Unless required by applicable law or agreed to in writing, software | ||
11 | - * distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | - * See the License for the specific language governing permissions and | ||
14 | - * limitations under the License. | ||
15 | - */ | ||
16 | -package org.thingsboard.server.dao.queue; | ||
17 | - | ||
18 | -import com.google.common.util.concurrent.ListenableFuture; | ||
19 | -import org.thingsboard.server.common.data.id.TenantId; | ||
20 | -import org.thingsboard.server.common.msg.TbMsg; | ||
21 | - | ||
22 | -import java.util.UUID; | ||
23 | - | ||
24 | -public interface MsgQueue { | ||
25 | - | ||
26 | - ListenableFuture<Void> put(TenantId tenantId, TbMsg msg, UUID nodeId, long clusterPartition); | ||
27 | - | ||
28 | - ListenableFuture<Void> ack(TenantId tenantId, TbMsg msg, UUID nodeId, long clusterPartition); | ||
29 | - | ||
30 | - Iterable<TbMsg> findUnprocessed(TenantId tenantId, UUID nodeId, long clusterPartition); | ||
31 | - | ||
32 | - ListenableFuture<Void> cleanUp(TenantId tenantId); | ||
33 | - | ||
34 | -} |
dao/src/main/java/org/thingsboard/server/dao/queue/QueueBenchmark.java
deleted
100644 → 0
1 | -/** | ||
2 | - * Copyright © 2016-2018 The Thingsboard Authors | ||
3 | - * | ||
4 | - * Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | - * you may not use this file except in compliance with the License. | ||
6 | - * You may obtain a copy of the License at | ||
7 | - * | ||
8 | - * http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | - * | ||
10 | - * Unless required by applicable law or agreed to in writing, software | ||
11 | - * distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | - * See the License for the specific language governing permissions and | ||
14 | - * limitations under the License. | ||
15 | - */ | ||
16 | -package org.thingsboard.server.dao.queue; | ||
17 | - | ||
18 | -import com.datastax.driver.core.Cluster; | ||
19 | -import com.datastax.driver.core.HostDistance; | ||
20 | -import com.datastax.driver.core.PoolingOptions; | ||
21 | -import com.datastax.driver.core.Session; | ||
22 | -import com.datastax.driver.core.utils.UUIDs; | ||
23 | -import com.google.common.util.concurrent.FutureCallback; | ||
24 | -import com.google.common.util.concurrent.Futures; | ||
25 | -import com.google.common.util.concurrent.ListenableFuture; | ||
26 | -import lombok.extern.slf4j.Slf4j; | ||
27 | -import org.springframework.beans.factory.annotation.Autowired; | ||
28 | -import org.springframework.boot.CommandLineRunner; | ||
29 | -import org.springframework.boot.SpringApplication; | ||
30 | -import org.springframework.context.annotation.Bean; | ||
31 | -import org.thingsboard.server.common.data.id.EntityId; | ||
32 | -import org.thingsboard.server.common.data.id.RuleChainId; | ||
33 | -import org.thingsboard.server.common.data.id.RuleNodeId; | ||
34 | -import org.thingsboard.server.common.data.id.TenantId; | ||
35 | -import org.thingsboard.server.common.msg.TbMsg; | ||
36 | -import org.thingsboard.server.common.msg.TbMsgDataType; | ||
37 | -import org.thingsboard.server.common.msg.TbMsgMetaData; | ||
38 | - | ||
39 | -import javax.annotation.Nullable; | ||
40 | -import java.net.InetSocketAddress; | ||
41 | -import java.util.UUID; | ||
42 | -import java.util.concurrent.CountDownLatch; | ||
43 | -import java.util.concurrent.ExecutorService; | ||
44 | -import java.util.concurrent.Executors; | ||
45 | -import java.util.concurrent.TimeUnit; | ||
46 | -import java.util.concurrent.atomic.AtomicLong; | ||
47 | - | ||
48 | -//@SpringBootApplication | ||
49 | -//@EnableAutoConfiguration | ||
50 | -//@ComponentScan({"org.thingsboard.rule.engine"}) | ||
51 | -//@PropertySource("classpath:processing-pipeline.properties") | ||
52 | -@Slf4j | ||
53 | -public class QueueBenchmark implements CommandLineRunner { | ||
54 | - | ||
55 | - public static void main(String[] args) { | ||
56 | - try { | ||
57 | - SpringApplication.run(QueueBenchmark.class, args); | ||
58 | - } catch (Throwable th) { | ||
59 | - th.printStackTrace(); | ||
60 | - System.exit(0); | ||
61 | - } | ||
62 | - } | ||
63 | - | ||
64 | - @Autowired | ||
65 | - private MsgQueue msgQueue; | ||
66 | - | ||
67 | - @Override | ||
68 | - public void run(String... strings) throws Exception { | ||
69 | - System.out.println("It works + " + msgQueue); | ||
70 | - | ||
71 | - | ||
72 | - long start = System.currentTimeMillis(); | ||
73 | - int msgCount = 10000000; | ||
74 | - AtomicLong count = new AtomicLong(0); | ||
75 | - ExecutorService service = Executors.newFixedThreadPool(100); | ||
76 | - | ||
77 | - CountDownLatch latch = new CountDownLatch(msgCount); | ||
78 | - for (int i = 0; i < msgCount; i++) { | ||
79 | - service.submit(() -> { | ||
80 | - boolean isFinished = false; | ||
81 | - while (!isFinished) { | ||
82 | - try { | ||
83 | - TbMsg msg = randomMsg(); | ||
84 | - UUID nodeId = UUIDs.timeBased(); | ||
85 | - ListenableFuture<Void> put = msgQueue.put(new TenantId(EntityId.NULL_UUID), msg, nodeId, 100L); | ||
86 | -// ListenableFuture<Void> put = msgQueue.ack(msg, nodeId, 100L); | ||
87 | - Futures.addCallback(put, new FutureCallback<Void>() { | ||
88 | - @Override | ||
89 | - public void onSuccess(@Nullable Void result) { | ||
90 | - latch.countDown(); | ||
91 | - } | ||
92 | - | ||
93 | - @Override | ||
94 | - public void onFailure(Throwable t) { | ||
95 | -// t.printStackTrace(); | ||
96 | - System.out.println("onFailure, because:" + t.getMessage()); | ||
97 | - latch.countDown(); | ||
98 | - } | ||
99 | - }); | ||
100 | - isFinished = true; | ||
101 | - } catch (Throwable th) { | ||
102 | -// th.printStackTrace(); | ||
103 | - System.out.println("Repeat query, because:" + th.getMessage()); | ||
104 | -// latch.countDown(); | ||
105 | - } | ||
106 | - } | ||
107 | - }); | ||
108 | - } | ||
109 | - | ||
110 | - long prev = 0L; | ||
111 | - while (latch.getCount() != 0) { | ||
112 | - TimeUnit.SECONDS.sleep(1); | ||
113 | - long curr = latch.getCount(); | ||
114 | - long rps = prev - curr; | ||
115 | - prev = curr; | ||
116 | - System.out.println("rps = " + rps); | ||
117 | - } | ||
118 | - | ||
119 | - long end = System.currentTimeMillis(); | ||
120 | - System.out.println("final rps = " + (msgCount / (end - start) * 1000)); | ||
121 | - | ||
122 | - System.out.println("Finished"); | ||
123 | - | ||
124 | - } | ||
125 | - | ||
126 | - private TbMsg randomMsg() { | ||
127 | - TbMsgMetaData metaData = new TbMsgMetaData(); | ||
128 | - metaData.putValue("key", "value"); | ||
129 | - String dataStr = "someContent"; | ||
130 | - return new TbMsg(UUIDs.timeBased(), "type", null, metaData, TbMsgDataType.JSON, dataStr, new RuleChainId(UUIDs.timeBased()), new RuleNodeId(UUIDs.timeBased()), 0L); | ||
131 | - } | ||
132 | - | ||
133 | - @Bean | ||
134 | - public Session session() { | ||
135 | - Cluster thingsboard = Cluster.builder() | ||
136 | - .addContactPointsWithPorts(new InetSocketAddress("127.0.0.1", 9042)) | ||
137 | - .withClusterName("thingsboard") | ||
138 | -// .withSocketOptions(socketOpts.getOpts()) | ||
139 | - .withPoolingOptions(new PoolingOptions() | ||
140 | - .setMaxRequestsPerConnection(HostDistance.LOCAL, 32768) | ||
141 | - .setMaxRequestsPerConnection(HostDistance.REMOTE, 32768)).build(); | ||
142 | - | ||
143 | - Session session = thingsboard.connect("thingsboard"); | ||
144 | - return session; | ||
145 | - } | ||
146 | - | ||
147 | - @Bean | ||
148 | - public int defaultTtl() { | ||
149 | - return 6000; | ||
150 | - } | ||
151 | - | ||
152 | -} |
1 | -/** | ||
2 | - * Copyright © 2016-2018 The Thingsboard Authors | ||
3 | - * | ||
4 | - * Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | - * you may not use this file except in compliance with the License. | ||
6 | - * You may obtain a copy of the License at | ||
7 | - * | ||
8 | - * http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | - * | ||
10 | - * Unless required by applicable law or agreed to in writing, software | ||
11 | - * distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | - * See the License for the specific language governing permissions and | ||
14 | - * limitations under the License. | ||
15 | - */ | ||
16 | -package org.thingsboard.server.dao.queue.memory; | ||
17 | - | ||
18 | -import lombok.Data; | ||
19 | - | ||
20 | -import java.util.UUID; | ||
21 | - | ||
22 | -/** | ||
23 | - * Created by ashvayka on 30.04.18. | ||
24 | - */ | ||
25 | -@Data | ||
26 | -public final class InMemoryMsgKey { | ||
27 | - final UUID nodeId; | ||
28 | - final long clusterPartition; | ||
29 | -} |
1 | -/** | ||
2 | - * Copyright © 2016-2018 The Thingsboard Authors | ||
3 | - * | ||
4 | - * Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | - * you may not use this file except in compliance with the License. | ||
6 | - * You may obtain a copy of the License at | ||
7 | - * | ||
8 | - * http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | - * | ||
10 | - * Unless required by applicable law or agreed to in writing, software | ||
11 | - * distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | - * See the License for the specific language governing permissions and | ||
14 | - * limitations under the License. | ||
15 | - */ | ||
16 | -package org.thingsboard.server.dao.queue.memory; | ||
17 | - | ||
18 | -import com.google.common.util.concurrent.ListenableFuture; | ||
19 | -import com.google.common.util.concurrent.ListeningExecutorService; | ||
20 | -import com.google.common.util.concurrent.MoreExecutors; | ||
21 | -import lombok.extern.slf4j.Slf4j; | ||
22 | -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; | ||
23 | -import org.springframework.stereotype.Component; | ||
24 | -import org.thingsboard.server.common.data.id.TenantId; | ||
25 | -import org.thingsboard.server.common.msg.TbMsg; | ||
26 | -import org.thingsboard.server.dao.queue.MsgQueue; | ||
27 | - | ||
28 | -import javax.annotation.PostConstruct; | ||
29 | -import javax.annotation.PreDestroy; | ||
30 | -import java.util.ArrayList; | ||
31 | -import java.util.Collections; | ||
32 | -import java.util.HashMap; | ||
33 | -import java.util.List; | ||
34 | -import java.util.Map; | ||
35 | -import java.util.UUID; | ||
36 | -import java.util.concurrent.ExecutionException; | ||
37 | -import java.util.concurrent.Executors; | ||
38 | - | ||
39 | -/** | ||
40 | - * Created by ashvayka on 27.04.18. | ||
41 | - */ | ||
42 | -@Component | ||
43 | -@ConditionalOnProperty(prefix = "actors.rule.queue", value = "type", havingValue = "memory", matchIfMissing = true) | ||
44 | -@Slf4j | ||
45 | -public class InMemoryMsgQueue implements MsgQueue { | ||
46 | - | ||
47 | - private ListeningExecutorService queueExecutor; | ||
48 | - private Map<TenantId, Map<InMemoryMsgKey, Map<UUID, TbMsg>>> data = new HashMap<>(); | ||
49 | - | ||
50 | - @PostConstruct | ||
51 | - public void init() { | ||
52 | - // Should be always single threaded due to absence of locks. | ||
53 | - queueExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()); | ||
54 | - } | ||
55 | - | ||
56 | - @PreDestroy | ||
57 | - public void stop() { | ||
58 | - if (queueExecutor != null) { | ||
59 | - queueExecutor.shutdownNow(); | ||
60 | - } | ||
61 | - } | ||
62 | - | ||
63 | - @Override | ||
64 | - public ListenableFuture<Void> put(TenantId tenantId, TbMsg msg, UUID nodeId, long clusterPartition) { | ||
65 | - return queueExecutor.submit(() -> { | ||
66 | - data.computeIfAbsent(tenantId, key -> new HashMap<>()). | ||
67 | - computeIfAbsent(new InMemoryMsgKey(nodeId, clusterPartition), key -> new HashMap<>()).put(msg.getId(), msg); | ||
68 | - return null; | ||
69 | - }); | ||
70 | - } | ||
71 | - | ||
72 | - @Override | ||
73 | - public ListenableFuture<Void> ack(TenantId tenantId, TbMsg msg, UUID nodeId, long clusterPartition) { | ||
74 | - return queueExecutor.submit(() -> { | ||
75 | - Map<InMemoryMsgKey, Map<UUID, TbMsg>> tenantMap = data.get(tenantId); | ||
76 | - if (tenantMap != null) { | ||
77 | - InMemoryMsgKey key = new InMemoryMsgKey(nodeId, clusterPartition); | ||
78 | - Map<UUID, TbMsg> map = tenantMap.get(key); | ||
79 | - if (map != null) { | ||
80 | - map.remove(msg.getId()); | ||
81 | - if (map.isEmpty()) { | ||
82 | - tenantMap.remove(key); | ||
83 | - } | ||
84 | - } | ||
85 | - if (tenantMap.isEmpty()) { | ||
86 | - data.remove(tenantId); | ||
87 | - } | ||
88 | - } | ||
89 | - return null; | ||
90 | - }); | ||
91 | - } | ||
92 | - | ||
93 | - @Override | ||
94 | - public Iterable<TbMsg> findUnprocessed(TenantId tenantId, UUID nodeId, long clusterPartition) { | ||
95 | - ListenableFuture<List<TbMsg>> list = queueExecutor.submit(() -> { | ||
96 | - Map<InMemoryMsgKey, Map<UUID, TbMsg>> tenantMap = data.get(tenantId); | ||
97 | - if (tenantMap != null) { | ||
98 | - InMemoryMsgKey key = new InMemoryMsgKey(nodeId, clusterPartition); | ||
99 | - Map<UUID, TbMsg> map = tenantMap.get(key); | ||
100 | - if (map != null) { | ||
101 | - return new ArrayList<>(map.values()); | ||
102 | - } else { | ||
103 | - return Collections.emptyList(); | ||
104 | - } | ||
105 | - } else { | ||
106 | - return Collections.emptyList(); | ||
107 | - } | ||
108 | - }); | ||
109 | - try { | ||
110 | - return list.get(); | ||
111 | - } catch (InterruptedException | ExecutionException e) { | ||
112 | - throw new RuntimeException(e); | ||
113 | - } | ||
114 | - } | ||
115 | - | ||
116 | - @Override | ||
117 | - public ListenableFuture<Void> cleanUp(TenantId tenantId) { | ||
118 | - return queueExecutor.submit(() -> { | ||
119 | - data.remove(tenantId); | ||
120 | - return null; | ||
121 | - }); | ||
122 | - } | ||
123 | -} |