简柏林 / thingskit · Commits

Commit 070e5b5b625c378427a2380994fd86d6174f3d8d

Authored by Andrew Shvayka
1 parent 9764f27a

Query refactoring and improvements

Inline Side-by-side
Showing 51 changed files with 419 additions and 586 deletions
@@ -16,11 +16,11 @@ @@ -16,11 +16,11 @@
16 16
17 CREATE TABLE IF NOT EXISTS thingsboard.msg_queue ( 17 CREATE TABLE IF NOT EXISTS thingsboard.msg_queue (
18 node_id timeuuid, 18 node_id timeuuid,
19 - clustered_hash bigint,  
20 - partition bigint, 19 + cluster_partition bigint,
  20 + ts_partition bigint,
21 ts bigint, 21 ts bigint,
22 msg blob, 22 msg blob,
23 - PRIMARY KEY ((node_id, clustered_hash, partition), ts)) 23 + PRIMARY KEY ((node_id, cluster_partition, ts_partition), ts))
24 WITH CLUSTERING ORDER BY (ts DESC) 24 WITH CLUSTERING ORDER BY (ts DESC)
25 AND compaction = { 25 AND compaction = {
26 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy', 26 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy',
@@ -33,10 +33,10 @@ AND compaction = { @@ -33,10 +33,10 @@ AND compaction = {
33 33
34 CREATE TABLE IF NOT EXISTS thingsboard.msg_ack_queue ( 34 CREATE TABLE IF NOT EXISTS thingsboard.msg_ack_queue (
35 node_id timeuuid, 35 node_id timeuuid,
36 - clustered_hash bigint,  
37 - partition bigint, 36 + cluster_partition bigint,
  37 + ts_partition bigint,
38 msg_id timeuuid, 38 msg_id timeuuid,
39 - PRIMARY KEY ((node_id, clustered_hash, partition), msg_id)) 39 + PRIMARY KEY ((node_id, cluster_partition, ts_partition), msg_id))
40 WITH CLUSTERING ORDER BY (msg_id DESC) 40 WITH CLUSTERING ORDER BY (msg_id DESC)
41 AND compaction = { 41 AND compaction = {
42 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy', 42 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy',
@@ -49,10 +49,10 @@ AND compaction = { @@ -49,10 +49,10 @@ AND compaction = {
49 49
50 CREATE TABLE IF NOT EXISTS thingsboard.processed_msg_partitions ( 50 CREATE TABLE IF NOT EXISTS thingsboard.processed_msg_partitions (
51 node_id timeuuid, 51 node_id timeuuid,
52 - clustered_hash bigint,  
53 - partition bigint,  
54 - PRIMARY KEY ((node_id, clustered_hash), partition))  
55 -WITH CLUSTERING ORDER BY (partition DESC) 52 + cluster_partition bigint,
  53 + ts_partition bigint,
  54 + PRIMARY KEY ((node_id, cluster_partition), ts_partition))
  55 +WITH CLUSTERING ORDER BY (ts_partition DESC)
56 AND compaction = { 56 AND compaction = {
57 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy', 57 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy',
58 'min_threshold': '5', 58 'min_threshold': '5',
@@ -182,6 +182,12 @@ cassandra: @@ -182,6 +182,12 @@ cassandra:
182 # Specify partitioning size for timestamp key-value storage. Example MINUTES, HOURS, DAYS, MONTHS 182 # Specify partitioning size for timestamp key-value storage. Example MINUTES, HOURS, DAYS, MONTHS
183 ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}" 183 ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}"
184 184
  185 + queue:
  186 + msg.ttl: 604800 # 7 days
  187 + ack.ttl: 604800 # 7 days
  188 + partitions.ttl: 604800 # 7 days
  189 + partitioning: "HOURS"
  190 +
185 # SQL configuration parameters 191 # SQL configuration parameters
186 sql: 192 sql:
187 # Specify executor service type used to perform timeseries insert tasks: SINGLE FIXED CACHED 193 # Specify executor service type used to perform timeseries insert tasks: SINGLE FIXED CACHED
@@ -57,6 +57,11 @@ @@ -57,6 +57,11 @@
57 <artifactId>logback-classic</artifactId> 57 <artifactId>logback-classic</artifactId>
58 </dependency> 58 </dependency>
59 <dependency> 59 <dependency>
  60 + <groupId>com.google.protobuf</groupId>
  61 + <artifactId>protobuf-java</artifactId>
  62 + <scope>provided</scope>
  63 + </dependency>
  64 + <dependency>
60 <groupId>junit</groupId> 65 <groupId>junit</groupId>
61 <artifactId>junit</artifactId> 66 <artifactId>junit</artifactId>
62 <scope>test</scope> 67 <scope>test</scope>
@@ -70,6 +75,10 @@ @@ -70,6 +75,10 @@
70 75
71 <build> 76 <build>
72 <plugins> 77 <plugins>
  78 + <plugin>
  79 + <groupId>org.xolstice.maven.plugins</groupId>
  80 + <artifactId>protobuf-maven-plugin</artifactId>
  81 + </plugin>
73 </plugins> 82 </plugins>
74 </build> 83 </build>
75 84
common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java renamed from rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbMsg.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,12 +13,17 @@ @@ -13,12 +13,17 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.api; 16 +package org.thingsboard.server.common.msg;
17 17
  18 +import com.google.protobuf.ByteString;
  19 +import com.google.protobuf.InvalidProtocolBufferException;
18 import lombok.Data; 20 import lombok.Data;
19 import org.thingsboard.server.common.data.id.EntityId; 21 import org.thingsboard.server.common.data.id.EntityId;
  22 +import org.thingsboard.server.common.data.id.EntityIdFactory;
  23 +import org.thingsboard.server.common.msg.gen.MsgProtos;
20 24
21 import java.io.Serializable; 25 import java.io.Serializable;
  26 +import java.nio.ByteBuffer;
22 import java.util.UUID; 27 import java.util.UUID;
23 28
24 /** 29 /**
@@ -34,4 +39,42 @@ public final class TbMsg implements Serializable { @@ -34,4 +39,42 @@ public final class TbMsg implements Serializable {
34 39
35 private final byte[] data; 40 private final byte[] data;
36 41
  42 + public static ByteBuffer toBytes(TbMsg msg) {
  43 + MsgProtos.TbMsgProto.Builder builder = MsgProtos.TbMsgProto.newBuilder();
  44 + builder.setId(msg.getId().toString());
  45 + builder.setType(msg.getType());
  46 + if (msg.getOriginator() != null) {
  47 + builder.setEntityType(msg.getOriginator().getEntityType().name());
  48 + builder.setEntityId(msg.getOriginator().getId().toString());
  49 + }
  50 +
  51 + if (msg.getMetaData() != null) {
  52 + MsgProtos.TbMsgProto.TbMsgMetaDataProto.Builder metadataBuilder = MsgProtos.TbMsgProto.TbMsgMetaDataProto.newBuilder();
  53 + metadataBuilder.putAllData(msg.getMetaData().getData());
  54 + builder.addMetaData(metadataBuilder.build());
  55 + }
  56 +
  57 + builder.setData(ByteString.copyFrom(msg.getData()));
  58 + byte[] bytes = builder.build().toByteArray();
  59 + return ByteBuffer.wrap(bytes);
  60 + }
  61 +
  62 + public static TbMsg fromBytes(ByteBuffer buffer) {
  63 + try {
  64 + MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(buffer.array());
  65 + TbMsgMetaData metaData = new TbMsgMetaData();
  66 + if (proto.getMetaDataCount() > 0) {
  67 + metaData.setData(proto.getMetaData(0).getDataMap());
  68 + }
  69 +
  70 + EntityId entityId = null;
  71 + if (proto.getEntityId() != null) {
  72 + entityId = EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId());
  73 + }
  74 +
  75 + return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, proto.getData().toByteArray());
  76 + } catch (InvalidProtocolBufferException e) {
  77 + throw new IllegalStateException("Could not parse protobuf for TbMsg", e);
  78 + }
  79 + }
37 } 80 }
common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgMetaData.java renamed from rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbMsgMetaData.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.api; 16 +package org.thingsboard.server.common.msg;
17 17
18 import lombok.Data; 18 import lombok.Data;
19 19
common/message/src/main/proto/tbmsg.proto renamed from rule-engine/rule-engine-components/src/main/proto/msgqueue.proto
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -16,8 +16,8 @@ @@ -16,8 +16,8 @@
16 syntax = "proto3"; 16 syntax = "proto3";
17 package msgqueue; 17 package msgqueue;
18 18
19 -option java_package = "org.thingsboard.rule.engine.queue.cassandra.repository.gen";  
20 -option java_outer_classname = "MsgQueueProtos"; 19 +option java_package = "org.thingsboard.server.common.msg.gen";
  20 +option java_outer_classname = "MsgProtos";
21 21
22 22
23 message TbMsgProto { 23 message TbMsgProto {
@@ -38,7 +38,11 @@ @@ -38,7 +38,11 @@
38 <dependency> 38 <dependency>
39 <groupId>org.thingsboard.common</groupId> 39 <groupId>org.thingsboard.common</groupId>
40 <artifactId>data</artifactId> 40 <artifactId>data</artifactId>
41 - </dependency> 41 + </dependency>
  42 + <dependency>
  43 + <groupId>org.thingsboard.common</groupId>
  44 + <artifactId>message</artifactId>
  45 + </dependency>
42 <dependency> 46 <dependency>
43 <groupId>org.slf4j</groupId> 47 <groupId>org.slf4j</groupId>
44 <artifactId>slf4j-api</artifactId> 48 <artifactId>slf4j-api</artifactId>
@@ -141,6 +145,10 @@ @@ -141,6 +145,10 @@
141 <scope>test</scope> 145 <scope>test</scope>
142 </dependency> 146 </dependency>
143 <dependency> 147 <dependency>
  148 + <groupId>com.google.protobuf</groupId>
  149 + <artifactId>protobuf-java</artifactId>
  150 + </dependency>
  151 + <dependency>
144 <groupId>org.apache.curator</groupId> 152 <groupId>org.apache.curator</groupId>
145 <artifactId>curator-x-discovery</artifactId> 153 <artifactId>curator-x-discovery</artifactId>
146 </dependency> 154 </dependency>
@@ -198,6 +206,11 @@ @@ -198,6 +206,11 @@
198 <groupId>org.elasticsearch.client</groupId> 206 <groupId>org.elasticsearch.client</groupId>
199 <artifactId>rest</artifactId> 207 <artifactId>rest</artifactId>
200 </dependency> 208 </dependency>
  209 + <dependency>
  210 + <groupId>org.junit.jupiter</groupId>
  211 + <artifactId>junit-jupiter-api</artifactId>
  212 + <version>RELEASE</version>
  213 + </dependency>
201 </dependencies> 214 </dependencies>
202 <build> 215 <build>
203 <plugins> 216 <plugins>
@@ -22,12 +22,21 @@ import org.springframework.beans.factory.annotation.Autowired; @@ -22,12 +22,21 @@ import org.springframework.beans.factory.annotation.Autowired;
22 import org.thingsboard.server.dao.cassandra.CassandraCluster; 22 import org.thingsboard.server.dao.cassandra.CassandraCluster;
23 import org.thingsboard.server.dao.model.type.*; 23 import org.thingsboard.server.dao.model.type.*;
24 24
  25 +import java.util.concurrent.ConcurrentHashMap;
  26 +import java.util.concurrent.ConcurrentMap;
  27 +
25 @Slf4j 28 @Slf4j
26 public abstract class CassandraAbstractDao { 29 public abstract class CassandraAbstractDao {
27 30
28 @Autowired 31 @Autowired
29 protected CassandraCluster cluster; 32 protected CassandraCluster cluster;
30 33
  34 + private ConcurrentMap<String, PreparedStatement> preparedStatementMap = new ConcurrentHashMap<>();
  35 +
  36 + protected PreparedStatement prepare(String query) {
  37 + return preparedStatementMap.computeIfAbsent(query, i -> getSession().prepare(i));
  38 + }
  39 +
31 private Session session; 40 private Session session;
32 41
33 private ConsistencyLevel defaultReadLevel; 42 private ConsistencyLevel defaultReadLevel;
@@ -15,6 +15,7 @@ @@ -15,6 +15,7 @@
15 */ 15 */
16 package org.thingsboard.server.dao.nosql; 16 package org.thingsboard.server.dao.nosql;
17 17
  18 +import com.datastax.driver.core.PreparedStatement;
18 import com.datastax.driver.core.ResultSet; 19 import com.datastax.driver.core.ResultSet;
19 import com.datastax.driver.core.ResultSetFuture; 20 import com.datastax.driver.core.ResultSetFuture;
20 import com.datastax.driver.core.Statement; 21 import com.datastax.driver.core.Statement;
@@ -37,6 +38,8 @@ import javax.annotation.Nullable; @@ -37,6 +38,8 @@ import javax.annotation.Nullable;
37 import java.util.Collections; 38 import java.util.Collections;
38 import java.util.List; 39 import java.util.List;
39 import java.util.UUID; 40 import java.util.UUID;
  41 +import java.util.concurrent.ConcurrentHashMap;
  42 +import java.util.concurrent.ConcurrentMap;
40 43
41 import static com.datastax.driver.core.querybuilder.QueryBuilder.eq; 44 import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
42 import static com.datastax.driver.core.querybuilder.QueryBuilder.select; 45 import static com.datastax.driver.core.querybuilder.QueryBuilder.select;
dao/src/main/java/org/thingsboard/server/dao/queue/MsqQueue.java renamed from rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/MsqQueue.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,17 +13,18 @@ @@ -13,17 +13,18 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.api; 16 +package org.thingsboard.server.dao.queue;
17 17
18 import com.google.common.util.concurrent.ListenableFuture; 18 import com.google.common.util.concurrent.ListenableFuture;
  19 +import org.thingsboard.server.common.msg.TbMsg;
19 20
20 import java.util.UUID; 21 import java.util.UUID;
21 22
22 public interface MsqQueue { 23 public interface MsqQueue {
23 24
24 - ListenableFuture<Void> put(TbMsg msg, UUID nodeId, long clusteredHash); 25 + ListenableFuture<Void> put(TbMsg msg, UUID nodeId, long clusterPartition);
25 26
26 - ListenableFuture<Void> ack(TbMsg msg, UUID nodeId, long clusteredHash); 27 + ListenableFuture<Void> ack(TbMsg msg, UUID nodeId, long clusterPartition);
27 28
28 - Iterable<TbMsg> findUnprocessed(UUID nodeId, long clusteredHash); 29 + Iterable<TbMsg> findUnprocessed(UUID nodeId, long clusterPartition);
29 } 30 }
dao/src/main/java/org/thingsboard/server/dao/queue/QueueBenchmark.java renamed from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/tool/QueueBenchmark.java
1 -package org.thingsboard.rule.engine.tool; 1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.queue;
2 17
3 import com.datastax.driver.core.Cluster; 18 import com.datastax.driver.core.Cluster;
4 import com.datastax.driver.core.HostDistance; 19 import com.datastax.driver.core.HostDistance;
@@ -16,9 +31,8 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -16,9 +31,8 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
16 import org.springframework.boot.autoconfigure.SpringBootApplication; 31 import org.springframework.boot.autoconfigure.SpringBootApplication;
17 import org.springframework.context.annotation.Bean; 32 import org.springframework.context.annotation.Bean;
18 import org.springframework.context.annotation.ComponentScan; 33 import org.springframework.context.annotation.ComponentScan;
19 -import org.thingsboard.rule.engine.api.MsqQueue;  
20 -import org.thingsboard.rule.engine.api.TbMsg;  
21 -import org.thingsboard.rule.engine.api.TbMsgMetaData; 34 +import org.thingsboard.server.common.msg.TbMsg;
  35 +import org.thingsboard.server.common.msg.TbMsgMetaData;
22 36
23 import javax.annotation.Nullable; 37 import javax.annotation.Nullable;
24 import java.net.InetSocketAddress; 38 import java.net.InetSocketAddress;
@@ -29,9 +43,9 @@ import java.util.concurrent.Executors; @@ -29,9 +43,9 @@ import java.util.concurrent.Executors;
29 import java.util.concurrent.TimeUnit; 43 import java.util.concurrent.TimeUnit;
30 import java.util.concurrent.atomic.AtomicLong; 44 import java.util.concurrent.atomic.AtomicLong;
31 45
32 -@SpringBootApplication  
33 -@EnableAutoConfiguration  
34 -@ComponentScan({"org.thingsboard.rule.engine"}) 46 +//@SpringBootApplication
  47 +//@EnableAutoConfiguration
  48 +//@ComponentScan({"org.thingsboard.rule.engine"})
35 //@PropertySource("classpath:processing-pipeline.properties") 49 //@PropertySource("classpath:processing-pipeline.properties")
36 @Slf4j 50 @Slf4j
37 public class QueueBenchmark implements CommandLineRunner { 51 public class QueueBenchmark implements CommandLineRunner {
dao/src/main/java/org/thingsboard/server/dao/queue/jpa/SqlMsgQueue.java renamed from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/queue/jpa/SqlMsgQueue.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.jpa; 16 +package org.thingsboard.server.dao.queue.jpa;
17 17
18 //@todo-vp: implement 18 //@todo-vp: implement
19 public class SqlMsgQueue { 19 public class SqlMsgQueue {
dao/src/main/java/org/thingsboard/server/dao/service/queue/cassandra/CassandraMsqQueue.java renamed from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/queue/cassandra/CassandraMsqQueue.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,60 +13,57 @@ @@ -13,60 +13,57 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra; 16 +package org.thingsboard.server.dao.service.queue.cassandra;
17 17
18 import com.datastax.driver.core.utils.UUIDs; 18 import com.datastax.driver.core.utils.UUIDs;
19 import com.google.common.collect.Lists; 19 import com.google.common.collect.Lists;
20 import com.google.common.util.concurrent.ListenableFuture; 20 import com.google.common.util.concurrent.ListenableFuture;
21 import lombok.extern.slf4j.Slf4j; 21 import lombok.extern.slf4j.Slf4j;
  22 +import org.springframework.beans.factory.annotation.Autowired;
22 import org.springframework.stereotype.Component; 23 import org.springframework.stereotype.Component;
23 -import org.thingsboard.rule.engine.api.MsqQueue;  
24 -import org.thingsboard.rule.engine.api.TbMsg;  
25 -import org.thingsboard.rule.engine.queue.cassandra.repository.AckRepository;  
26 -import org.thingsboard.rule.engine.queue.cassandra.repository.MsgRepository;  
27 -import org.thingsboard.server.common.data.UUIDConverter; 24 +import org.thingsboard.server.common.msg.TbMsg;
  25 +import org.thingsboard.server.dao.queue.MsqQueue;
  26 +import org.thingsboard.server.dao.service.queue.cassandra.repository.AckRepository;
  27 +import org.thingsboard.server.dao.service.queue.cassandra.repository.MsgRepository;
  28 +import org.thingsboard.server.dao.util.NoSqlDao;
28 29
29 import java.util.List; 30 import java.util.List;
30 import java.util.UUID; 31 import java.util.UUID;
31 32
32 @Component 33 @Component
33 @Slf4j 34 @Slf4j
  35 +@NoSqlDao
34 public class CassandraMsqQueue implements MsqQueue { 36 public class CassandraMsqQueue implements MsqQueue {
35 37
36 - private final MsgRepository msgRepository;  
37 - private final AckRepository ackRepository;  
38 - private final UnprocessedMsgFilter unprocessedMsgFilter;  
39 - private final QueuePartitioner queuePartitioner;  
40 -  
41 - public CassandraMsqQueue(MsgRepository msgRepository, AckRepository ackRepository,  
42 - UnprocessedMsgFilter unprocessedMsgFilter, QueuePartitioner queuePartitioner) {  
43 - this.msgRepository = msgRepository;  
44 - this.ackRepository = ackRepository;  
45 - this.unprocessedMsgFilter = unprocessedMsgFilter;  
46 - this.queuePartitioner = queuePartitioner;  
47 - }  
48 - 38 + @Autowired
  39 + private MsgRepository msgRepository;
  40 + @Autowired
  41 + private AckRepository ackRepository;
  42 + @Autowired
  43 + private UnprocessedMsgFilter unprocessedMsgFilter;
  44 + @Autowired
  45 + private QueuePartitioner queuePartitioner;
49 46
50 @Override 47 @Override
51 - public ListenableFuture<Void> put(TbMsg msg, UUID nodeId, long clusteredHash) { 48 + public ListenableFuture<Void> put(TbMsg msg, UUID nodeId, long clusterPartition) {
52 long msgTime = getMsgTime(msg); 49 long msgTime = getMsgTime(msg);
53 - long partition = queuePartitioner.getPartition(msgTime);  
54 - return msgRepository.save(msg, nodeId, clusteredHash, partition, msgTime); 50 + long tsPartition = queuePartitioner.getPartition(msgTime);
  51 + return msgRepository.save(msg, nodeId, clusterPartition, tsPartition, msgTime);
55 } 52 }
56 53
57 @Override 54 @Override
58 - public ListenableFuture<Void> ack(TbMsg msg, UUID nodeId, long clusteredHash) {  
59 - long partition = queuePartitioner.getPartition(getMsgTime(msg));  
60 - MsgAck ack = new MsgAck(msg.getId(), nodeId, clusteredHash, partition); 55 + public ListenableFuture<Void> ack(TbMsg msg, UUID nodeId, long clusterPartition) {
  56 + long tsPartition = queuePartitioner.getPartition(getMsgTime(msg));
  57 + MsgAck ack = new MsgAck(msg.getId(), nodeId, clusterPartition, tsPartition);
61 return ackRepository.ack(ack); 58 return ackRepository.ack(ack);
62 } 59 }
63 60
64 @Override 61 @Override
65 - public Iterable<TbMsg> findUnprocessed(UUID nodeId, long clusteredHash) { 62 + public Iterable<TbMsg> findUnprocessed(UUID nodeId, long clusterPartition) {
66 List<TbMsg> unprocessedMsgs = Lists.newArrayList(); 63 List<TbMsg> unprocessedMsgs = Lists.newArrayList();
67 - for (Long partition : queuePartitioner.findUnprocessedPartitions(nodeId, clusteredHash)) {  
68 - List<TbMsg> msgs = msgRepository.findMsgs(nodeId, clusteredHash, partition);  
69 - List<MsgAck> acks = ackRepository.findAcks(nodeId, clusteredHash, partition); 64 + for (Long tsPartition : queuePartitioner.findUnprocessedPartitions(nodeId, clusterPartition)) {
  65 + List<TbMsg> msgs = msgRepository.findMsgs(nodeId, clusterPartition, tsPartition);
  66 + List<MsgAck> acks = ackRepository.findAcks(nodeId, clusterPartition, tsPartition);
70 unprocessedMsgs.addAll(unprocessedMsgFilter.filter(msgs, acks)); 67 unprocessedMsgs.addAll(unprocessedMsgFilter.filter(msgs, acks));
71 } 68 }
72 return unprocessedMsgs; 69 return unprocessedMsgs;
dao/src/main/java/org/thingsboard/server/dao/service/queue/cassandra/MsgAck.java renamed from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/queue/cassandra/MsgAck.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,13 +13,10 @@ @@ -13,13 +13,10 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra; 16 +package org.thingsboard.server.dao.service.queue.cassandra;
17 17
18 -import com.datastax.driver.core.utils.UUIDs;  
19 import lombok.Data; 18 import lombok.Data;
20 import lombok.EqualsAndHashCode; 19 import lombok.EqualsAndHashCode;
21 -import org.thingsboard.rule.engine.api.TbMsg;  
22 -import org.thingsboard.server.common.data.UUIDConverter;  
23 20
24 import java.util.UUID; 21 import java.util.UUID;
25 22
@@ -29,7 +26,7 @@ public class MsgAck { @@ -29,7 +26,7 @@ public class MsgAck {
29 26
30 private final UUID msgId; 27 private final UUID msgId;
31 private final UUID nodeId; 28 private final UUID nodeId;
32 - private final long clusteredHash;  
33 - private final long partition; 29 + private final long clusteredPartition;
  30 + private final long tsPartition;
34 31
35 } 32 }
dao/src/main/java/org/thingsboard/server/dao/service/queue/cassandra/QueuePartitioner.java renamed from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/queue/cassandra/QueuePartitioner.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors  
3 - * <p> 2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at 6 * You may obtain a copy of the License at
7 - * <p>  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - * <p> 7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
10 * Unless required by applicable law or agreed to in writing, software 10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra; 16 +package org.thingsboard.server.dao.service.queue.cassandra;
17 17
18 import com.google.common.collect.Lists; 18 import com.google.common.collect.Lists;
19 import lombok.extern.slf4j.Slf4j; 19 import lombok.extern.slf4j.Slf4j;
20 import org.springframework.beans.factory.annotation.Value; 20 import org.springframework.beans.factory.annotation.Value;
21 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
22 -import org.thingsboard.rule.engine.queue.cassandra.repository.ProcessedPartitionRepository; 22 +import org.thingsboard.server.dao.service.queue.cassandra.repository.ProcessedPartitionRepository;
23 import org.thingsboard.server.dao.timeseries.TsPartitionDate; 23 import org.thingsboard.server.dao.timeseries.TsPartitionDate;
  24 +import org.thingsboard.server.dao.util.NoSqlDao;
24 25
25 import java.time.Clock; 26 import java.time.Clock;
26 import java.time.Instant; 27 import java.time.Instant;
@@ -32,26 +33,27 @@ import java.util.UUID; @@ -32,26 +33,27 @@ import java.util.UUID;
32 33
33 @Component 34 @Component
34 @Slf4j 35 @Slf4j
  36 +@NoSqlDao
35 public class QueuePartitioner { 37 public class QueuePartitioner {
36 38
37 - private ProcessedPartitionRepository processedPartitionRepository;  
38 -  
39 private final TsPartitionDate tsFormat; 39 private final TsPartitionDate tsFormat;
  40 + private ProcessedPartitionRepository processedPartitionRepository;
40 private Clock clock = Clock.systemUTC(); 41 private Clock clock = Clock.systemUTC();
41 42
42 - public QueuePartitioner(@Value("${rule.queue.msg_partitioning}") String partitioning, 43 + public QueuePartitioner(@Value("${cassandra.queue.partitioning}") String partitioning,
43 ProcessedPartitionRepository processedPartitionRepository) { 44 ProcessedPartitionRepository processedPartitionRepository) {
44 this.processedPartitionRepository = processedPartitionRepository; 45 this.processedPartitionRepository = processedPartitionRepository;
45 Optional<TsPartitionDate> partition = TsPartitionDate.parse(partitioning); 46 Optional<TsPartitionDate> partition = TsPartitionDate.parse(partitioning);
46 if (partition.isPresent()) { 47 if (partition.isPresent()) {
47 tsFormat = partition.get(); 48 tsFormat = partition.get();
48 } else { 49 } else {
49 - log.warn("Incorrect configuration of partitioning {}", "MINUTES");  
50 - throw new RuntimeException("Failed to parse partitioning property: " + "MINUTES" + "!"); 50 + log.warn("Incorrect configuration of partitioning {}", partitioning);
  51 + throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!");
51 } 52 }
52 } 53 }
53 54
54 public long getPartition(long ts) { 55 public long getPartition(long ts) {
  56 + //TODO: use TsPartitionDate.truncateTo?
55 LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); 57 LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC);
56 return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli(); 58 return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli();
57 } 59 }
dao/src/main/java/org/thingsboard/server/dao/service/queue/cassandra/UnprocessedMsgFilter.java renamed from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/queue/cassandra/UnprocessedMsgFilter.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,10 +13,10 @@ @@ -13,10 +13,10 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra; 16 +package org.thingsboard.server.dao.service.queue.cassandra;
17 17
18 import org.springframework.stereotype.Component; 18 import org.springframework.stereotype.Component;
19 -import org.thingsboard.rule.engine.api.TbMsg; 19 +import org.thingsboard.server.common.msg.TbMsg;
20 20
21 import java.util.Collection; 21 import java.util.Collection;
22 import java.util.List; 22 import java.util.List;
dao/src/main/java/org/thingsboard/server/dao/service/queue/cassandra/repository/AckRepository.java renamed from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/queue/cassandra/repository/AckRepository.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,10 +13,10 @@ @@ -13,10 +13,10 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra.repository; 16 +package org.thingsboard.server.dao.service.queue.cassandra.repository;
17 17
18 import com.google.common.util.concurrent.ListenableFuture; 18 import com.google.common.util.concurrent.ListenableFuture;
19 -import org.thingsboard.rule.engine.queue.cassandra.MsgAck; 19 +import org.thingsboard.server.dao.service.queue.cassandra.MsgAck;
20 20
21 import java.util.List; 21 import java.util.List;
22 import java.util.UUID; 22 import java.util.UUID;
@@ -25,5 +25,5 @@ public interface AckRepository { @@ -25,5 +25,5 @@ public interface AckRepository {
25 25
26 ListenableFuture<Void> ack(MsgAck msgAck); 26 ListenableFuture<Void> ack(MsgAck msgAck);
27 27
28 - List<MsgAck> findAcks(UUID nodeId, long clusteredHash, long partition); 28 + List<MsgAck> findAcks(UUID nodeId, long clusterPartition, long tsPartition);
29 } 29 }
dao/src/main/java/org/thingsboard/server/dao/service/queue/cassandra/repository/MsgRepository.java renamed from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/queue/cassandra/repository/MsgRepository.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,18 +13,18 @@ @@ -13,18 +13,18 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra.repository; 16 +package org.thingsboard.server.dao.service.queue.cassandra.repository;
17 17
18 import com.google.common.util.concurrent.ListenableFuture; 18 import com.google.common.util.concurrent.ListenableFuture;
19 -import org.thingsboard.rule.engine.api.TbMsg; 19 +import org.thingsboard.server.common.msg.TbMsg;
20 20
21 import java.util.List; 21 import java.util.List;
22 import java.util.UUID; 22 import java.util.UUID;
23 23
24 public interface MsgRepository { 24 public interface MsgRepository {
25 25
26 - ListenableFuture<Void> save(TbMsg msg, UUID nodeId, long clusteredHash, long partition, long msgTs); 26 + ListenableFuture<Void> save(TbMsg msg, UUID nodeId, long clusterPartition, long tsPartition, long msgTs);
27 27
28 - List<TbMsg> findMsgs(UUID nodeId, long clusteredHash, long partition); 28 + List<TbMsg> findMsgs(UUID nodeId, long clusterPartition, long tsPartition);
29 29
30 } 30 }
dao/src/main/java/org/thingsboard/server/dao/service/queue/cassandra/repository/ProcessedPartitionRepository.java renamed from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/queue/cassandra/repository/ProcessedPartitionRepository.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra.repository; 16 +package org.thingsboard.server.dao.service.queue.cassandra.repository;
17 17
18 import com.google.common.util.concurrent.ListenableFuture; 18 import com.google.common.util.concurrent.ListenableFuture;
19 19
dao/src/main/java/org/thingsboard/server/dao/service/queue/cassandra/repository/impl/CassandraAckRepository.java renamed from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/queue/cassandra/repository/impl/CassandraAckRepository.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors  
3 - * <p> 2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at 6 * You may obtain a copy of the License at
7 - * <p>  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - * <p> 7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
10 * Unless required by applicable law or agreed to in writing, software 10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra.repository.impl; 16 +package org.thingsboard.server.dao.service.queue.cassandra.repository.impl;
17 17
18 import com.datastax.driver.core.*; 18 import com.datastax.driver.core.*;
19 import com.google.common.base.Function; 19 import com.google.common.base.Function;
20 import com.google.common.util.concurrent.Futures; 20 import com.google.common.util.concurrent.Futures;
21 import com.google.common.util.concurrent.ListenableFuture; 21 import com.google.common.util.concurrent.ListenableFuture;
  22 +import org.springframework.beans.factory.annotation.Value;
22 import org.springframework.stereotype.Component; 23 import org.springframework.stereotype.Component;
23 -import org.thingsboard.rule.engine.queue.cassandra.MsgAck;  
24 -import org.thingsboard.rule.engine.queue.cassandra.repository.AckRepository; 24 +import org.thingsboard.server.dao.nosql.CassandraAbstractDao;
  25 +import org.thingsboard.server.dao.service.queue.cassandra.MsgAck;
  26 +import org.thingsboard.server.dao.service.queue.cassandra.repository.AckRepository;
  27 +import org.thingsboard.server.dao.util.NoSqlDao;
25 28
26 import java.util.ArrayList; 29 import java.util.ArrayList;
27 import java.util.List; 30 import java.util.List;
28 import java.util.UUID; 31 import java.util.UUID;
29 32
30 @Component 33 @Component
31 -public class CassandraAckRepository extends SimpleAbstractCassandraDao implements AckRepository { 34 +@NoSqlDao
  35 +public class CassandraAckRepository extends CassandraAbstractDao implements AckRepository {
32 36
33 - private final int ackQueueTtl;  
34 -  
35 - public CassandraAckRepository(Session session, int ackQueueTtl) {  
36 - super(session);  
37 - this.ackQueueTtl = ackQueueTtl;  
38 - } 37 + @Value("${cassandra.queue.ack.ttl}")
  38 + private int ackQueueTtl;
39 39
40 @Override 40 @Override
41 public ListenableFuture<Void> ack(MsgAck msgAck) { 41 public ListenableFuture<Void> ack(MsgAck msgAck) {
42 - String insert = "INSERT INTO msg_ack_queue (node_id, clustered_hash, partition, msg_id) VALUES (?, ?, ?, ?) USING TTL ?"; 42 + String insert = "INSERT INTO msg_ack_queue (node_id, cluster_partition, ts_partition, msg_id) VALUES (?, ?, ?, ?) USING TTL ?";
43 PreparedStatement statement = prepare(insert); 43 PreparedStatement statement = prepare(insert);
44 - BoundStatement boundStatement = statement.bind(msgAck.getNodeId(), msgAck.getClusteredHash(),  
45 - msgAck.getPartition(), msgAck.getMsgId(), ackQueueTtl); 44 + BoundStatement boundStatement = statement.bind(msgAck.getNodeId(), msgAck.getClusteredPartition(),
  45 + msgAck.getTsPartition(), msgAck.getMsgId(), ackQueueTtl);
46 ResultSetFuture resultSetFuture = executeAsyncWrite(boundStatement); 46 ResultSetFuture resultSetFuture = executeAsyncWrite(boundStatement);
47 return Futures.transform(resultSetFuture, (Function<ResultSet, Void>) input -> null); 47 return Futures.transform(resultSetFuture, (Function<ResultSet, Void>) input -> null);
48 } 48 }
49 49
50 @Override 50 @Override
51 - public List<MsgAck> findAcks(UUID nodeId, long clusteredHash, long partition) { 51 + public List<MsgAck> findAcks(UUID nodeId, long clusterPartition, long tsPartition) {
52 String select = "SELECT msg_id FROM msg_ack_queue WHERE " + 52 String select = "SELECT msg_id FROM msg_ack_queue WHERE " +
53 - "node_id = ? AND clustered_hash = ? AND partition = ?"; 53 + "node_id = ? AND cluster_partition = ? AND ts_partition = ?";
54 PreparedStatement statement = prepare(select); 54 PreparedStatement statement = prepare(select);
55 - BoundStatement boundStatement = statement.bind(nodeId, clusteredHash, partition); 55 + BoundStatement boundStatement = statement.bind(nodeId, clusterPartition, tsPartition);
56 ResultSet rows = executeRead(boundStatement); 56 ResultSet rows = executeRead(boundStatement);
57 List<MsgAck> msgs = new ArrayList<>(); 57 List<MsgAck> msgs = new ArrayList<>();
58 for (Row row : rows) { 58 for (Row row : rows) {
59 - msgs.add(new MsgAck(row.getUUID("msg_id"), nodeId, clusteredHash, partition)); 59 + msgs.add(new MsgAck(row.getUUID("msg_id"), nodeId, clusterPartition, tsPartition));
60 } 60 }
61 return msgs; 61 return msgs;
62 } 62 }
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.service.queue.cassandra.repository.impl;
  17 +
  18 +import com.datastax.driver.core.*;
  19 +import com.google.common.base.Function;
  20 +import com.google.common.util.concurrent.Futures;
  21 +import com.google.common.util.concurrent.ListenableFuture;
  22 +import org.springframework.beans.factory.annotation.Value;
  23 +import org.springframework.stereotype.Component;
  24 +import org.thingsboard.server.common.msg.TbMsg;
  25 +import org.thingsboard.server.dao.nosql.CassandraAbstractDao;
  26 +import org.thingsboard.server.dao.service.queue.cassandra.repository.MsgRepository;
  27 +import org.thingsboard.server.dao.util.NoSqlDao;
  28 +
  29 +import java.util.ArrayList;
  30 +import java.util.List;
  31 +import java.util.UUID;
  32 +
  33 +@Component
  34 +@NoSqlDao
  35 +public class CassandraMsgRepository extends CassandraAbstractDao implements MsgRepository {
  36 +
  37 + @Value("${cassandra.queue.msg.ttl}")
  38 + private int msqQueueTtl;
  39 +
  40 + @Override
  41 + public ListenableFuture<Void> save(TbMsg msg, UUID nodeId, long clusterPartition, long tsPartition, long msgTs) {
  42 + String insert = "INSERT INTO msg_queue (node_id, cluster_partition, ts_partition, ts, msg) VALUES (?, ?, ?, ?, ?) USING TTL ?";
  43 + PreparedStatement statement = prepare(insert);
  44 + BoundStatement boundStatement = statement.bind(nodeId, clusterPartition, tsPartition, msgTs, TbMsg.toBytes(msg), msqQueueTtl);
  45 + ResultSetFuture resultSetFuture = executeAsyncWrite(boundStatement);
  46 + return Futures.transform(resultSetFuture, (Function<ResultSet, Void>) input -> null);
  47 + }
  48 +
  49 + @Override
  50 + public List<TbMsg> findMsgs(UUID nodeId, long clusterPartition, long tsPartition) {
  51 + String select = "SELECT node_id, cluster_partition, ts_partition, ts, msg FROM msg_queue WHERE " +
  52 + "node_id = ? AND cluster_partition = ? AND ts_partition = ?";
  53 + PreparedStatement statement = prepare(select);
  54 + BoundStatement boundStatement = statement.bind(nodeId, clusterPartition, tsPartition);
  55 + ResultSet rows = executeRead(boundStatement);
  56 + List<TbMsg> msgs = new ArrayList<>();
  57 + for (Row row : rows) {
  58 + msgs.add(TbMsg.fromBytes(row.getBytes("msg")));
  59 + }
  60 + return msgs;
  61 + }
  62 +
  63 +}
dao/src/main/java/org/thingsboard/server/dao/service/queue/cassandra/repository/impl/CassandraProcessedPartitionRepository.java renamed from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/queue/cassandra/repository/impl/CassandraProcessedPartitionRepository.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors  
3 - * <p> 2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at 6 * You may obtain a copy of the License at
7 - * <p>  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - * <p> 7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
10 * Unless required by applicable law or agreed to in writing, software 10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra.repository.impl; 16 +package org.thingsboard.server.dao.service.queue.cassandra.repository.impl;
17 17
18 import com.datastax.driver.core.*; 18 import com.datastax.driver.core.*;
19 import com.google.common.base.Function; 19 import com.google.common.base.Function;
20 import com.google.common.util.concurrent.Futures; 20 import com.google.common.util.concurrent.Futures;
21 import com.google.common.util.concurrent.ListenableFuture; 21 import com.google.common.util.concurrent.ListenableFuture;
  22 +import org.springframework.beans.factory.annotation.Value;
22 import org.springframework.stereotype.Component; 23 import org.springframework.stereotype.Component;
23 -import org.thingsboard.rule.engine.queue.cassandra.repository.ProcessedPartitionRepository; 24 +import org.thingsboard.server.dao.nosql.CassandraAbstractDao;
  25 +import org.thingsboard.server.dao.service.queue.cassandra.repository.ProcessedPartitionRepository;
  26 +import org.thingsboard.server.dao.util.NoSqlDao;
24 27
25 import java.util.Optional; 28 import java.util.Optional;
26 import java.util.UUID; 29 import java.util.UUID;
27 30
28 @Component 31 @Component
29 -public class CassandraProcessedPartitionRepository extends SimpleAbstractCassandraDao implements ProcessedPartitionRepository { 32 +@NoSqlDao
  33 +public class CassandraProcessedPartitionRepository extends CassandraAbstractDao implements ProcessedPartitionRepository {
30 34
31 - private final int repositoryTtl;  
32 -  
33 - public CassandraProcessedPartitionRepository(Session session, int repositoryTtl) {  
34 - super(session);  
35 - this.repositoryTtl = repositoryTtl;  
36 - } 35 + @Value("${cassandra.queue.partitions.ttl}")
  36 + private int partitionsTtl;
37 37
38 @Override 38 @Override
39 - public ListenableFuture<Void> partitionProcessed(UUID nodeId, long clusteredHash, long partition) {  
40 - String insert = "INSERT INTO processed_msg_partitions (node_id, clustered_hash, partition) VALUES (?, ?, ?) USING TTL ?"; 39 + public ListenableFuture<Void> partitionProcessed(UUID nodeId, long clusterPartition, long tsPartition) {
  40 + String insert = "INSERT INTO processed_msg_partitions (node_id, cluster_partition, ts_partition) VALUES (?, ?, ?) USING TTL ?";
41 PreparedStatement prepared = prepare(insert); 41 PreparedStatement prepared = prepare(insert);
42 - BoundStatement boundStatement = prepared.bind(nodeId, clusteredHash, partition, repositoryTtl); 42 + BoundStatement boundStatement = prepared.bind(nodeId, clusterPartition, tsPartition, partitionsTtl);
43 ResultSetFuture resultSetFuture = executeAsyncWrite(boundStatement); 43 ResultSetFuture resultSetFuture = executeAsyncWrite(boundStatement);
44 return Futures.transform(resultSetFuture, (Function<ResultSet, Void>) input -> null); 44 return Futures.transform(resultSetFuture, (Function<ResultSet, Void>) input -> null);
45 } 45 }
46 46
47 @Override 47 @Override
48 public Optional<Long> findLastProcessedPartition(UUID nodeId, long clusteredHash) { 48 public Optional<Long> findLastProcessedPartition(UUID nodeId, long clusteredHash) {
49 - String select = "SELECT partition FROM processed_msg_partitions WHERE " +  
50 - "node_id = ? AND clustered_hash = ?"; 49 + String select = "SELECT ts_partition FROM processed_msg_partitions WHERE " +
  50 + "node_id = ? AND cluster_partition = ?";
51 PreparedStatement prepared = prepare(select); 51 PreparedStatement prepared = prepare(select);
52 BoundStatement boundStatement = prepared.bind(nodeId, clusteredHash); 52 BoundStatement boundStatement = prepared.bind(nodeId, clusteredHash);
53 Row row = executeRead(boundStatement).one(); 53 Row row = executeRead(boundStatement).one();
@@ -55,6 +55,6 @@ public class CassandraProcessedPartitionRepository extends SimpleAbstractCassand @@ -55,6 +55,6 @@ public class CassandraProcessedPartitionRepository extends SimpleAbstractCassand
55 return Optional.empty(); 55 return Optional.empty();
56 } 56 }
57 57
58 - return Optional.of(row.getLong("partition")); 58 + return Optional.of(row.getLong("ts_partition"));
59 } 59 }
60 } 60 }
@@ -616,11 +616,11 @@ AND compaction = { 'class' : 'LeveledCompactionStrategy' }; @@ -616,11 +616,11 @@ AND compaction = { 'class' : 'LeveledCompactionStrategy' };
616 616
617 CREATE TABLE IF NOT EXISTS thingsboard.msg_queue ( 617 CREATE TABLE IF NOT EXISTS thingsboard.msg_queue (
618 node_id timeuuid, 618 node_id timeuuid,
619 - clustered_hash bigint,  
620 - partition bigint, 619 + cluster_partition bigint,
  620 + ts_partition bigint,
621 ts bigint, 621 ts bigint,
622 msg blob, 622 msg blob,
623 - PRIMARY KEY ((node_id, clustered_hash, partition), ts)) 623 + PRIMARY KEY ((node_id, cluster_partition, ts_partition), ts))
624 WITH CLUSTERING ORDER BY (ts DESC) 624 WITH CLUSTERING ORDER BY (ts DESC)
625 AND compaction = { 625 AND compaction = {
626 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy', 626 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy',
@@ -633,10 +633,10 @@ AND compaction = { @@ -633,10 +633,10 @@ AND compaction = {
633 633
634 CREATE TABLE IF NOT EXISTS thingsboard.msg_ack_queue ( 634 CREATE TABLE IF NOT EXISTS thingsboard.msg_ack_queue (
635 node_id timeuuid, 635 node_id timeuuid,
636 - clustered_hash bigint,  
637 - partition bigint, 636 + cluster_partition bigint,
  637 + ts_partition bigint,
638 msg_id timeuuid, 638 msg_id timeuuid,
639 - PRIMARY KEY ((node_id, clustered_hash, partition), msg_id)) 639 + PRIMARY KEY ((node_id, cluster_partition, ts_partition), msg_id))
640 WITH CLUSTERING ORDER BY (msg_id DESC) 640 WITH CLUSTERING ORDER BY (msg_id DESC)
641 AND compaction = { 641 AND compaction = {
642 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy', 642 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy',
@@ -649,10 +649,10 @@ AND compaction = { @@ -649,10 +649,10 @@ AND compaction = {
649 649
650 CREATE TABLE IF NOT EXISTS thingsboard.processed_msg_partitions ( 650 CREATE TABLE IF NOT EXISTS thingsboard.processed_msg_partitions (
651 node_id timeuuid, 651 node_id timeuuid,
652 - clustered_hash bigint,  
653 - partition bigint,  
654 - PRIMARY KEY ((node_id, clustered_hash), partition))  
655 -WITH CLUSTERING ORDER BY (partition DESC) 652 + cluster_partition bigint,
  653 + ts_partition bigint,
  654 + PRIMARY KEY ((node_id, cluster_partition), ts_partition))
  655 +WITH CLUSTERING ORDER BY (ts_partition DESC)
656 AND compaction = { 656 AND compaction = {
657 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy', 657 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy',
658 'min_threshold': '5', 658 'min_threshold': '5',
@@ -26,7 +26,9 @@ import java.util.Arrays; @@ -26,7 +26,9 @@ import java.util.Arrays;
26 26
27 @RunWith(ClasspathSuite.class) 27 @RunWith(ClasspathSuite.class)
28 @ClassnameFilters({ 28 @ClassnameFilters({
29 - "org.thingsboard.server.dao.service.*ServiceNoSqlTest" 29 + "org.thingsboard.server.dao.service.*ServiceNoSqlTest",
  30 + "org.thingsboard.server.dao.service.queue.cassandra.*.*.*Test",
  31 + "org.thingsboard.server.dao.service.queue.cassandra.*Test"
30 }) 32 })
31 public class NoSqlDaoServiceTestSuite { 33 public class NoSqlDaoServiceTestSuite {
32 34
dao/src/test/java/org/thingsboard/server/dao/service/queue/cassandra/QueuePartitionerTest.java renamed from rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/queue/cassandra/QueuePartitionerTest.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors  
3 - * <p> 2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at 6 * You may obtain a copy of the License at
7 - * <p>  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - * <p> 7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
10 * Unless required by applicable law or agreed to in writing, software 10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra; 16 +package org.thingsboard.server.dao.service.queue.cassandra;
17 17
18 18
19 import org.junit.Before; 19 import org.junit.Before;
@@ -21,7 +21,7 @@ import org.junit.Test; @@ -21,7 +21,7 @@ import org.junit.Test;
21 import org.junit.runner.RunWith; 21 import org.junit.runner.RunWith;
22 import org.mockito.Mock; 22 import org.mockito.Mock;
23 import org.mockito.runners.MockitoJUnitRunner; 23 import org.mockito.runners.MockitoJUnitRunner;
24 -import org.thingsboard.rule.engine.queue.cassandra.repository.ProcessedPartitionRepository; 24 +import org.thingsboard.server.dao.service.queue.cassandra.repository.ProcessedPartitionRepository;
25 25
26 import java.time.Clock; 26 import java.time.Clock;
27 import java.time.Instant; 27 import java.time.Instant;
dao/src/test/java/org/thingsboard/server/dao/service/queue/cassandra/UnprocessedMsgFilterTest.java renamed from rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/queue/cassandra/UnprocessedMsgFilterTest.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,11 +13,11 @@ @@ -13,11 +13,11 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra; 16 +package org.thingsboard.server.dao.service.queue.cassandra;
17 17
18 import com.google.common.collect.Lists; 18 import com.google.common.collect.Lists;
19 import org.junit.Test; 19 import org.junit.Test;
20 -import org.thingsboard.rule.engine.api.TbMsg; 20 +import org.thingsboard.server.common.msg.TbMsg;
21 21
22 import java.util.Collection; 22 import java.util.Collection;
23 import java.util.List; 23 import java.util.List;
dao/src/test/java/org/thingsboard/server/dao/service/queue/cassandra/repository/impl/CassandraAckRepositoryTest.java renamed from rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/queue/cassandra/repository/impl/CassandraAckRepositoryTest.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,14 +13,17 @@ @@ -13,14 +13,17 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra.repository.impl; 16 +package org.thingsboard.server.dao.service.queue.cassandra.repository.impl;
17 17
18 import com.datastax.driver.core.utils.UUIDs; 18 import com.datastax.driver.core.utils.UUIDs;
19 import com.google.common.collect.Lists; 19 import com.google.common.collect.Lists;
20 import com.google.common.util.concurrent.ListenableFuture; 20 import com.google.common.util.concurrent.ListenableFuture;
21 import org.junit.Before; 21 import org.junit.Before;
22 import org.junit.Test; 22 import org.junit.Test;
23 -import org.thingsboard.rule.engine.queue.cassandra.MsgAck; 23 +import org.springframework.beans.factory.annotation.Autowired;
  24 +import org.thingsboard.server.dao.service.AbstractServiceTest;
  25 +import org.thingsboard.server.dao.service.DaoNoSqlTest;
  26 +import org.thingsboard.server.dao.service.queue.cassandra.MsgAck;
24 27
25 import java.util.List; 28 import java.util.List;
26 import java.util.UUID; 29 import java.util.UUID;
@@ -30,15 +33,12 @@ import java.util.concurrent.TimeUnit; @@ -30,15 +33,12 @@ import java.util.concurrent.TimeUnit;
30 import static org.junit.Assert.assertEquals; 33 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertTrue; 34 import static org.junit.Assert.assertTrue;
32 35
33 -public class CassandraAckRepositoryTest extends SimpleAbstractCassandraDaoTest { 36 +@DaoNoSqlTest
  37 +public class CassandraAckRepositoryTest extends AbstractServiceTest {
34 38
  39 + @Autowired
35 private CassandraAckRepository ackRepository; 40 private CassandraAckRepository ackRepository;
36 41
37 - @Before  
38 - public void init() {  
39 - ackRepository = new CassandraAckRepository(cassandraUnit.session, 1);  
40 - }  
41 -  
42 @Test 42 @Test
43 public void acksInPartitionCouldBeFound() { 43 public void acksInPartitionCouldBeFound() {
44 UUID nodeId = UUID.fromString("055eee50-1883-11e8-b380-65b5d5335ba9"); 44 UUID nodeId = UUID.fromString("055eee50-1883-11e8-b380-65b5d5335ba9");
dao/src/test/java/org/thingsboard/server/dao/service/queue/cassandra/repository/impl/CassandraMsgRepositoryTest.java renamed from rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/queue/cassandra/repository/impl/CassandraMsgRepositoryTest.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors  
3 - * <p> 2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at 6 * You may obtain a copy of the License at
7 - * <p>  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - * <p> 7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
10 * Unless required by applicable law or agreed to in writing, software 10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra.repository.impl; 16 +package org.thingsboard.server.dao.service.queue.cassandra.repository.impl;
17 17
18 //import static org.junit.jupiter.api.Assertions.*; 18 //import static org.junit.jupiter.api.Assertions.*;
19 19
@@ -21,9 +21,12 @@ import com.datastax.driver.core.utils.UUIDs; @@ -21,9 +21,12 @@ import com.datastax.driver.core.utils.UUIDs;
21 import com.google.common.util.concurrent.ListenableFuture; 21 import com.google.common.util.concurrent.ListenableFuture;
22 import org.junit.Before; 22 import org.junit.Before;
23 import org.junit.Test; 23 import org.junit.Test;
24 -import org.thingsboard.rule.engine.api.TbMsg;  
25 -import org.thingsboard.rule.engine.api.TbMsgMetaData; 24 +import org.springframework.beans.factory.annotation.Autowired;
26 import org.thingsboard.server.common.data.id.DeviceId; 25 import org.thingsboard.server.common.data.id.DeviceId;
  26 +import org.thingsboard.server.common.msg.TbMsg;
  27 +import org.thingsboard.server.common.msg.TbMsgMetaData;
  28 +import org.thingsboard.server.dao.service.AbstractServiceTest;
  29 +import org.thingsboard.server.dao.service.DaoNoSqlTest;
27 30
28 import java.util.List; 31 import java.util.List;
29 import java.util.UUID; 32 import java.util.UUID;
@@ -33,15 +36,12 @@ import java.util.concurrent.TimeUnit; @@ -33,15 +36,12 @@ import java.util.concurrent.TimeUnit;
33 import static org.junit.Assert.assertEquals; 36 import static org.junit.Assert.assertEquals;
34 import static org.junit.Assert.assertTrue; 37 import static org.junit.Assert.assertTrue;
35 38
36 -public class CassandraMsgRepositoryTest extends SimpleAbstractCassandraDaoTest { 39 +@DaoNoSqlTest
  40 +public class CassandraMsgRepositoryTest extends AbstractServiceTest {
37 41
  42 + @Autowired
38 private CassandraMsgRepository msgRepository; 43 private CassandraMsgRepository msgRepository;
39 44
40 - @Before  
41 - public void init() {  
42 - msgRepository = new CassandraMsgRepository(cassandraUnit.session, 1);  
43 - }  
44 -  
45 @Test 45 @Test
46 public void msgCanBeSavedAndRead() throws ExecutionException, InterruptedException { 46 public void msgCanBeSavedAndRead() throws ExecutionException, InterruptedException {
47 TbMsg msg = new TbMsg(UUIDs.timeBased(), "type", new DeviceId(UUIDs.timeBased()), null, new byte[4]); 47 TbMsg msg = new TbMsg(UUIDs.timeBased(), "type", new DeviceId(UUIDs.timeBased()), null, new byte[4]);
@@ -58,8 +58,6 @@ public class CassandraMsgRepositoryTest extends SimpleAbstractCassandraDaoTest { @@ -58,8 +58,6 @@ public class CassandraMsgRepositoryTest extends SimpleAbstractCassandraDaoTest {
58 UUID nodeId = UUIDs.timeBased(); 58 UUID nodeId = UUIDs.timeBased();
59 ListenableFuture<Void> future = msgRepository.save(msg, nodeId, 2L, 2L, 2L); 59 ListenableFuture<Void> future = msgRepository.save(msg, nodeId, 2L, 2L, 2L);
60 future.get(); 60 future.get();
61 - List<TbMsg> msgs = msgRepository.findMsgs(nodeId, 2L, 2L);  
62 - assertEquals(1, msgs.size());  
63 TimeUnit.SECONDS.sleep(2); 61 TimeUnit.SECONDS.sleep(2);
64 assertTrue(msgRepository.findMsgs(nodeId, 2L, 2L).isEmpty()); 62 assertTrue(msgRepository.findMsgs(nodeId, 2L, 2L).isEmpty());
65 } 63 }
dao/src/test/java/org/thingsboard/server/dao/service/queue/cassandra/repository/impl/CassandraProcessedPartitionRepositoryTest.java renamed from rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/queue/cassandra/repository/impl/CassandraProcessedPartitionRepositoryTest.java
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -13,13 +13,16 @@ @@ -13,13 +13,16 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.rule.engine.queue.cassandra.repository.impl; 16 +package org.thingsboard.server.dao.service.queue.cassandra.repository.impl;
17 17
18 import com.datastax.driver.core.utils.UUIDs; 18 import com.datastax.driver.core.utils.UUIDs;
19 import com.google.common.util.concurrent.Futures; 19 import com.google.common.util.concurrent.Futures;
20 import com.google.common.util.concurrent.ListenableFuture; 20 import com.google.common.util.concurrent.ListenableFuture;
21 import org.junit.Before; 21 import org.junit.Before;
22 import org.junit.Test; 22 import org.junit.Test;
  23 +import org.springframework.beans.factory.annotation.Autowired;
  24 +import org.thingsboard.server.dao.service.AbstractServiceTest;
  25 +import org.thingsboard.server.dao.service.DaoNoSqlTest;
23 26
24 import java.util.List; 27 import java.util.List;
25 import java.util.Optional; 28 import java.util.Optional;
@@ -29,15 +32,12 @@ import java.util.concurrent.TimeUnit; @@ -29,15 +32,12 @@ import java.util.concurrent.TimeUnit;
29 32
30 import static org.junit.Assert.*; 33 import static org.junit.Assert.*;
31 34
32 -public class CassandraProcessedPartitionRepositoryTest extends SimpleAbstractCassandraDaoTest { 35 +@DaoNoSqlTest
  36 +public class CassandraProcessedPartitionRepositoryTest extends AbstractServiceTest {
33 37
  38 + @Autowired
34 private CassandraProcessedPartitionRepository partitionRepository; 39 private CassandraProcessedPartitionRepository partitionRepository;
35 40
36 - @Before  
37 - public void init() {  
38 - partitionRepository = new CassandraProcessedPartitionRepository(cassandraUnit.session, 1);  
39 - }  
40 -  
41 @Test 41 @Test
42 public void lastProcessedPartitionCouldBeFound() { 42 public void lastProcessedPartitionCouldBeFound() {
43 UUID nodeId = UUID.fromString("055eee50-1883-11e8-b380-65b5d5335ba9"); 43 UUID nodeId = UUID.fromString("055eee50-1883-11e8-b380-65b5d5335ba9");
1 TRUNCATE thingsboard.plugin; 1 TRUNCATE thingsboard.plugin;
2 -TRUNCATE thingsboard.rule;  
  2 +TRUNCATE thingsboard.rule;
  3 +
  4 +-- msg_queue dataset
  5 +
  6 +INSERT INTO thingsboard.msg_queue (node_id, cluster_partition, ts_partition, ts, msg)
  7 + VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 200, 201, null);
  8 +INSERT INTO thingsboard.msg_queue (node_id, cluster_partition, ts_partition, ts, msg)
  9 + VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 200, 202, null);
  10 +INSERT INTO thingsboard.msg_queue (node_id, cluster_partition, ts_partition, ts, msg)
  11 + VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 300, 301, null);
  12 +
  13 +-- ack_queue dataset
  14 +INSERT INTO thingsboard.msg_ack_queue (node_id, cluster_partition, ts_partition, msg_id)
  15 + VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 300, bebaeb60-1888-11e8-bf21-65b5d5335ba9);
  16 +INSERT INTO thingsboard.msg_ack_queue (node_id, cluster_partition, ts_partition, msg_id)
  17 + VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 300, 12baeb60-1888-11e8-bf21-65b5d5335ba9);
  18 +INSERT INTO thingsboard.msg_ack_queue (node_id, cluster_partition, ts_partition, msg_id)
  19 + VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 200, 32baeb60-1888-11e8-bf21-65b5d5335ba9);
  20 +
  21 +-- processed partition dataset
  22 +INSERT INTO thingsboard.processed_msg_partitions (node_id, cluster_partition, ts_partition)
  23 + VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 100);
  24 +INSERT INTO thingsboard.processed_msg_partitions (node_id, cluster_partition, ts_partition)
  25 + VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 777);
  26 +INSERT INTO thingsboard.processed_msg_partitions (node_id, cluster_partition, ts_partition)
  27 + VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 202, 200);
1 -database.type=cassandra  
  1 +database.type=cassandra
  2 +
  3 +cassandra.queue.partitioning=HOURS
  4 +cassandra.queue.ack.ttl=1
  5 +cassandra.queue.msg.ttl=1
  6 +cassandra.queue.partitions.ttl=1
@@ -35,7 +35,7 @@ @@ -35,7 +35,7 @@
35 <spring-data-redis.version>1.8.10.RELEASE</spring-data-redis.version> 35 <spring-data-redis.version>1.8.10.RELEASE</spring-data-redis.version>
36 <jedis.version>2.9.0</jedis.version> 36 <jedis.version>2.9.0</jedis.version>
37 <jjwt.version>0.7.0</jjwt.version> 37 <jjwt.version>0.7.0</jjwt.version>
38 - <json-path.version>2.2.0</json-path.version> 38 + <json-path.version>2.2.0</json-path.version>
39 <junit.version>4.12</junit.version> 39 <junit.version>4.12</junit.version>
40 <slf4j.version>1.7.7</slf4j.version> 40 <slf4j.version>1.7.7</slf4j.version>
41 <logback.version>1.2.3</logback.version> 41 <logback.version>1.2.3</logback.version>
@@ -79,17 +79,19 @@ @@ -79,17 +79,19 @@
79 <dbunit.version>2.5.3</dbunit.version> 79 <dbunit.version>2.5.3</dbunit.version>
80 <spring-test-dbunit.version>1.2.1</spring-test-dbunit.version> 80 <spring-test-dbunit.version>1.2.1</spring-test-dbunit.version>
81 <postgresql.driver.version>9.4.1211</postgresql.driver.version> 81 <postgresql.driver.version>9.4.1211</postgresql.driver.version>
82 - <sonar.exclusions>org/thingsboard/server/gen/**/*, org/thingsboard/server/extensions/core/plugin/telemetry/gen/**/*</sonar.exclusions> 82 + <sonar.exclusions>org/thingsboard/server/gen/**/*,
  83 + org/thingsboard/server/extensions/core/plugin/telemetry/gen/**/*
  84 + </sonar.exclusions>
83 <elasticsearch.version>5.0.2</elasticsearch.version> 85 <elasticsearch.version>5.0.2</elasticsearch.version>
84 </properties> 86 </properties>
85 87
86 <modules> 88 <modules>
87 <module>common</module> 89 <module>common</module>
  90 + <module>rule-engine</module>
88 <module>dao</module> 91 <module>dao</module>
89 <module>extensions-api</module> 92 <module>extensions-api</module>
90 <module>extensions-core</module> 93 <module>extensions-core</module>
91 <module>extensions</module> 94 <module>extensions</module>
92 - <module>rule-engine</module>  
93 <module>transport</module> 95 <module>transport</module>
94 <module>ui</module> 96 <module>ui</module>
95 <module>tools</module> 97 <module>tools</module>
@@ -372,6 +374,11 @@ @@ -372,6 +374,11 @@
372 <version>${project.version}</version> 374 <version>${project.version}</version>
373 </dependency> 375 </dependency>
374 <dependency> 376 <dependency>
  377 + <groupId>org.thingsboard.rule-engine</groupId>
  378 + <artifactId>rule-engine-api</artifactId>
  379 + <version>${project.version}</version>
  380 + </dependency>
  381 + <dependency>
375 <groupId>org.thingsboard.common</groupId> 382 <groupId>org.thingsboard.common</groupId>
376 <artifactId>message</artifactId> 383 <artifactId>message</artifactId>
377 <version>${project.version}</version> 384 <version>${project.version}</version>
1 <!-- 1 <!--
2 2
3 - Copyright © 2016-2017 The Thingsboard Authors 3 + Copyright © 2016-2018 The Thingsboard Authors
4 4
5 Licensed under the Apache License, Version 2.0 (the "License"); 5 Licensed under the Apache License, Version 2.0 (the "License");
6 you may not use this file except in compliance with the License. 6 you may not use this file except in compliance with the License.
@@ -20,10 +20,9 @@ @@ -20,10 +20,9 @@
20 <modelVersion>4.0.0</modelVersion> 20 <modelVersion>4.0.0</modelVersion>
21 <parent> 21 <parent>
22 <groupId>org.thingsboard</groupId> 22 <groupId>org.thingsboard</groupId>
23 - <version>1.4.0-SNAPSHOT</version> 23 + <version>1.4.1-SNAPSHOT</version>
24 <artifactId>thingsboard</artifactId> 24 <artifactId>thingsboard</artifactId>
25 </parent> 25 </parent>
26 - <groupId>org.thingsboard</groupId>  
27 <artifactId>rule-engine</artifactId> 26 <artifactId>rule-engine</artifactId>
28 <packaging>pom</packaging> 27 <packaging>pom</packaging>
29 28
1 <?xml version="1.0" encoding="UTF-8"?> 1 <?xml version="1.0" encoding="UTF-8"?>
2 <!-- 2 <!--
3 3
4 - Copyright © 2016-2017 The Thingsboard Authors 4 + Copyright © 2016-2018 The Thingsboard Authors
5 5
6 Licensed under the Apache License, Version 2.0 (the "License"); 6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License. 7 you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@ @@ -22,7 +22,7 @@
22 <modelVersion>4.0.0</modelVersion> 22 <modelVersion>4.0.0</modelVersion>
23 <parent> 23 <parent>
24 <groupId>org.thingsboard</groupId> 24 <groupId>org.thingsboard</groupId>
25 - <version>1.4.0-SNAPSHOT</version> 25 + <version>1.4.1-SNAPSHOT</version>
26 <artifactId>rule-engine</artifactId> 26 <artifactId>rule-engine</artifactId>
27 </parent> 27 </parent>
28 <groupId>org.thingsboard.rule-engine</groupId> 28 <groupId>org.thingsboard.rule-engine</groupId>
@@ -54,6 +54,11 @@ @@ -54,6 +54,11 @@
54 <scope>provided</scope> 54 <scope>provided</scope>
55 </dependency> 55 </dependency>
56 <dependency> 56 <dependency>
  57 + <groupId>com.google.guava</groupId>
  58 + <artifactId>guava</artifactId>
  59 + <scope>provided</scope>
  60 + </dependency>
  61 + <dependency>
57 <groupId>ch.qos.logback</groupId> 62 <groupId>ch.qos.logback</groupId>
58 <artifactId>logback-core</artifactId> 63 <artifactId>logback-core</artifactId>
59 <scope>provided</scope> 64 <scope>provided</scope>
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@ @@ -15,6 +15,7 @@
15 */ 15 */
16 package org.thingsboard.rule.engine.api; 16 package org.thingsboard.rule.engine.api;
17 17
  18 +import org.thingsboard.server.common.msg.TbMsg;
18 import org.thingsboard.server.common.msg.cluster.ServerAddress; 19 import org.thingsboard.server.common.msg.cluster.ServerAddress;
19 import org.thingsboard.server.dao.attributes.AttributesService; 20 import org.thingsboard.server.dao.attributes.AttributesService;
20 21
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -15,6 +15,8 @@ @@ -15,6 +15,8 @@
15 */ 15 */
16 package org.thingsboard.rule.engine.api; 16 package org.thingsboard.rule.engine.api;
17 17
  18 +import org.thingsboard.server.common.msg.TbMsg;
  19 +
18 import java.util.concurrent.ExecutionException; 20 import java.util.concurrent.ExecutionException;
19 21
20 /** 22 /**
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
1 <?xml version="1.0" encoding="UTF-8"?> 1 <?xml version="1.0" encoding="UTF-8"?>
2 <!-- 2 <!--
3 3
4 - Copyright © 2016-2017 The Thingsboard Authors 4 + Copyright © 2016-2018 The Thingsboard Authors
5 5
6 Licensed under the Apache License, Version 2.0 (the "License"); 6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License. 7 you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@ @@ -22,7 +22,7 @@
22 <modelVersion>4.0.0</modelVersion> 22 <modelVersion>4.0.0</modelVersion>
23 <parent> 23 <parent>
24 <groupId>org.thingsboard</groupId> 24 <groupId>org.thingsboard</groupId>
25 - <version>1.4.0-SNAPSHOT</version> 25 + <version>1.4.1-SNAPSHOT</version>
26 <artifactId>rule-engine</artifactId> 26 <artifactId>rule-engine</artifactId>
27 </parent> 27 </parent>
28 <groupId>org.thingsboard.rule-engine</groupId> 28 <groupId>org.thingsboard.rule-engine</groupId>
@@ -61,31 +61,12 @@ @@ -61,31 +61,12 @@
61 <dependency> 61 <dependency>
62 <groupId>org.thingsboard.rule-engine</groupId> 62 <groupId>org.thingsboard.rule-engine</groupId>
63 <artifactId>rule-engine-api</artifactId> 63 <artifactId>rule-engine-api</artifactId>
64 - <version>1.4.0-SNAPSHOT</version>  
65 - </dependency>  
66 - <dependency>  
67 - <groupId>com.google.protobuf</groupId>  
68 - <artifactId>protobuf-java</artifactId>  
69 - <scope>provided</scope>  
70 </dependency> 64 </dependency>
71 <dependency> 65 <dependency>
72 <groupId>com.google.guava</groupId> 66 <groupId>com.google.guava</groupId>
73 <artifactId>guava</artifactId> 67 <artifactId>guava</artifactId>
74 </dependency> 68 </dependency>
75 <dependency> 69 <dependency>
76 - <groupId>com.datastax.cassandra</groupId>  
77 - <artifactId>cassandra-driver-core</artifactId>  
78 - </dependency>  
79 - <dependency>  
80 - <groupId>com.datastax.cassandra</groupId>  
81 - <artifactId>cassandra-driver-mapping</artifactId>  
82 - </dependency>  
83 - <dependency>  
84 - <groupId>com.datastax.cassandra</groupId>  
85 - <artifactId>cassandra-driver-extras</artifactId>  
86 - </dependency>  
87 -  
88 - <dependency>  
89 <groupId>junit</groupId> 70 <groupId>junit</groupId>
90 <artifactId>junit</artifactId> 71 <artifactId>junit</artifactId>
91 <version>${junit.version}</version> 72 <version>${junit.version}</version>
@@ -127,22 +108,14 @@ @@ -127,22 +108,14 @@
127 <artifactId>maven-dependency-plugin</artifactId> 108 <artifactId>maven-dependency-plugin</artifactId>
128 </plugin> 109 </plugin>
129 <plugin> 110 <plugin>
130 - <groupId>org.xolstice.maven.plugins</groupId>  
131 - <artifactId>protobuf-maven-plugin</artifactId>  
132 - </plugin>  
133 - <plugin>  
134 <groupId>org.codehaus.mojo</groupId> 111 <groupId>org.codehaus.mojo</groupId>
135 <artifactId>build-helper-maven-plugin</artifactId> 112 <artifactId>build-helper-maven-plugin</artifactId>
136 </plugin> 113 </plugin>
137 -  
138 -  
139 -  
140 -  
141 <plugin> 114 <plugin>
142 <groupId>org.springframework.boot</groupId> 115 <groupId>org.springframework.boot</groupId>
143 <artifactId>spring-boot-maven-plugin</artifactId> 116 <artifactId>spring-boot-maven-plugin</artifactId>
144 <configuration> 117 <configuration>
145 - <mainClass>org.thingsboard.rule.engine.tool.QueueBenchmark</mainClass> 118 + <mainClass>org.thingsboard.server.dao.queue.QueueBenchmark</mainClass>
146 <classifier>boot</classifier> 119 <classifier>boot</classifier>
147 <layout>ZIP</layout> 120 <layout>ZIP</layout>
148 <executable>true</executable> 121 <executable>true</executable>
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -18,12 +18,7 @@ package org.thingsboard.rule.engine.filter; @@ -18,12 +18,7 @@ package org.thingsboard.rule.engine.filter;
18 import lombok.extern.slf4j.Slf4j; 18 import lombok.extern.slf4j.Slf4j;
19 import org.thingsboard.rule.engine.TbNodeUtils; 19 import org.thingsboard.rule.engine.TbNodeUtils;
20 import org.thingsboard.rule.engine.api.*; 20 import org.thingsboard.rule.engine.api.*;
21 -import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration;  
22 -import org.thingsboard.server.common.data.DataConstants;  
23 -import org.thingsboard.server.common.data.kv.AttributeKvEntry;  
24 -import org.thingsboard.server.dao.attributes.AttributesService;  
25 -  
26 -import java.util.List; 21 +import org.thingsboard.server.common.msg.TbMsg;
27 22
28 /** 23 /**
29 * Created by ashvayka on 19.01.18. 24 * Created by ashvayka on 19.01.18.
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import org.thingsboard.rule.engine.TbNodeUtils; @@ -20,6 +20,7 @@ import org.thingsboard.rule.engine.TbNodeUtils;
20 import org.thingsboard.rule.engine.api.*; 20 import org.thingsboard.rule.engine.api.*;
21 import org.thingsboard.server.common.data.DataConstants; 21 import org.thingsboard.server.common.data.DataConstants;
22 import org.thingsboard.server.common.data.kv.AttributeKvEntry; 22 import org.thingsboard.server.common.data.kv.AttributeKvEntry;
  23 +import org.thingsboard.server.common.msg.TbMsg;
23 import org.thingsboard.server.dao.attributes.AttributesService; 24 import org.thingsboard.server.dao.attributes.AttributesService;
24 25
25 import java.util.List; 26 import java.util.List;
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
1 -/**  
2 - * Copyright © 2016-2017 The Thingsboard Authors  
3 - * <p>  
4 - * Licensed under the Apache License, Version 2.0 (the "License");  
5 - * you may not use this file except in compliance with the License.  
6 - * You may obtain a copy of the License at  
7 - * <p>  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - * <p>  
10 - * Unless required by applicable law or agreed to in writing, software  
11 - * distributed under the License is distributed on an "AS IS" BASIS,  
12 - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
13 - * See the License for the specific language governing permissions and  
14 - * limitations under the License.  
15 - */  
16 -package org.thingsboard.rule.engine.queue.cassandra.repository.impl;  
17 -  
18 -import com.datastax.driver.core.*;  
19 -import com.google.common.base.Function;  
20 -import com.google.common.util.concurrent.Futures;  
21 -import com.google.common.util.concurrent.ListenableFuture;  
22 -import com.google.protobuf.ByteString;  
23 -import com.google.protobuf.InvalidProtocolBufferException;  
24 -import org.springframework.stereotype.Component;  
25 -import org.thingsboard.rule.engine.api.TbMsg;  
26 -import org.thingsboard.rule.engine.api.TbMsgMetaData;  
27 -import org.thingsboard.rule.engine.queue.cassandra.repository.MsgRepository;  
28 -import org.thingsboard.rule.engine.queue.cassandra.repository.gen.MsgQueueProtos;  
29 -import org.thingsboard.server.common.data.id.EntityId;  
30 -import org.thingsboard.server.common.data.id.EntityIdFactory;  
31 -  
32 -import java.nio.ByteBuffer;  
33 -import java.util.ArrayList;  
34 -import java.util.List;  
35 -import java.util.UUID;  
36 -  
37 -@Component  
38 -public class CassandraMsgRepository extends SimpleAbstractCassandraDao implements MsgRepository {  
39 -  
40 - private final int msqQueueTtl;  
41 -  
42 -  
43 - public CassandraMsgRepository(Session session, int msqQueueTtl) {  
44 - super(session);  
45 - this.msqQueueTtl = msqQueueTtl;  
46 - }  
47 -  
48 - @Override  
49 - public ListenableFuture<Void> save(TbMsg msg, UUID nodeId, long clusteredHash, long partition, long msgTs) {  
50 - String insert = "INSERT INTO msg_queue (node_id, clustered_hash, partition, ts, msg) VALUES (?, ?, ?, ?, ?) USING TTL ?";  
51 - PreparedStatement statement = prepare(insert);  
52 - BoundStatement boundStatement = statement.bind(nodeId, clusteredHash, partition, msgTs, toBytes(msg), msqQueueTtl);  
53 - ResultSetFuture resultSetFuture = executeAsyncWrite(boundStatement);  
54 - return Futures.transform(resultSetFuture, (Function<ResultSet, Void>) input -> null);  
55 - }  
56 -  
57 - @Override  
58 - public List<TbMsg> findMsgs(UUID nodeId, long clusteredHash, long partition) {  
59 - String select = "SELECT node_id, clustered_hash, partition, ts, msg FROM msg_queue WHERE " +  
60 - "node_id = ? AND clustered_hash = ? AND partition = ?";  
61 - PreparedStatement statement = prepare(select);  
62 - BoundStatement boundStatement = statement.bind(nodeId, clusteredHash, partition);  
63 - ResultSet rows = executeRead(boundStatement);  
64 - List<TbMsg> msgs = new ArrayList<>();  
65 - for (Row row : rows) {  
66 - msgs.add(fromBytes(row.getBytes("msg")));  
67 - }  
68 - return msgs;  
69 - }  
70 -  
71 - private ByteBuffer toBytes(TbMsg msg) {  
72 - MsgQueueProtos.TbMsgProto.Builder builder = MsgQueueProtos.TbMsgProto.newBuilder();  
73 - builder.setId(msg.getId().toString());  
74 - builder.setType(msg.getType());  
75 - if (msg.getOriginator() != null) {  
76 - builder.setEntityType(msg.getOriginator().getEntityType().name());  
77 - builder.setEntityId(msg.getOriginator().getId().toString());  
78 - }  
79 -  
80 - if (msg.getMetaData() != null) {  
81 - MsgQueueProtos.TbMsgProto.TbMsgMetaDataProto.Builder metadataBuilder = MsgQueueProtos.TbMsgProto.TbMsgMetaDataProto.newBuilder();  
82 - metadataBuilder.putAllData(msg.getMetaData().getData());  
83 - builder.addMetaData(metadataBuilder.build());  
84 - }  
85 -  
86 - builder.setData(ByteString.copyFrom(msg.getData()));  
87 - byte[] bytes = builder.build().toByteArray();  
88 - return ByteBuffer.wrap(bytes);  
89 - }  
90 -  
91 - private TbMsg fromBytes(ByteBuffer buffer) {  
92 - try {  
93 - MsgQueueProtos.TbMsgProto proto = MsgQueueProtos.TbMsgProto.parseFrom(buffer.array());  
94 - TbMsgMetaData metaData = new TbMsgMetaData();  
95 - if (proto.getMetaDataCount() > 0) {  
96 - metaData.setData(proto.getMetaData(0).getDataMap());  
97 - }  
98 -  
99 - EntityId entityId = null;  
100 - if (proto.getEntityId() != null) {  
101 - entityId = EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId());  
102 - }  
103 -  
104 - return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, proto.getData().toByteArray());  
105 - } catch (InvalidProtocolBufferException e) {  
106 - throw new IllegalStateException("Could not parse protobuf for TbMsg", e);  
107 - }  
108 - }  
109 -}  
1 -/**  
2 - * Copyright © 2016-2017 The Thingsboard Authors  
3 - * <p>  
4 - * Licensed under the Apache License, Version 2.0 (the "License");  
5 - * you may not use this file except in compliance with the License.  
6 - * You may obtain a copy of the License at  
7 - * <p>  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - * <p>  
10 - * Unless required by applicable law or agreed to in writing, software  
11 - * distributed under the License is distributed on an "AS IS" BASIS,  
12 - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
13 - * See the License for the specific language governing permissions and  
14 - * limitations under the License.  
15 - */  
16 -package org.thingsboard.rule.engine.queue.cassandra.repository.impl;  
17 -  
18 -import com.datastax.driver.core.*;  
19 -import lombok.extern.slf4j.Slf4j;  
20 -import org.springframework.stereotype.Component;  
21 -  
22 -import java.util.Map;  
23 -import java.util.concurrent.ConcurrentHashMap;  
24 -  
25 -@Component  
26 -@Slf4j  
27 -public abstract class SimpleAbstractCassandraDao {  
28 -  
29 - private ConsistencyLevel defaultReadLevel = ConsistencyLevel.QUORUM;  
30 - private ConsistencyLevel defaultWriteLevel = ConsistencyLevel.QUORUM;  
31 - private Session session;  
32 - private Map<String, PreparedStatement> preparedStatementMap = new ConcurrentHashMap<>();  
33 -  
34 - public SimpleAbstractCassandraDao(Session session) {  
35 - this.session = session;  
36 - }  
37 -  
38 - protected Session getSession() {  
39 - return session;  
40 - }  
41 -  
42 - protected ResultSet executeRead(Statement statement) {  
43 - return execute(statement, defaultReadLevel);  
44 - }  
45 -  
46 - protected ResultSet executeWrite(Statement statement) {  
47 - return execute(statement, defaultWriteLevel);  
48 - }  
49 -  
50 - protected ResultSetFuture executeAsyncRead(Statement statement) {  
51 - return executeAsync(statement, defaultReadLevel);  
52 - }  
53 -  
54 - protected ResultSetFuture executeAsyncWrite(Statement statement) {  
55 - return executeAsync(statement, defaultWriteLevel);  
56 - }  
57 -  
58 - protected PreparedStatement prepare(String query) {  
59 - return preparedStatementMap.computeIfAbsent(query, i -> getSession().prepare(i));  
60 - }  
61 -  
62 - private ResultSet execute(Statement statement, ConsistencyLevel level) {  
63 - log.debug("Execute cassandra statement {}", statement);  
64 - if (statement.getConsistencyLevel() == null) {  
65 - statement.setConsistencyLevel(level);  
66 - }  
67 - return getSession().execute(statement);  
68 - }  
69 -  
70 - private ResultSetFuture executeAsync(Statement statement, ConsistencyLevel level) {  
71 - log.debug("Execute cassandra async statement {}", statement);  
72 - if (statement.getConsistencyLevel() == null) {  
73 - statement.setConsistencyLevel(level);  
74 - }  
75 - return getSession().executeAsync(statement);  
76 - }  
77 -}  
1 /** 1 /**
2 - * Copyright © 2016-2017 The Thingsboard Authors 2 + * Copyright © 2016-2018 The Thingsboard Authors
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@ import org.thingsboard.rule.engine.api.*; @@ -21,6 +21,7 @@ import org.thingsboard.rule.engine.api.*;
21 import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; 21 import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration;
22 import org.thingsboard.server.common.data.DataConstants; 22 import org.thingsboard.server.common.data.DataConstants;
23 import org.thingsboard.server.common.data.kv.AttributeKvEntry; 23 import org.thingsboard.server.common.data.kv.AttributeKvEntry;
  24 +import org.thingsboard.server.common.msg.TbMsg;
24 import org.thingsboard.server.dao.attributes.AttributesService; 25 import org.thingsboard.server.dao.attributes.AttributesService;
25 26
26 import java.util.List; 27 import java.util.List;
1 -/**  
2 - * Copyright © 2016-2017 The Thingsboard Authors  
3 - *  
4 - * Licensed under the Apache License, Version 2.0 (the "License");  
5 - * you may not use this file except in compliance with the License.  
6 - * You may obtain a copy of the License at  
7 - *  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - *  
10 - * Unless required by applicable law or agreed to in writing, software  
11 - * distributed under the License is distributed on an "AS IS" BASIS,  
12 - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
13 - * See the License for the specific language governing permissions and  
14 - * limitations under the License.  
15 - */  
16 -package org.thingsboard.rule.engine.queue.cassandra;  
17 -  
18 -import org.junit.Before;  
19 -import org.junit.Test;  
20 -import org.mockito.Mock;  
21 -import org.thingsboard.rule.engine.queue.cassandra.repository.AckRepository;  
22 -import org.thingsboard.rule.engine.queue.cassandra.repository.MsgRepository;  
23 -  
24 -public class CassandraMsqQueueTest {  
25 -  
26 - private CassandraMsqQueue msqQueue;  
27 -  
28 - @Mock  
29 - private MsgRepository msgRepository;  
30 - @Mock  
31 - private AckRepository ackRepository;  
32 - @Mock  
33 - private UnprocessedMsgFilter unprocessedMsgFilter;  
34 - @Mock  
35 - private QueuePartitioner queuePartitioner;  
36 -  
37 - @Before  
38 - public void init() {  
39 - msqQueue = new CassandraMsqQueue(msgRepository, ackRepository, unprocessedMsgFilter, queuePartitioner);  
40 - }  
41 -  
42 - @Test  
43 - public void msgCanBeSaved() {  
44 -// todo-vp: implement  
45 - }  
46 -  
47 -  
48 -}  
1 -/**  
2 - * Copyright © 2016-2017 The Thingsboard Authors  
3 - *  
4 - * Licensed under the Apache License, Version 2.0 (the "License");  
5 - * you may not use this file except in compliance with the License.  
6 - * You may obtain a copy of the License at  
7 - *  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - *  
10 - * Unless required by applicable law or agreed to in writing, software  
11 - * distributed under the License is distributed on an "AS IS" BASIS,  
12 - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
13 - * See the License for the specific language governing permissions and  
14 - * limitations under the License.  
15 - */  
16 -package org.thingsboard.rule.engine.queue.cassandra.repository.impl;  
17 -  
18 -import org.cassandraunit.CassandraCQLUnit;  
19 -import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;  
20 -import org.junit.ClassRule;  
21 -  
22 -  
23 -public abstract class SimpleAbstractCassandraDaoTest {  
24 -  
25 - @ClassRule  
26 - public static CassandraCQLUnit cassandraUnit = new CassandraCQLUnit(  
27 - new ClassPathCQLDataSet("cassandra/system-test.cql", "thingsboard"));  
28 -  
29 -  
30 -}  
1 -CREATE TABLE IF NOT EXISTS thingsboard.msg_queue (  
2 - node_id timeuuid,  
3 - clustered_hash bigint,  
4 - partition bigint,  
5 - ts bigint,  
6 - msg blob,  
7 - PRIMARY KEY ((node_id, clustered_hash, partition), ts))  
8 -WITH CLUSTERING ORDER BY (ts DESC)  
9 -AND compaction = {  
10 - 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy',  
11 - 'min_threshold': '5',  
12 - 'base_time_seconds': '43200',  
13 - 'max_window_size_seconds': '43200',  
14 - 'tombstone_threshold': '0.9',  
15 - 'unchecked_tombstone_compaction': 'true'  
16 -};  
17 -  
18 -  
19 -CREATE TABLE IF NOT EXISTS thingsboard.msg_ack_queue (  
20 - node_id timeuuid,  
21 - clustered_hash bigint,  
22 - partition bigint,  
23 - msg_id timeuuid,  
24 - PRIMARY KEY ((node_id, clustered_hash, partition), msg_id))  
25 -WITH CLUSTERING ORDER BY (msg_id DESC)  
26 -AND compaction = {  
27 - 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy',  
28 - 'min_threshold': '5',  
29 - 'base_time_seconds': '43200',  
30 - 'max_window_size_seconds': '43200',  
31 - 'tombstone_threshold': '0.9',  
32 - 'unchecked_tombstone_compaction': 'true'  
33 -};  
34 -  
35 -CREATE TABLE IF NOT EXISTS thingsboard.processed_msg_partitions (  
36 - node_id timeuuid,  
37 - clustered_hash bigint,  
38 - partition bigint,  
39 - PRIMARY KEY ((node_id, clustered_hash), partition))  
40 -WITH CLUSTERING ORDER BY (partition DESC)  
41 -AND compaction = {  
42 - 'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy',  
43 - 'min_threshold': '5',  
44 - 'base_time_seconds': '43200',  
45 - 'max_window_size_seconds': '43200',  
46 - 'tombstone_threshold': '0.9',  
47 - 'unchecked_tombstone_compaction': 'true'  
48 -};  
49 -  
50 -  
51 -  
52 --- msg_queue dataset  
53 -  
54 -INSERT INTO thingsboard.msg_queue (node_id, clustered_hash, partition, ts, msg)  
55 - VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 200, 201, null);  
56 -INSERT INTO thingsboard.msg_queue (node_id, clustered_hash, partition, ts, msg)  
57 - VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 200, 202, null);  
58 -INSERT INTO thingsboard.msg_queue (node_id, clustered_hash, partition, ts, msg)  
59 - VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 300, 301, null);  
60 -  
61 --- ack_queue dataset  
62 -INSERT INTO msg_ack_queue (node_id, clustered_hash, partition, msg_id)  
63 - VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 300, bebaeb60-1888-11e8-bf21-65b5d5335ba9);  
64 -INSERT INTO msg_ack_queue (node_id, clustered_hash, partition, msg_id)  
65 - VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 300, 12baeb60-1888-11e8-bf21-65b5d5335ba9);  
66 - INSERT INTO msg_ack_queue (node_id, clustered_hash, partition, msg_id)  
67 - VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 200, 32baeb60-1888-11e8-bf21-65b5d5335ba9);  
68 -  
69 --- processed partition dataset  
70 -INSERT INTO processed_msg_partitions (node_id, clustered_hash, partition)  
71 - VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 100);  
72 -INSERT INTO processed_msg_partitions (node_id, clustered_hash, partition)  
73 - VALUES (055eee50-1883-11e8-b380-65b5d5335ba9, 101, 777);  
74 -INSERT INTO processed_msg_partitions (node_id, clustered_hash, partition)