简柏林 / thingskit · Commits

Commit f6bc0791f10dc20260ea04dec20429c93ed18840

Authored by Andrew Shvayka
1 parent 406438c8

Aggregation

Inline Side-by-side
Showing 15 changed files with 494 additions and 176 deletions
1 /** 1 /**
2 * Copyright © 2016-2017 The Thingsboard Authors 2 * Copyright © 2016-2017 The Thingsboard Authors
3 - * 3 + * <p>
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at 6 * You may obtain a copy of the License at
7 - *  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - * 7 + * <p>
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + * <p>
10 * Unless required by applicable law or agreed to in writing, software 10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,6 +17,7 @@ package org.thingsboard.server.actors.plugin; @@ -17,6 +17,7 @@ package org.thingsboard.server.actors.plugin;
17 17
18 import java.io.IOException; 18 import java.io.IOException;
19 import java.util.*; 19 import java.util.*;
  20 +import java.util.concurrent.ExecutionException;
20 import java.util.concurrent.Executor; 21 import java.util.concurrent.Executor;
21 import java.util.concurrent.Executors; 22 import java.util.concurrent.Executors;
22 import java.util.stream.Collectors; 23 import java.util.stream.Collectors;
@@ -152,7 +153,19 @@ public final class PluginProcessingContext implements PluginContext { @@ -152,7 +153,19 @@ public final class PluginProcessingContext implements PluginContext {
152 @Override 153 @Override
153 public List<TsKvEntry> loadTimeseries(DeviceId deviceId, TsKvQuery query) { 154 public List<TsKvEntry> loadTimeseries(DeviceId deviceId, TsKvQuery query) {
154 validate(deviceId); 155 validate(deviceId);
155 - return pluginCtx.tsService.find(DataConstants.DEVICE, deviceId, query); 156 + try {
  157 + return pluginCtx.tsService.findAll(DataConstants.DEVICE, deviceId, query).get();
  158 + } catch (Exception e) {
  159 + log.error("TODO", e);
  160 + throw new RuntimeException(e);
  161 + }
  162 + }
  163 +
  164 + @Override
  165 + public void loadTimeseries(DeviceId deviceId, TsKvQuery query, PluginCallback<List<TsKvEntry>> callback) {
  166 + validate(deviceId);
  167 + ListenableFuture<List<TsKvEntry>> future = pluginCtx.tsService.findAll(DataConstants.DEVICE, deviceId, query);
  168 + Futures.addCallback(future, getCallback(callback, v -> v), executor);
156 } 169 }
157 170
158 @Override 171 @Override
@@ -235,10 +248,10 @@ public final class PluginProcessingContext implements PluginContext { @@ -235,10 +248,10 @@ public final class PluginProcessingContext implements PluginContext {
235 }; 248 };
236 } 249 }
237 250
238 - private <T> FutureCallback<ResultSet> getCallback(final PluginCallback<T> callback, Function<ResultSet, T> transformer) {  
239 - return new FutureCallback<ResultSet>() { 251 + private <T, R> FutureCallback<R> getCallback(final PluginCallback<T> callback, Function<R, T> transformer) {
  252 + return new FutureCallback<R>() {
240 @Override 253 @Override
241 - public void onSuccess(@Nullable ResultSet result) { 254 + public void onSuccess(@Nullable R result) {
242 pluginCtx.self().tell(PluginCallbackMessage.onSuccess(callback, transformer.apply(result)), ActorRef.noSender()); 255 pluginCtx.self().tell(PluginCallbackMessage.onSuccess(callback, transformer.apply(result)), ActorRef.noSender());
243 } 256 }
244 257
  1 +package org.thingsboard.server.common.data.kv;
  2 +
  3 +/**
  4 + * Created by ashvayka on 20.02.17.
  5 + */
  6 +public enum Aggregation {
  7 +
  8 + MIN, MAX, AVG, SUM, COUNT, NONE;
  9 +
  10 +}
1 /** 1 /**
2 * Copyright © 2016-2017 The Thingsboard Authors 2 * Copyright © 2016-2017 The Thingsboard Authors
3 - * 3 + * <p>
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at 6 * You may obtain a copy of the License at
7 - *  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - * 7 + * <p>
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + * <p>
10 * Unless required by applicable law or agreed to in writing, software 10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,59 +15,27 @@ @@ -15,59 +15,27 @@
15 */ 15 */
16 package org.thingsboard.server.common.data.kv; 16 package org.thingsboard.server.common.data.kv;
17 17
18 -import java.util.Optional; 18 +import lombok.Data;
19 19
  20 +@Data
20 public class BaseTsKvQuery implements TsKvQuery { 21 public class BaseTsKvQuery implements TsKvQuery {
21 22
22 - private String key;  
23 - private Optional<Long> startTs;  
24 - private Optional<Long> endTs;  
25 - private Optional<Integer> limit; 23 + private final String key;
  24 + private final long startTs;
  25 + private final long endTs;
  26 + private final int limit;
  27 + private final Aggregation aggregation;
26 28
27 - public BaseTsKvQuery(String key, Optional<Long> startTs, Optional<Long> endTs, Optional<Integer> limit) { 29 + public BaseTsKvQuery(String key, long startTs, long endTs, int limit, Aggregation aggregation) {
28 this.key = key; 30 this.key = key;
29 this.startTs = startTs; 31 this.startTs = startTs;
30 this.endTs = endTs; 32 this.endTs = endTs;
31 this.limit = limit; 33 this.limit = limit;
32 - }  
33 -  
34 - public BaseTsKvQuery(String key, Long startTs, Long endTs, Integer limit) {  
35 - this(key, Optional.ofNullable(startTs), Optional.ofNullable(endTs), Optional.ofNullable(limit));  
36 - }  
37 -  
38 - public BaseTsKvQuery(String key, Long startTs, Integer limit) {  
39 - this(key, startTs, null, limit);  
40 - }  
41 -  
42 - public BaseTsKvQuery(String key, Long startTs, Long endTs) {  
43 - this(key, startTs, endTs, null);  
44 - }  
45 -  
46 - public BaseTsKvQuery(String key, Long startTs) {  
47 - this(key, startTs, null, null); 34 + this.aggregation = aggregation;
48 } 35 }
49 36
50 - public BaseTsKvQuery(String key, Integer limit) {  
51 - this(key, null, null, limit); 37 + public BaseTsKvQuery(String key, long startTs, long endTs) {
  38 + this(key, startTs, endTs, 1, Aggregation.AVG);
52 } 39 }
53 40
54 - @Override  
55 - public String getKey() {  
56 - return key;  
57 - }  
58 -  
59 - @Override  
60 - public Optional<Long> getStartTs() {  
61 - return startTs;  
62 - }  
63 -  
64 - @Override  
65 - public Optional<Long> getEndTs() {  
66 - return endTs;  
67 - }  
68 -  
69 - @Override  
70 - public Optional<Integer> getLimit() {  
71 - return limit;  
72 - }  
73 } 41 }
@@ -21,10 +21,12 @@ public interface TsKvQuery { @@ -21,10 +21,12 @@ public interface TsKvQuery {
21 21
22 String getKey(); 22 String getKey();
23 23
24 - Optional<Long> getStartTs(); 24 + long getStartTs();
25 25
26 - Optional<Long> getEndTs(); 26 + long getEndTs();
27 27
28 - Optional<Integer> getLimit(); 28 + int getLimit();
  29 +
  30 + Aggregation getAggregation();
29 31
30 } 32 }
1 /** 1 /**
2 * Copyright © 2016-2017 The Thingsboard Authors 2 * Copyright © 2016-2017 The Thingsboard Authors
3 - * 3 + * <p>
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at 6 * You may obtain a copy of the License at
7 - *  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - * 7 + * <p>
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + * <p>
10 * Unless required by applicable law or agreed to in writing, software 10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,14 +18,15 @@ package org.thingsboard.server.dao.model; @@ -18,14 +18,15 @@ package org.thingsboard.server.dao.model;
18 import java.util.UUID; 18 import java.util.UUID;
19 19
20 import com.datastax.driver.core.utils.UUIDs; 20 import com.datastax.driver.core.utils.UUIDs;
  21 +import org.apache.commons.lang3.ArrayUtils;
21 22
22 public class ModelConstants { 23 public class ModelConstants {
23 24
24 private ModelConstants() { 25 private ModelConstants() {
25 } 26 }
26 - 27 +
27 public static UUID NULL_UUID = UUIDs.startOf(0); 28 public static UUID NULL_UUID = UUIDs.startOf(0);
28 - 29 +
29 /** 30 /**
30 * Generic constants. 31 * Generic constants.
31 */ 32 */
@@ -38,7 +39,7 @@ public class ModelConstants { @@ -38,7 +39,7 @@ public class ModelConstants {
38 public static final String ALIAS_PROPERTY = "alias"; 39 public static final String ALIAS_PROPERTY = "alias";
39 public static final String SEARCH_TEXT_PROPERTY = "search_text"; 40 public static final String SEARCH_TEXT_PROPERTY = "search_text";
40 public static final String ADDITIONAL_INFO_PROPERTY = "additional_info"; 41 public static final String ADDITIONAL_INFO_PROPERTY = "additional_info";
41 - 42 +
42 /** 43 /**
43 * Cassandra user constants. 44 * Cassandra user constants.
44 */ 45 */
@@ -50,11 +51,11 @@ public class ModelConstants { @@ -50,11 +51,11 @@ public class ModelConstants {
50 public static final String USER_FIRST_NAME_PROPERTY = "first_name"; 51 public static final String USER_FIRST_NAME_PROPERTY = "first_name";
51 public static final String USER_LAST_NAME_PROPERTY = "last_name"; 52 public static final String USER_LAST_NAME_PROPERTY = "last_name";
52 public static final String USER_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY; 53 public static final String USER_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY;
53 - 54 +
54 public static final String USER_BY_EMAIL_COLUMN_FAMILY_NAME = "user_by_email"; 55 public static final String USER_BY_EMAIL_COLUMN_FAMILY_NAME = "user_by_email";
55 public static final String USER_BY_TENANT_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "user_by_tenant_and_search_text"; 56 public static final String USER_BY_TENANT_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "user_by_tenant_and_search_text";
56 public static final String USER_BY_CUSTOMER_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "user_by_customer_and_search_text"; 57 public static final String USER_BY_CUSTOMER_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "user_by_customer_and_search_text";
57 - 58 +
58 /** 59 /**
59 * Cassandra user_credentials constants. 60 * Cassandra user_credentials constants.
60 */ 61 */
@@ -64,20 +65,20 @@ public class ModelConstants { @@ -64,20 +65,20 @@ public class ModelConstants {
64 public static final String USER_CREDENTIALS_PASSWORD_PROPERTY = "password"; 65 public static final String USER_CREDENTIALS_PASSWORD_PROPERTY = "password";
65 public static final String USER_CREDENTIALS_ACTIVATE_TOKEN_PROPERTY = "activate_token"; 66 public static final String USER_CREDENTIALS_ACTIVATE_TOKEN_PROPERTY = "activate_token";
66 public static final String USER_CREDENTIALS_RESET_TOKEN_PROPERTY = "reset_token"; 67 public static final String USER_CREDENTIALS_RESET_TOKEN_PROPERTY = "reset_token";
67 - 68 +
68 public static final String USER_CREDENTIALS_BY_USER_COLUMN_FAMILY_NAME = "user_credentials_by_user"; 69 public static final String USER_CREDENTIALS_BY_USER_COLUMN_FAMILY_NAME = "user_credentials_by_user";
69 public static final String USER_CREDENTIALS_BY_ACTIVATE_TOKEN_COLUMN_FAMILY_NAME = "user_credentials_by_activate_token"; 70 public static final String USER_CREDENTIALS_BY_ACTIVATE_TOKEN_COLUMN_FAMILY_NAME = "user_credentials_by_activate_token";
70 public static final String USER_CREDENTIALS_BY_RESET_TOKEN_COLUMN_FAMILY_NAME = "user_credentials_by_reset_token"; 71 public static final String USER_CREDENTIALS_BY_RESET_TOKEN_COLUMN_FAMILY_NAME = "user_credentials_by_reset_token";
71 - 72 +
72 /** 73 /**
73 * Cassandra admin_settings constants. 74 * Cassandra admin_settings constants.
74 */ 75 */
75 public static final String ADMIN_SETTINGS_COLUMN_FAMILY_NAME = "admin_settings"; 76 public static final String ADMIN_SETTINGS_COLUMN_FAMILY_NAME = "admin_settings";
76 public static final String ADMIN_SETTINGS_KEY_PROPERTY = "key"; 77 public static final String ADMIN_SETTINGS_KEY_PROPERTY = "key";
77 public static final String ADMIN_SETTINGS_JSON_VALUE_PROPERTY = "json_value"; 78 public static final String ADMIN_SETTINGS_JSON_VALUE_PROPERTY = "json_value";
78 - 79 +
79 public static final String ADMIN_SETTINGS_BY_KEY_COLUMN_FAMILY_NAME = "admin_settings_by_key"; 80 public static final String ADMIN_SETTINGS_BY_KEY_COLUMN_FAMILY_NAME = "admin_settings_by_key";
80 - 81 +
81 /** 82 /**
82 * Cassandra contact constants. 83 * Cassandra contact constants.
83 */ 84 */
@@ -97,9 +98,9 @@ public class ModelConstants { @@ -97,9 +98,9 @@ public class ModelConstants {
97 public static final String TENANT_TITLE_PROPERTY = TITLE_PROPERTY; 98 public static final String TENANT_TITLE_PROPERTY = TITLE_PROPERTY;
98 public static final String TENANT_REGION_PROPERTY = "region"; 99 public static final String TENANT_REGION_PROPERTY = "region";
99 public static final String TENANT_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY; 100 public static final String TENANT_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY;
100 - 101 +
101 public static final String TENANT_BY_REGION_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "tenant_by_region_and_search_text"; 102 public static final String TENANT_BY_REGION_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "tenant_by_region_and_search_text";
102 - 103 +
103 /** 104 /**
104 * Cassandra customer constants. 105 * Cassandra customer constants.
105 */ 106 */
@@ -107,9 +108,9 @@ public class ModelConstants { @@ -107,9 +108,9 @@ public class ModelConstants {
107 public static final String CUSTOMER_TENANT_ID_PROPERTY = TENTANT_ID_PROPERTY; 108 public static final String CUSTOMER_TENANT_ID_PROPERTY = TENTANT_ID_PROPERTY;
108 public static final String CUSTOMER_TITLE_PROPERTY = TITLE_PROPERTY; 109 public static final String CUSTOMER_TITLE_PROPERTY = TITLE_PROPERTY;
109 public static final String CUSTOMER_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY; 110 public static final String CUSTOMER_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY;
110 - 111 +
111 public static final String CUSTOMER_BY_TENANT_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "customer_by_tenant_and_search_text"; 112 public static final String CUSTOMER_BY_TENANT_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "customer_by_tenant_and_search_text";
112 - 113 +
113 /** 114 /**
114 * Cassandra device constants. 115 * Cassandra device constants.
115 */ 116 */
@@ -118,12 +119,12 @@ public class ModelConstants { @@ -118,12 +119,12 @@ public class ModelConstants {
118 public static final String DEVICE_CUSTOMER_ID_PROPERTY = CUSTOMER_ID_PROPERTY; 119 public static final String DEVICE_CUSTOMER_ID_PROPERTY = CUSTOMER_ID_PROPERTY;
119 public static final String DEVICE_NAME_PROPERTY = "name"; 120 public static final String DEVICE_NAME_PROPERTY = "name";
120 public static final String DEVICE_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY; 121 public static final String DEVICE_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY;
121 - 122 +
122 public static final String DEVICE_BY_TENANT_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "device_by_tenant_and_search_text"; 123 public static final String DEVICE_BY_TENANT_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "device_by_tenant_and_search_text";
123 public static final String DEVICE_BY_CUSTOMER_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "device_by_customer_and_search_text"; 124 public static final String DEVICE_BY_CUSTOMER_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "device_by_customer_and_search_text";
124 public static final String DEVICE_BY_TENANT_AND_NAME_VIEW_NAME = "device_by_tenant_and_name"; 125 public static final String DEVICE_BY_TENANT_AND_NAME_VIEW_NAME = "device_by_tenant_and_name";
125 126
126 - 127 +
127 /** 128 /**
128 * Cassandra device_credentials constants. 129 * Cassandra device_credentials constants.
129 */ 130 */
@@ -132,7 +133,7 @@ public class ModelConstants { @@ -132,7 +133,7 @@ public class ModelConstants {
132 public static final String DEVICE_CREDENTIALS_CREDENTIALS_TYPE_PROPERTY = "credentials_type"; 133 public static final String DEVICE_CREDENTIALS_CREDENTIALS_TYPE_PROPERTY = "credentials_type";
133 public static final String DEVICE_CREDENTIALS_CREDENTIALS_ID_PROPERTY = "credentials_id"; 134 public static final String DEVICE_CREDENTIALS_CREDENTIALS_ID_PROPERTY = "credentials_id";
134 public static final String DEVICE_CREDENTIALS_CREDENTIALS_VALUE_PROPERTY = "credentials_value"; 135 public static final String DEVICE_CREDENTIALS_CREDENTIALS_VALUE_PROPERTY = "credentials_value";
135 - 136 +
136 public static final String DEVICE_CREDENTIALS_BY_DEVICE_COLUMN_FAMILY_NAME = "device_credentials_by_device"; 137 public static final String DEVICE_CREDENTIALS_BY_DEVICE_COLUMN_FAMILY_NAME = "device_credentials_by_device";
137 public static final String DEVICE_CREDENTIALS_BY_CREDENTIALS_ID_COLUMN_FAMILY_NAME = "device_credentials_by_credentials_id"; 138 public static final String DEVICE_CREDENTIALS_BY_CREDENTIALS_ID_COLUMN_FAMILY_NAME = "device_credentials_by_credentials_id";
138 139
@@ -203,9 +204,9 @@ public class ModelConstants { @@ -203,9 +204,9 @@ public class ModelConstants {
203 public static final String COMPONENT_DESCRIPTOR_BY_SCOPE_TYPE_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "component_desc_by_scope_type_search_text"; 204 public static final String COMPONENT_DESCRIPTOR_BY_SCOPE_TYPE_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "component_desc_by_scope_type_search_text";
204 public static final String COMPONENT_DESCRIPTOR_BY_ID = "component_desc_by_id"; 205 public static final String COMPONENT_DESCRIPTOR_BY_ID = "component_desc_by_id";
205 206
206 - /**  
207 - * Cassandra rule metadata constants.  
208 - */ 207 + /**
  208 + * Cassandra rule metadata constants.
  209 + */
209 public static final String RULE_COLUMN_FAMILY_NAME = "rule"; 210 public static final String RULE_COLUMN_FAMILY_NAME = "rule";
210 public static final String RULE_TENANT_ID_PROPERTY = TENTANT_ID_PROPERTY; 211 public static final String RULE_TENANT_ID_PROPERTY = TENTANT_ID_PROPERTY;
211 public static final String RULE_NAME_PROPERTY = "name"; 212 public static final String RULE_NAME_PROPERTY = "name";
@@ -259,4 +260,31 @@ public class ModelConstants { @@ -259,4 +260,31 @@ public class ModelConstants {
259 public static final String STRING_VALUE_COLUMN = "str_v"; 260 public static final String STRING_VALUE_COLUMN = "str_v";
260 public static final String LONG_VALUE_COLUMN = "long_v"; 261 public static final String LONG_VALUE_COLUMN = "long_v";
261 public static final String DOUBLE_VALUE_COLUMN = "dbl_v"; 262 public static final String DOUBLE_VALUE_COLUMN = "dbl_v";
  263 +
  264 + public static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN)};
  265 +
  266 + public static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN,};
  267 + public static final String[] MIN_AGGREGATION_COLUMNS = ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS,
  268 + new String[]{min(LONG_VALUE_COLUMN), min(DOUBLE_VALUE_COLUMN), min(BOOLEAN_VALUE_COLUMN), min(STRING_VALUE_COLUMN)});
  269 + public static final String[] MAX_AGGREGATION_COLUMNS = ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS,
  270 + new String[]{max(LONG_VALUE_COLUMN), max(DOUBLE_VALUE_COLUMN), max(BOOLEAN_VALUE_COLUMN), max(STRING_VALUE_COLUMN)});
  271 + public static final String[] SUM_AGGREGATION_COLUMNS = ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS,
  272 + new String[]{sum(LONG_VALUE_COLUMN), sum(DOUBLE_VALUE_COLUMN)});
  273 + public static final String[] AVG_AGGREGATION_COLUMNS = ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS, SUM_AGGREGATION_COLUMNS);
  274 +
  275 + public static String min(String s) {
  276 + return "min(" + s + ")";
  277 + }
  278 +
  279 + public static String max(String s) {
  280 + return "max(" + s + ")";
  281 + }
  282 +
  283 + public static String sum(String s) {
  284 + return "sum(" + s + ")";
  285 + }
  286 +
  287 + public static String count(String s) {
  288 + return "count(" + s + ")";
  289 + }
262 } 290 }
  1 +package org.thingsboard.server.dao.timeseries;
  2 +
  3 +import com.datastax.driver.core.ResultSet;
  4 +import com.datastax.driver.core.Row;
  5 +import org.thingsboard.server.common.data.kv.*;
  6 +
  7 +import javax.annotation.Nullable;
  8 +import java.util.List;
  9 +import java.util.Optional;
  10 +
  11 +/**
  12 + * Created by ashvayka on 20.02.17.
  13 + */
  14 +public class AggregatePartitionsFunction implements com.google.common.base.Function<List<ResultSet>, Optional<TsKvEntry>> {
  15 +
  16 + private static final int LONG_CNT_POS = 0;
  17 + private static final int DOUBLE_CNT_POS = 1;
  18 + private static final int BOOL_CNT_POS = 2;
  19 + private static final int STR_CNT_POS = 3;
  20 + private static final int LONG_POS = 4;
  21 + private static final int DOUBLE_POS = 5;
  22 + private static final int BOOL_POS = 6;
  23 + private static final int STR_POS = 7;
  24 +
  25 + private final Aggregation aggregation;
  26 + private final String key;
  27 + private final long ts;
  28 +
  29 + public AggregatePartitionsFunction(Aggregation aggregation, String key, long ts) {
  30 + this.aggregation = aggregation;
  31 + this.key = key;
  32 + this.ts = ts;
  33 + }
  34 +
  35 + @Nullable
  36 + @Override
  37 + public Optional<TsKvEntry> apply(@Nullable List<ResultSet> rsList) {
  38 + if (rsList == null || rsList.size() == 0) {
  39 + return Optional.empty();
  40 + }
  41 + long count = 0;
  42 + DataType dataType = null;
  43 +
  44 + Boolean bValue = null;
  45 + String sValue = null;
  46 + Double dValue = null;
  47 + Long lValue = null;
  48 +
  49 + for (ResultSet rs : rsList) {
  50 + for (Row row : rs.all()) {
  51 + long curCount;
  52 +
  53 + Long curLValue = null;
  54 + Double curDValue = null;
  55 + Boolean curBValue = null;
  56 + String curSValue = null;
  57 +
  58 + long longCount = row.getLong(LONG_CNT_POS);
  59 + long doubleCount = row.getLong(DOUBLE_CNT_POS);
  60 + long boolCount = row.getLong(BOOL_CNT_POS);
  61 + long strCount = row.getLong(STR_CNT_POS);
  62 +
  63 + if (longCount > 0) {
  64 + dataType = DataType.LONG;
  65 + curCount = longCount;
  66 + curLValue = getLongValue(row);
  67 + } else if (doubleCount > 0) {
  68 + dataType = DataType.DOUBLE;
  69 + curCount = doubleCount;
  70 + curDValue = getDoubleValue(row);
  71 + } else if (boolCount > 0) {
  72 + dataType = DataType.BOOLEAN;
  73 + curCount = boolCount;
  74 + curBValue = getBooleanValue(row);
  75 + } else if (strCount > 0) {
  76 + dataType = DataType.STRING;
  77 + curCount = strCount;
  78 + curSValue = getStringValue(row);
  79 + } else {
  80 + continue;
  81 + }
  82 +
  83 + if (aggregation == Aggregation.COUNT) {
  84 + count += curCount;
  85 + } else if (aggregation == Aggregation.AVG || aggregation == Aggregation.SUM) {
  86 + count += curCount;
  87 + dValue = dValue == null ? curDValue : dValue + curDValue;
  88 + lValue = lValue == null ? curLValue : lValue + curLValue;
  89 + } else if (aggregation == Aggregation.MIN) {
  90 + if (curDValue != null) {
  91 + dValue = dValue == null ? curDValue : Math.min(dValue, curDValue);
  92 + } else if (curLValue != null) {
  93 + lValue = lValue == null ? curLValue : Math.min(lValue, curLValue);
  94 + } else if (curBValue != null) {
  95 + bValue = bValue == null ? curBValue : bValue && curBValue;
  96 + } else if (curSValue != null) {
  97 + if (sValue == null || curSValue.compareTo(sValue) < 0) {
  98 + sValue = curSValue;
  99 + }
  100 + }
  101 + } else if (aggregation == Aggregation.MAX) {
  102 + if (curDValue != null) {
  103 + dValue = dValue == null ? curDValue : Math.max(dValue, curDValue);
  104 + } else if (curLValue != null) {
  105 + lValue = lValue == null ? curLValue : Math.max(lValue, curLValue);
  106 + } else if (curBValue != null) {
  107 + bValue = bValue == null ? curBValue : bValue || curBValue;
  108 + } else if (curSValue != null) {
  109 + if (sValue == null || curSValue.compareTo(sValue) > 0) {
  110 + sValue = curSValue;
  111 + }
  112 + }
  113 + }
  114 + }
  115 + }
  116 + if (dataType == null) {
  117 + return Optional.empty();
  118 + } else if (aggregation == Aggregation.COUNT) {
  119 + return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, (long) count)));
  120 + } else if (aggregation == Aggregation.AVG || aggregation == Aggregation.SUM) {
  121 + if (count == 0 || (dataType == DataType.DOUBLE && dValue == null) || (dataType == DataType.LONG && lValue == null)) {
  122 + return Optional.empty();
  123 + } else if (dataType == DataType.DOUBLE) {
  124 + return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.SUM ? dValue : (dValue / count))));
  125 + } else if (dataType == DataType.LONG) {
  126 + return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, aggregation == Aggregation.SUM ? lValue : (lValue / count))));
  127 + }
  128 + } else if (aggregation == Aggregation.MIN || aggregation == Aggregation.MAX) {
  129 + if (dataType == DataType.DOUBLE) {
  130 + return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, dValue)));
  131 + } else if (dataType == DataType.LONG) {
  132 + return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, lValue)));
  133 + } else if (dataType == DataType.STRING) {
  134 + return Optional.of(new BasicTsKvEntry(ts, new StringDataEntry(key, sValue)));
  135 + } else {
  136 + return Optional.of(new BasicTsKvEntry(ts, new BooleanDataEntry(key, bValue)));
  137 + }
  138 + }
  139 + return null;
  140 + }
  141 +
  142 + private Boolean getBooleanValue(Row row) {
  143 + if (aggregation == Aggregation.MIN || aggregation == Aggregation.MAX) {
  144 + return row.getBool(BOOL_POS);
  145 + } else {
  146 + return null;
  147 + }
  148 + }
  149 +
  150 + private String getStringValue(Row row) {
  151 + if (aggregation == Aggregation.MIN || aggregation == Aggregation.MAX) {
  152 + return row.getString(STR_POS);
  153 + } else {
  154 + return null;
  155 + }
  156 + }
  157 +
  158 + private Long getLongValue(Row row) {
  159 + if (aggregation == Aggregation.MIN || aggregation == Aggregation.MAX
  160 + || aggregation == Aggregation.SUM || aggregation == Aggregation.AVG) {
  161 + return row.getLong(LONG_POS);
  162 + } else {
  163 + return null;
  164 + }
  165 + }
  166 +
  167 + private Double getDoubleValue(Row row) {
  168 + if (aggregation == Aggregation.MIN || aggregation == Aggregation.MAX
  169 + || aggregation == Aggregation.SUM || aggregation == Aggregation.AVG) {
  170 + return row.getDouble(DOUBLE_POS);
  171 + } else {
  172 + return null;
  173 + }
  174 + }
  175 +}
1 /** 1 /**
2 * Copyright © 2016-2017 The Thingsboard Authors 2 * Copyright © 2016-2017 The Thingsboard Authors
3 - * 3 + * <p>
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at 6 * You may obtain a copy of the License at
7 - *  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - * 7 + * <p>
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + * <p>
10 * Unless required by applicable law or agreed to in writing, software 10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,6 +18,10 @@ package org.thingsboard.server.dao.timeseries; @@ -18,6 +18,10 @@ package org.thingsboard.server.dao.timeseries;
18 import com.datastax.driver.core.*; 18 import com.datastax.driver.core.*;
19 import com.datastax.driver.core.querybuilder.QueryBuilder; 19 import com.datastax.driver.core.querybuilder.QueryBuilder;
20 import com.datastax.driver.core.querybuilder.Select; 20 import com.datastax.driver.core.querybuilder.Select;
  21 +import com.google.common.base.Function;
  22 +import com.google.common.util.concurrent.AsyncFunction;
  23 +import com.google.common.util.concurrent.Futures;
  24 +import com.google.common.util.concurrent.ListenableFuture;
21 import lombok.extern.slf4j.Slf4j; 25 import lombok.extern.slf4j.Slf4j;
22 import org.springframework.beans.factory.annotation.Value; 26 import org.springframework.beans.factory.annotation.Value;
23 import org.springframework.stereotype.Component; 27 import org.springframework.stereotype.Component;
@@ -26,7 +30,16 @@ import org.thingsboard.server.common.data.kv.DataType; @@ -26,7 +30,16 @@ import org.thingsboard.server.common.data.kv.DataType;
26 import org.thingsboard.server.dao.AbstractDao; 30 import org.thingsboard.server.dao.AbstractDao;
27 import org.thingsboard.server.dao.model.ModelConstants; 31 import org.thingsboard.server.dao.model.ModelConstants;
28 32
  33 +import javax.annotation.Nullable;
  34 +import javax.annotation.PostConstruct;
  35 +import javax.annotation.PreDestroy;
  36 +import java.time.Instant;
  37 +import java.time.LocalDateTime;
  38 +import java.time.ZoneOffset;
29 import java.util.*; 39 import java.util.*;
  40 +import java.util.concurrent.ExecutorService;
  41 +import java.util.concurrent.Executors;
  42 +import java.util.stream.Collectors;
30 43
31 import static com.datastax.driver.core.querybuilder.QueryBuilder.eq; 44 import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
32 import static com.datastax.driver.core.querybuilder.QueryBuilder.select; 45 import static com.datastax.driver.core.querybuilder.QueryBuilder.select;
@@ -41,48 +54,136 @@ public class BaseTimeseriesDao extends AbstractDao implements TimeseriesDao { @@ -41,48 +54,136 @@ public class BaseTimeseriesDao extends AbstractDao implements TimeseriesDao {
41 @Value("${cassandra.query.max_limit_per_request}") 54 @Value("${cassandra.query.max_limit_per_request}")
42 protected Integer maxLimitPerRequest; 55 protected Integer maxLimitPerRequest;
43 56
  57 + @Value("${cassandra.query.read_result_processing_threads}")
  58 + private int readResultsProcessingThreads;
  59 +
  60 + @Value("${cassandra.query.min_read_step}")
  61 + private int minReadStep;
  62 +
  63 + @Value("${cassandra.query.ts_key_value_partitioning}")
  64 + private String partitioning;
  65 +
  66 + private TsPartitionDate tsFormat;
  67 +
  68 + private ExecutorService readResultsProcessingExecutor;
  69 +
44 private PreparedStatement partitionInsertStmt; 70 private PreparedStatement partitionInsertStmt;
45 private PreparedStatement[] latestInsertStmts; 71 private PreparedStatement[] latestInsertStmts;
46 private PreparedStatement[] saveStmts; 72 private PreparedStatement[] saveStmts;
  73 + private PreparedStatement[] fetchStmts;
47 private PreparedStatement findLatestStmt; 74 private PreparedStatement findLatestStmt;
48 private PreparedStatement findAllLatestStmt; 75 private PreparedStatement findAllLatestStmt;
49 76
  77 + @PostConstruct
  78 + public void init() {
  79 + getFetchStmt(Aggregation.NONE);
  80 + readResultsProcessingExecutor = Executors.newFixedThreadPool(readResultsProcessingThreads);
  81 + Optional<TsPartitionDate> partition = TsPartitionDate.parse(partitioning);
  82 + if (partition.isPresent()) {
  83 + tsFormat = partition.get();
  84 + } else {
  85 + log.warn("Incorrect configuration of partitioning {}", partitioning);
  86 + throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!");
  87 + }
  88 + }
  89 +
  90 + @PreDestroy
  91 + public void stop() {
  92 + if (readResultsProcessingExecutor != null) {
  93 + readResultsProcessingExecutor.shutdownNow();
  94 + }
  95 + }
  96 +
50 @Override 97 @Override
51 - public List<TsKvEntry> find(String entityType, UUID entityId, TsKvQuery query, Optional<Long> minPartition, Optional<Long> maxPartition) {  
52 - List<Row> rows = Collections.emptyList();  
53 - Long[] parts = fetchPartitions(entityType, entityId, query.getKey(), minPartition, maxPartition);  
54 - int partsLength = parts.length;  
55 - if (parts != null && partsLength > 0) {  
56 - int limit = maxLimitPerRequest;  
57 - Optional<Integer> lim = query.getLimit();  
58 - if (lim.isPresent() && lim.get() < maxLimitPerRequest) {  
59 - limit = lim.get();  
60 - } 98 + public long toPartitionTs(long ts) {
  99 + LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC);
  100 + return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli();
  101 + }
61 102
62 - rows = new ArrayList<>(limit);  
63 - int lastIdx = partsLength - 1;  
64 - for (int i = 0; i < partsLength; i++) {  
65 - int currentLimit;  
66 - if (rows.size() >= limit) {  
67 - break;  
68 - } else {  
69 - currentLimit = limit - rows.size(); 103 +
  104 + private static String[] getFetchColumnNames(Aggregation aggregation) {
  105 + switch (aggregation) {
  106 + case NONE:
  107 + return ModelConstants.NONE_AGGREGATION_COLUMNS;
  108 + case MIN:
  109 + return ModelConstants.MIN_AGGREGATION_COLUMNS;
  110 + case MAX:
  111 + return ModelConstants.MAX_AGGREGATION_COLUMNS;
  112 + case SUM:
  113 + return ModelConstants.SUM_AGGREGATION_COLUMNS;
  114 + case COUNT:
  115 + return ModelConstants.COUNT_AGGREGATION_COLUMNS;
  116 + case AVG:
  117 + return ModelConstants.AVG_AGGREGATION_COLUMNS;
  118 + default:
  119 + throw new RuntimeException("Aggregation type: " + aggregation + " is not supported!");
  120 + }
  121 + }
  122 +
  123 + @Override
  124 + public ListenableFuture<List<TsKvEntry>> findAllAsync(String entityType, UUID entityId, TsKvQuery query, long minPartition, long maxPartition) {
  125 + if (query.getAggregation() == Aggregation.NONE) {
  126 + //TODO:
  127 + return null;
  128 + } else {
  129 + long step = Math.max((query.getEndTs() - query.getStartTs()) / query.getLimit(), minReadStep);
  130 + long stepTs = query.getStartTs();
  131 + List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>();
  132 + while (stepTs < query.getEndTs()) {
  133 + long startTs = stepTs;
  134 + long endTs = stepTs + step;
  135 + TsKvQuery subQuery = new BaseTsKvQuery(query.getKey(), startTs, endTs, 1, query.getAggregation());
  136 + futures.add(findAndAggregateAsync(entityType, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs)));
  137 + stepTs = endTs;
  138 + }
  139 + ListenableFuture<List<Optional<TsKvEntry>>> future = Futures.allAsList(futures);
  140 + return Futures.transform(future, new Function<List<Optional<TsKvEntry>>, List<TsKvEntry>>() {
  141 + @Nullable
  142 + @Override
  143 + public List<TsKvEntry> apply(@Nullable List<Optional<TsKvEntry>> input) {
  144 + return input.stream().filter(v -> v.isPresent()).map(v -> v.get()).collect(Collectors.toList());
70 } 145 }
71 - Long partition = parts[i];  
72 - Select.Where where = select().from(ModelConstants.TS_KV_CF).where(eq(ModelConstants.ENTITY_TYPE_COLUMN, entityType))  
73 - .and(eq(ModelConstants.ENTITY_ID_COLUMN, entityId))  
74 - .and(eq(ModelConstants.KEY_COLUMN, query.getKey()))  
75 - .and(eq(ModelConstants.PARTITION_COLUMN, partition));  
76 - if (i == 0 && query.getStartTs().isPresent()) {  
77 - where.and(QueryBuilder.gt(ModelConstants.TS_COLUMN, query.getStartTs().get()));  
78 - } else if (i == lastIdx && query.getEndTs().isPresent()) {  
79 - where.and(QueryBuilder.lte(ModelConstants.TS_COLUMN, query.getEndTs().get())); 146 + });
  147 + }
  148 + }
  149 +
  150 + private ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(String entityType, UUID entityId, TsKvQuery query, long minPartition, long maxPartition) {
  151 + final Aggregation aggregation = query.getAggregation();
  152 + final long startTs = query.getStartTs();
  153 + final long endTs = query.getEndTs();
  154 + final long ts = startTs + (endTs - startTs) / 2;
  155 +
  156 + ResultSetFuture partitionsFuture = fetchPartitions(entityType, entityId, query.getKey(), minPartition, maxPartition);
  157 + com.google.common.base.Function<ResultSet, List<Long>> toArrayFunction = rows -> rows.all().stream()
  158 + .map(row -> row.getLong(ModelConstants.PARTITION_COLUMN)).collect(Collectors.toList());
  159 +
  160 + ListenableFuture<List<Long>> partitionsListFuture = Futures.transform(partitionsFuture, toArrayFunction, readResultsProcessingExecutor);
  161 +
  162 + AsyncFunction<List<Long>, List<ResultSet>> fetchChunksFunction = partitions -> {
  163 + try {
  164 + PreparedStatement proto = getFetchStmt(aggregation);
  165 + List<ResultSetFuture> futures = new ArrayList<>(partitions.size());
  166 + for (Long partition : partitions) {
  167 + BoundStatement stmt = proto.bind();
  168 + stmt.setString(0, entityType);
  169 + stmt.setUUID(1, entityId);
  170 + stmt.setString(2, query.getKey());
  171 + stmt.setLong(3, partition);
  172 + stmt.setLong(4, startTs);
  173 + stmt.setLong(5, endTs);
  174 + log.debug("Generated query [{}] for entityType {} and entityId {}", stmt, entityType, entityId);
  175 + futures.add(executeAsyncRead(stmt));
80 } 176 }
81 - where.limit(currentLimit);  
82 - rows.addAll(executeRead(where).all()); 177 + return Futures.allAsList(futures);
  178 + } catch (Throwable e) {
  179 + log.error("Failed to fetch data", e);
  180 + throw e;
83 } 181 }
84 - }  
85 - return convertResultToTsKvEntryList(rows); 182 + };
  183 +
  184 + ListenableFuture<List<ResultSet>> aggregationChunks = Futures.transform(partitionsListFuture, fetchChunksFunction, readResultsProcessingExecutor);
  185 +
  186 + return Futures.transform(aggregationChunks, new AggregatePartitionsFunction(aggregation, query.getKey(), ts), readResultsProcessingExecutor);
86 } 187 }
87 188
88 @Override 189 @Override
@@ -190,13 +291,12 @@ public class BaseTimeseriesDao extends AbstractDao implements TimeseriesDao { @@ -190,13 +291,12 @@ public class BaseTimeseriesDao extends AbstractDao implements TimeseriesDao {
190 * Select existing partitions from the table 291 * Select existing partitions from the table
191 * <code>{@link ModelConstants#TS_KV_PARTITIONS_CF}</code> for the given entity 292 * <code>{@link ModelConstants#TS_KV_PARTITIONS_CF}</code> for the given entity
192 */ 293 */
193 - private Long[] fetchPartitions(String entityType, UUID entityId, String key, Optional<Long> minPartition, Optional<Long> maxPartition) { 294 + private ResultSetFuture fetchPartitions(String entityType, UUID entityId, String key, long minPartition, long maxPartition) {
194 Select.Where select = QueryBuilder.select(ModelConstants.PARTITION_COLUMN).from(ModelConstants.TS_KV_PARTITIONS_CF).where(eq(ModelConstants.ENTITY_TYPE_COLUMN, entityType)) 295 Select.Where select = QueryBuilder.select(ModelConstants.PARTITION_COLUMN).from(ModelConstants.TS_KV_PARTITIONS_CF).where(eq(ModelConstants.ENTITY_TYPE_COLUMN, entityType))
195 .and(eq(ModelConstants.ENTITY_ID_COLUMN, entityId)).and(eq(ModelConstants.KEY_COLUMN, key)); 296 .and(eq(ModelConstants.ENTITY_ID_COLUMN, entityId)).and(eq(ModelConstants.KEY_COLUMN, key));
196 - minPartition.ifPresent(startTs -> select.and(QueryBuilder.gte(ModelConstants.PARTITION_COLUMN, minPartition.get())));  
197 - maxPartition.ifPresent(endTs -> select.and(QueryBuilder.lte(ModelConstants.PARTITION_COLUMN, maxPartition.get())));  
198 - ResultSet resultSet = executeRead(select);  
199 - return resultSet.all().stream().map(row -> row.getLong(ModelConstants.PARTITION_COLUMN)).toArray(Long[]::new); 297 + select.and(QueryBuilder.gte(ModelConstants.PARTITION_COLUMN, minPartition));
  298 + select.and(QueryBuilder.lte(ModelConstants.PARTITION_COLUMN, maxPartition));
  299 + return executeAsyncRead(select);
200 } 300 }
201 301
202 private PreparedStatement getSaveStmt(DataType dataType) { 302 private PreparedStatement getSaveStmt(DataType dataType) {
@@ -216,6 +316,23 @@ public class BaseTimeseriesDao extends AbstractDao implements TimeseriesDao { @@ -216,6 +316,23 @@ public class BaseTimeseriesDao extends AbstractDao implements TimeseriesDao {
216 return saveStmts[dataType.ordinal()]; 316 return saveStmts[dataType.ordinal()];
217 } 317 }
218 318
  319 + private PreparedStatement getFetchStmt(Aggregation aggType) {
  320 + if (fetchStmts == null) {
  321 + fetchStmts = new PreparedStatement[Aggregation.values().length];
  322 + for (Aggregation type : Aggregation.values()) {
  323 + fetchStmts[type.ordinal()] = getSession().prepare("SELECT " +
  324 + String.join(", ", getFetchColumnNames(type)) + " FROM " + ModelConstants.TS_KV_CF
  325 + + " WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + " = ? "
  326 + + "AND " + ModelConstants.ENTITY_ID_COLUMN + " = ? "
  327 + + "AND " + ModelConstants.KEY_COLUMN + " = ? "
  328 + + "AND " + ModelConstants.PARTITION_COLUMN + " = ? "
  329 + + "AND " + ModelConstants.TS_COLUMN + " > ? "
  330 + + "AND " + ModelConstants.TS_COLUMN + " <= ?");
  331 + }
  332 + }
  333 + return fetchStmts[aggType.ordinal()];
  334 + }
  335 +
219 private PreparedStatement getLatestStmt(DataType dataType) { 336 private PreparedStatement getLatestStmt(DataType dataType) {
220 if (latestInsertStmts == null) { 337 if (latestInsertStmts == null) {
221 latestInsertStmts = new PreparedStatement[DataType.values().length]; 338 latestInsertStmts = new PreparedStatement[DataType.values().length];
1 /** 1 /**
2 * Copyright © 2016-2017 The Thingsboard Authors 2 * Copyright © 2016-2017 The Thingsboard Authors
3 - * 3 + * <p>
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at 6 * You may obtain a copy of the License at
7 - *  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - * 7 + * <p>
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + * <p>
10 * Unless required by applicable law or agreed to in writing, software 10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -23,21 +23,23 @@ import com.google.common.util.concurrent.Futures; @@ -23,21 +23,23 @@ import com.google.common.util.concurrent.Futures;
23 import com.google.common.util.concurrent.ListenableFuture; 23 import com.google.common.util.concurrent.ListenableFuture;
24 import lombok.extern.slf4j.Slf4j; 24 import lombok.extern.slf4j.Slf4j;
25 import org.thingsboard.server.common.data.id.UUIDBased; 25 import org.thingsboard.server.common.data.id.UUIDBased;
  26 +import org.thingsboard.server.common.data.kv.BaseTsKvQuery;
26 import org.thingsboard.server.common.data.kv.TsKvEntry; 27 import org.thingsboard.server.common.data.kv.TsKvEntry;
27 import org.thingsboard.server.common.data.kv.TsKvQuery; 28 import org.thingsboard.server.common.data.kv.TsKvQuery;
28 import org.thingsboard.server.dao.exception.IncorrectParameterException; 29 import org.thingsboard.server.dao.exception.IncorrectParameterException;
29 -import org.slf4j.Logger;  
30 -import org.slf4j.LoggerFactory;  
31 import org.springframework.beans.factory.annotation.Autowired; 30 import org.springframework.beans.factory.annotation.Autowired;
32 import org.springframework.beans.factory.annotation.Value; 31 import org.springframework.beans.factory.annotation.Value;
33 import org.springframework.stereotype.Service; 32 import org.springframework.stereotype.Service;
34 import org.thingsboard.server.dao.service.Validator; 33 import org.thingsboard.server.dao.service.Validator;
35 34
36 import javax.annotation.PostConstruct; 35 import javax.annotation.PostConstruct;
  36 +import javax.annotation.PreDestroy;
37 import java.time.Instant; 37 import java.time.Instant;
38 import java.time.LocalDateTime; 38 import java.time.LocalDateTime;
39 import java.time.ZoneOffset; 39 import java.time.ZoneOffset;
40 import java.util.*; 40 import java.util.*;
  41 +import java.util.concurrent.ExecutorService;
  42 +import java.util.concurrent.Executors;
41 43
42 import static org.apache.commons.lang3.StringUtils.isBlank; 44 import static org.apache.commons.lang3.StringUtils.isBlank;
43 45
@@ -50,38 +52,14 @@ public class BaseTimeseriesService implements TimeseriesService { @@ -50,38 +52,14 @@ public class BaseTimeseriesService implements TimeseriesService {
50 52
51 public static final int INSERTS_PER_ENTRY = 3; 53 public static final int INSERTS_PER_ENTRY = 3;
52 54
53 - @Value("${cassandra.query.ts_key_value_partitioning}")  
54 - private String partitioning;  
55 -  
56 @Autowired 55 @Autowired
57 private TimeseriesDao timeseriesDao; 56 private TimeseriesDao timeseriesDao;
58 57
59 - private TsPartitionDate tsFormat;  
60 -  
61 - @PostConstruct  
62 - public void init() {  
63 - Optional<TsPartitionDate> partition = TsPartitionDate.parse(partitioning);  
64 - if (partition.isPresent()) {  
65 - tsFormat = partition.get();  
66 - } else {  
67 - log.warn("Incorrect configuration of partitioning {}", partitioning);  
68 - throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!");  
69 - }  
70 - }  
71 -  
72 @Override 58 @Override
73 - public List<TsKvEntry> find(String entityType, UUIDBased entityId, TsKvQuery query) { 59 + public ListenableFuture<List<TsKvEntry>> findAll(String entityType, UUIDBased entityId, TsKvQuery query) {
74 validate(entityType, entityId); 60 validate(entityType, entityId);
75 validate(query); 61 validate(query);
76 - return timeseriesDao.find(entityType, entityId.getId(), query, toPartitionTs(query.getStartTs()), toPartitionTs(query.getEndTs()));  
77 - }  
78 -  
79 - private Optional<Long> toPartitionTs(Optional<Long> ts) {  
80 - if (ts.isPresent()) {  
81 - return Optional.of(toPartitionTs(ts.get()));  
82 - } else {  
83 - return Optional.empty();  
84 - } 62 + return timeseriesDao.findAllAsync(entityType, entityId.getId(), query, timeseriesDao.toPartitionTs(query.getStartTs()), timeseriesDao.toPartitionTs(query.getEndTs()));
85 } 63 }
86 64
87 @Override 65 @Override
@@ -106,7 +84,7 @@ public class BaseTimeseriesService implements TimeseriesService { @@ -106,7 +84,7 @@ public class BaseTimeseriesService implements TimeseriesService {
106 throw new IncorrectParameterException("Key value entry can't be null"); 84 throw new IncorrectParameterException("Key value entry can't be null");
107 } 85 }
108 UUID uid = entityId.getId(); 86 UUID uid = entityId.getId();
109 - long partitionTs = toPartitionTs(tsKvEntry.getTs()); 87 + long partitionTs = timeseriesDao.toPartitionTs(tsKvEntry.getTs());
110 88
111 List<ResultSetFuture> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY); 89 List<ResultSetFuture> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY);
112 saveAndRegisterFutures(futures, entityType, tsKvEntry, uid, partitionTs); 90 saveAndRegisterFutures(futures, entityType, tsKvEntry, uid, partitionTs);
@@ -122,7 +100,7 @@ public class BaseTimeseriesService implements TimeseriesService { @@ -122,7 +100,7 @@ public class BaseTimeseriesService implements TimeseriesService {
122 throw new IncorrectParameterException("Key value entry can't be null"); 100 throw new IncorrectParameterException("Key value entry can't be null");
123 } 101 }
124 UUID uid = entityId.getId(); 102 UUID uid = entityId.getId();
125 - long partitionTs = toPartitionTs(tsKvEntry.getTs()); 103 + long partitionTs = timeseriesDao.toPartitionTs(tsKvEntry.getTs());
126 saveAndRegisterFutures(futures, entityType, tsKvEntry, uid, partitionTs); 104 saveAndRegisterFutures(futures, entityType, tsKvEntry, uid, partitionTs);
127 } 105 }
128 return Futures.allAsList(futures); 106 return Futures.allAsList(futures);
@@ -144,14 +122,6 @@ public class BaseTimeseriesService implements TimeseriesService { @@ -144,14 +122,6 @@ public class BaseTimeseriesService implements TimeseriesService {
144 futures.add(timeseriesDao.save(entityType, uid, partitionTs, tsKvEntry)); 122 futures.add(timeseriesDao.save(entityType, uid, partitionTs, tsKvEntry));
145 } 123 }
146 124
147 - private long toPartitionTs(long ts) {  
148 - LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC);  
149 -  
150 - LocalDateTime parititonTime = tsFormat.truncatedTo(time);  
151 -  
152 - return parititonTime.toInstant(ZoneOffset.UTC).toEpochMilli();  
153 - }  
154 -  
155 private static void validate(String entityType, UUIDBased entityId) { 125 private static void validate(String entityType, UUIDBased entityId) {
156 Validator.validateString(entityType, "Incorrect entityType " + entityType); 126 Validator.validateString(entityType, "Incorrect entityType " + entityType);
157 Validator.validateId(entityId, "Incorrect entityId " + entityId); 127 Validator.validateId(entityId, "Incorrect entityId " + entityId);
@@ -163,5 +133,6 @@ public class BaseTimeseriesService implements TimeseriesService { @@ -163,5 +133,6 @@ public class BaseTimeseriesService implements TimeseriesService {
163 } else if (isBlank(query.getKey())) { 133 } else if (isBlank(query.getKey())) {
164 throw new IncorrectParameterException("Incorrect TsKvQuery. Key can't be empty"); 134 throw new IncorrectParameterException("Incorrect TsKvQuery. Key can't be empty");
165 } 135 }
  136 + //TODO: add validation of all params
166 } 137 }
167 } 138 }
@@ -17,6 +17,7 @@ package org.thingsboard.server.dao.timeseries; @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.timeseries;
17 17
18 import com.datastax.driver.core.ResultSetFuture; 18 import com.datastax.driver.core.ResultSetFuture;
19 import com.datastax.driver.core.Row; 19 import com.datastax.driver.core.Row;
  20 +import com.google.common.util.concurrent.ListenableFuture;
20 import org.thingsboard.server.common.data.kv.TsKvEntry; 21 import org.thingsboard.server.common.data.kv.TsKvEntry;
21 import org.thingsboard.server.common.data.kv.TsKvQuery; 22 import org.thingsboard.server.common.data.kv.TsKvQuery;
22 23
@@ -30,7 +31,11 @@ import java.util.UUID; @@ -30,7 +31,11 @@ import java.util.UUID;
30 */ 31 */
31 public interface TimeseriesDao { 32 public interface TimeseriesDao {
32 33
33 - List<TsKvEntry> find(String entityType, UUID entityId, TsKvQuery query, Optional<Long> minPartition, Optional<Long> maxPartition); 34 + long toPartitionTs(long ts);
  35 +
  36 + ListenableFuture<List<TsKvEntry>> findAllAsync(String entityType, UUID entityId, TsKvQuery query, long minPartition, long maxPartition);
  37 +
  38 +// List<TsKvEntry> find(String entityType, UUID entityId, TsKvQuery query, Optional<Long> minPartition, Optional<Long> maxPartition);
34 39
35 ResultSetFuture findLatest(String entityType, UUID entityId, String key); 40 ResultSetFuture findLatest(String entityType, UUID entityId, String key);
36 41
@@ -19,6 +19,7 @@ import com.datastax.driver.core.ResultSet; @@ -19,6 +19,7 @@ import com.datastax.driver.core.ResultSet;
19 import com.datastax.driver.core.ResultSetFuture; 19 import com.datastax.driver.core.ResultSetFuture;
20 import com.datastax.driver.core.Row; 20 import com.datastax.driver.core.Row;
21 import com.google.common.util.concurrent.ListenableFuture; 21 import com.google.common.util.concurrent.ListenableFuture;
  22 +import org.thingsboard.server.common.data.id.DeviceId;
22 import org.thingsboard.server.common.data.id.UUIDBased; 23 import org.thingsboard.server.common.data.id.UUIDBased;
23 import org.thingsboard.server.common.data.kv.TsKvEntry; 24 import org.thingsboard.server.common.data.kv.TsKvEntry;
24 import org.thingsboard.server.common.data.kv.TsKvQuery; 25 import org.thingsboard.server.common.data.kv.TsKvQuery;
@@ -32,8 +33,7 @@ import java.util.Set; @@ -32,8 +33,7 @@ import java.util.Set;
32 */ 33 */
33 public interface TimeseriesService { 34 public interface TimeseriesService {
34 35
35 - //TODO: Replace this with async operation  
36 - List<TsKvEntry> find(String entityType, UUIDBased entityId, TsKvQuery query); 36 + ListenableFuture<List<TsKvEntry>> findAll(String entityType, UUIDBased entityId, TsKvQuery query);
37 37
38 ListenableFuture<List<ResultSet>> findLatest(String entityType, UUIDBased entityId, Collection<String> keys); 38 ListenableFuture<List<ResultSet>> findLatest(String entityType, UUIDBased entityId, Collection<String> keys);
39 39
@@ -25,11 +25,11 @@ import java.util.Arrays; @@ -25,11 +25,11 @@ import java.util.Arrays;
25 25
26 @RunWith(ClasspathSuite.class) 26 @RunWith(ClasspathSuite.class)
27 @ClassnameFilters({ 27 @ClassnameFilters({
28 - "org.thingsboard.server.dao.service.*Test",  
29 - "org.thingsboard.server.dao.kv.*Test",  
30 - "org.thingsboard.server.dao.plugin.*Test",  
31 - "org.thingsboard.server.dao.rule.*Test",  
32 - "org.thingsboard.server.dao.attributes.*Test", 28 +// "org.thingsboard.server.dao.service.*Test",
  29 +// "org.thingsboard.server.dao.kv.*Test",
  30 +// "org.thingsboard.server.dao.plugin.*Test",
  31 +// "org.thingsboard.server.dao.rule.*Test",
  32 +// "org.thingsboard.server.dao.attributes.*Test",
33 "org.thingsboard.server.dao.timeseries.*Test" 33 "org.thingsboard.server.dao.timeseries.*Test"
34 }) 34 })
35 public class DaoTestSuite { 35 public class DaoTestSuite {
@@ -116,14 +116,36 @@ public class TimeseriesServiceTest extends AbstractServiceTest { @@ -116,14 +116,36 @@ public class TimeseriesServiceTest extends AbstractServiceTest {
116 entries.add(tsKvEntry); 116 entries.add(tsKvEntry);
117 } 117 }
118 log.debug("Saved all records {}", localDateTime); 118 log.debug("Saved all records {}", localDateTime);
119 - List<TsKvEntry> list = tsService.find(DataConstants.DEVICE, deviceId, new BaseTsKvQuery(STRING_KEY, entries.get(599).getTs(),  
120 - LocalDateTime.now(ZoneOffset.UTC).toInstant(ZoneOffset.UTC).toEpochMilli())); 119 + List<TsKvEntry> list = tsService.findAll(DataConstants.DEVICE, deviceId, new BaseTsKvQuery(STRING_KEY, entries.get(599).getTs(),
  120 + LocalDateTime.now(ZoneOffset.UTC).toInstant(ZoneOffset.UTC).toEpochMilli(), PARTITION_MINUTES - 599, Aggregation.MIN)).get();
121 log.debug("Fetched records {}", localDateTime); 121 log.debug("Fetched records {}", localDateTime);
122 List<TsKvEntry> expected = entries.subList(600, PARTITION_MINUTES); 122 List<TsKvEntry> expected = entries.subList(600, PARTITION_MINUTES);
123 assertEquals(expected.size(), list.size()); 123 assertEquals(expected.size(), list.size());
124 assertEquals(expected, list); 124 assertEquals(expected, list);
125 } 125 }
126 126
  127 +// @Test
  128 +// public void testFindDeviceTsDataByQuery() throws Exception {
  129 +// DeviceId deviceId = new DeviceId(UUIDs.timeBased());
  130 +// LocalDateTime localDateTime = LocalDateTime.now(ZoneOffset.UTC).minusMinutes(PARTITION_MINUTES);
  131 +// log.debug("Start event time is {}", localDateTime);
  132 +// List<TsKvEntry> entries = new ArrayList<>(PARTITION_MINUTES);
  133 +//
  134 +// for (int i = 0; i < PARTITION_MINUTES; i++) {
  135 +// long time = localDateTime.plusMinutes(i).toInstant(ZoneOffset.UTC).toEpochMilli();
  136 +// BasicTsKvEntry tsKvEntry = new BasicTsKvEntry(time, stringKvEntry);
  137 +// tsService.save(DataConstants.DEVICE, deviceId, tsKvEntry).get();
  138 +// entries.add(tsKvEntry);
  139 +// }
  140 +// log.debug("Saved all records {}", localDateTime);
  141 +// List<TsKvEntry> list = tsService.findAll(DataConstants.DEVICE, deviceId, new BaseTsKvQuery(STRING_KEY, entries.get(599).getTs(),
  142 +// LocalDateTime.now(ZoneOffset.UTC).toInstant(ZoneOffset.UTC).toEpochMilli(), PARTITION_MINUTES - 599, Aggregation.MIN)).get();
  143 +// log.debug("Fetched records {}", localDateTime);
  144 +// List<TsKvEntry> expected = entries.subList(600, PARTITION_MINUTES);
  145 +// assertEquals(expected.size(), list.size());
  146 +// assertEquals(expected, list);
  147 +// }
  148 +
127 149
128 private void saveEntries(DeviceId deviceId, long ts) throws ExecutionException, InterruptedException { 150 private void saveEntries(DeviceId deviceId, long ts) throws ExecutionException, InterruptedException {
129 tsService.save(DataConstants.DEVICE, deviceId, toTsEntry(ts, stringKvEntry)).get(); 151 tsService.save(DataConstants.DEVICE, deviceId, toTsEntry(ts, stringKvEntry)).get();
@@ -2,7 +2,7 @@ cassandra.cluster_name=Thingsboard Cluster @@ -2,7 +2,7 @@ cassandra.cluster_name=Thingsboard Cluster
2 2
3 cassandra.keyspace_name=thingsboard 3 cassandra.keyspace_name=thingsboard
4 4
5 -cassandra.url=127.0.0.1:9142 5 +cassandra.url=127.0.0.1:9042
6 6
7 cassandra.ssl=false 7 cassandra.ssl=false
8 8
@@ -47,3 +47,7 @@ cassandra.query.default_fetch_size=2000 @@ -47,3 +47,7 @@ cassandra.query.default_fetch_size=2000
47 cassandra.query.ts_key_value_partitioning=HOURS 47 cassandra.query.ts_key_value_partitioning=HOURS
48 48
49 cassandra.query.max_limit_per_request=1000 49 cassandra.query.max_limit_per_request=1000
  50 +
  51 +cassandra.query.read_result_processing_threads=3
  52 +
  53 +cassandra.query.min_read_step=100
@@ -84,6 +84,8 @@ public interface PluginContext { @@ -84,6 +84,8 @@ public interface PluginContext {
84 84
85 List<TsKvEntry> loadTimeseries(DeviceId deviceId, TsKvQuery query); 85 List<TsKvEntry> loadTimeseries(DeviceId deviceId, TsKvQuery query);
86 86
  87 + void loadTimeseries(DeviceId deviceId, TsKvQuery query, PluginCallback<List<TsKvEntry>> callback);
  88 +
87 void loadLatestTimeseries(DeviceId deviceId, Collection<String> keys, PluginCallback<List<TsKvEntry>> callback); 89 void loadLatestTimeseries(DeviceId deviceId, Collection<String> keys, PluginCallback<List<TsKvEntry>> callback);
88 90
89 void loadLatestTimeseries(DeviceId deviceId, PluginCallback<List<TsKvEntry>> callback); 91 void loadLatestTimeseries(DeviceId deviceId, PluginCallback<List<TsKvEntry>> callback);
@@ -95,8 +95,9 @@ public class TelemetryRestMsgHandler extends DefaultRestMsgHandler { @@ -95,8 +95,9 @@ public class TelemetryRestMsgHandler extends DefaultRestMsgHandler {
95 Optional<Integer> limit = request.getIntParamValue("limit"); 95 Optional<Integer> limit = request.getIntParamValue("limit");
96 Map<String, List<TsData>> data = new LinkedHashMap<>(); 96 Map<String, List<TsData>> data = new LinkedHashMap<>();
97 for (String key : keys.split(",")) { 97 for (String key : keys.split(",")) {
98 - List<TsKvEntry> entries = ctx.loadTimeseries(deviceId, new BaseTsKvQuery(key, startTs, endTs, limit));  
99 - data.put(key, entries.stream().map(v -> new TsData(v.getTs(), v.getValueAsString())).collect(Collectors.toList())); 98 + //TODO: refactoring
  99 +// List<TsKvEntry> entries = ctx.loadTimeseries(deviceId, new BaseTsKvQuery(key, startTs, endTs, limit));
  100 +// data.put(key, entries.stream().map(v -> new TsData(v.getTs(), v.getValueAsString())).collect(Collectors.toList()));
100 } 101 }
101 msg.getResponseHolder().setResult(new ResponseEntity<>(data, HttpStatus.OK)); 102 msg.getResponseHolder().setResult(new ResponseEntity<>(data, HttpStatus.OK));
102 } else if ("attributes".equals(entity)) { 103 } else if ("attributes".equals(entity)) {