简柏林 / thingskit · Commits

Commit f6bc0791f10dc20260ea04dec20429c93ed18840

Authored by Andrew Shvayka
1 parent 406438c8

Aggregation

Inline Side-by-side
Showing 15 changed files with 494 additions and 176 deletions
1 1 /**
2 2 * Copyright © 2016-2017 The Thingsboard Authors
3   - *
  3 + * <p>
4 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 5 * you may not use this file except in compliance with the License.
6 6 * You may obtain a copy of the License at
7   - *
8   - * http://www.apache.org/licenses/LICENSE-2.0
9   - *
  7 + * <p>
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + * <p>
10 10 * Unless required by applicable law or agreed to in writing, software
11 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.actors.plugin;
17 17
18 18 import java.io.IOException;
19 19 import java.util.*;
  20 +import java.util.concurrent.ExecutionException;
20 21 import java.util.concurrent.Executor;
21 22 import java.util.concurrent.Executors;
22 23 import java.util.stream.Collectors;
... ... @@ -152,7 +153,19 @@ public final class PluginProcessingContext implements PluginContext {
152 153 @Override
153 154 public List<TsKvEntry> loadTimeseries(DeviceId deviceId, TsKvQuery query) {
154 155 validate(deviceId);
155   - return pluginCtx.tsService.find(DataConstants.DEVICE, deviceId, query);
  156 + try {
  157 + return pluginCtx.tsService.findAll(DataConstants.DEVICE, deviceId, query).get();
  158 + } catch (Exception e) {
  159 + log.error("TODO", e);
  160 + throw new RuntimeException(e);
  161 + }
  162 + }
  163 +
  164 + @Override
  165 + public void loadTimeseries(DeviceId deviceId, TsKvQuery query, PluginCallback<List<TsKvEntry>> callback) {
  166 + validate(deviceId);
  167 + ListenableFuture<List<TsKvEntry>> future = pluginCtx.tsService.findAll(DataConstants.DEVICE, deviceId, query);
  168 + Futures.addCallback(future, getCallback(callback, v -> v), executor);
156 169 }
157 170
158 171 @Override
... ... @@ -235,10 +248,10 @@ public final class PluginProcessingContext implements PluginContext {
235 248 };
236 249 }
237 250
238   - private <T> FutureCallback<ResultSet> getCallback(final PluginCallback<T> callback, Function<ResultSet, T> transformer) {
239   - return new FutureCallback<ResultSet>() {
  251 + private <T, R> FutureCallback<R> getCallback(final PluginCallback<T> callback, Function<R, T> transformer) {
  252 + return new FutureCallback<R>() {
240 253 @Override
241   - public void onSuccess(@Nullable ResultSet result) {
  254 + public void onSuccess(@Nullable R result) {
242 255 pluginCtx.self().tell(PluginCallbackMessage.onSuccess(callback, transformer.apply(result)), ActorRef.noSender());
243 256 }
244 257
... ...
  1 +package org.thingsboard.server.common.data.kv;
  2 +
  3 +/**
  4 + * Created by ashvayka on 20.02.17.
  5 + */
  6 +public enum Aggregation {
  7 +
  8 + MIN, MAX, AVG, SUM, COUNT, NONE;
  9 +
  10 +}
... ...
1 1 /**
2 2 * Copyright © 2016-2017 The Thingsboard Authors
3   - *
  3 + * <p>
4 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 5 * you may not use this file except in compliance with the License.
6 6 * You may obtain a copy of the License at
7   - *
8   - * http://www.apache.org/licenses/LICENSE-2.0
9   - *
  7 + * <p>
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + * <p>
10 10 * Unless required by applicable law or agreed to in writing, software
11 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
... ... @@ -15,59 +15,27 @@
15 15 */
16 16 package org.thingsboard.server.common.data.kv;
17 17
18   -import java.util.Optional;
  18 +import lombok.Data;
19 19
  20 +@Data
20 21 public class BaseTsKvQuery implements TsKvQuery {
21 22
22   - private String key;
23   - private Optional<Long> startTs;
24   - private Optional<Long> endTs;
25   - private Optional<Integer> limit;
  23 + private final String key;
  24 + private final long startTs;
  25 + private final long endTs;
  26 + private final int limit;
  27 + private final Aggregation aggregation;
26 28
27   - public BaseTsKvQuery(String key, Optional<Long> startTs, Optional<Long> endTs, Optional<Integer> limit) {
  29 + public BaseTsKvQuery(String key, long startTs, long endTs, int limit, Aggregation aggregation) {
28 30 this.key = key;
29 31 this.startTs = startTs;
30 32 this.endTs = endTs;
31 33 this.limit = limit;
32   - }
33   -
34   - public BaseTsKvQuery(String key, Long startTs, Long endTs, Integer limit) {
35   - this(key, Optional.ofNullable(startTs), Optional.ofNullable(endTs), Optional.ofNullable(limit));
36   - }
37   -
38   - public BaseTsKvQuery(String key, Long startTs, Integer limit) {
39   - this(key, startTs, null, limit);
40   - }
41   -
42   - public BaseTsKvQuery(String key, Long startTs, Long endTs) {
43   - this(key, startTs, endTs, null);
44   - }
45   -
46   - public BaseTsKvQuery(String key, Long startTs) {
47   - this(key, startTs, null, null);
  34 + this.aggregation = aggregation;
48 35 }
49 36
50   - public BaseTsKvQuery(String key, Integer limit) {
51   - this(key, null, null, limit);
  37 + public BaseTsKvQuery(String key, long startTs, long endTs) {
  38 + this(key, startTs, endTs, 1, Aggregation.AVG);
52 39 }
53 40
54   - @Override
55   - public String getKey() {
56   - return key;
57   - }
58   -
59   - @Override
60   - public Optional<Long> getStartTs() {
61   - return startTs;
62   - }
63   -
64   - @Override
65   - public Optional<Long> getEndTs() {
66   - return endTs;
67   - }
68   -
69   - @Override
70   - public Optional<Integer> getLimit() {
71   - return limit;
72   - }
73 41 }
... ...
... ... @@ -21,10 +21,12 @@ public interface TsKvQuery {
21 21
22 22 String getKey();
23 23
24   - Optional<Long> getStartTs();
  24 + long getStartTs();
25 25
26   - Optional<Long> getEndTs();
  26 + long getEndTs();
27 27
28   - Optional<Integer> getLimit();
  28 + int getLimit();
  29 +
  30 + Aggregation getAggregation();
29 31
30 32 }
... ...
1 1 /**
2 2 * Copyright © 2016-2017 The Thingsboard Authors
3   - *
  3 + * <p>
4 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 5 * you may not use this file except in compliance with the License.
6 6 * You may obtain a copy of the License at
7   - *
8   - * http://www.apache.org/licenses/LICENSE-2.0
9   - *
  7 + * <p>
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + * <p>
10 10 * Unless required by applicable law or agreed to in writing, software
11 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
... ... @@ -18,14 +18,15 @@ package org.thingsboard.server.dao.model;
18 18 import java.util.UUID;
19 19
20 20 import com.datastax.driver.core.utils.UUIDs;
  21 +import org.apache.commons.lang3.ArrayUtils;
21 22
22 23 public class ModelConstants {
23 24
24 25 private ModelConstants() {
25 26 }
26   -
  27 +
27 28 public static UUID NULL_UUID = UUIDs.startOf(0);
28   -
  29 +
29 30 /**
30 31 * Generic constants.
31 32 */
... ... @@ -38,7 +39,7 @@ public class ModelConstants {
38 39 public static final String ALIAS_PROPERTY = "alias";
39 40 public static final String SEARCH_TEXT_PROPERTY = "search_text";
40 41 public static final String ADDITIONAL_INFO_PROPERTY = "additional_info";
41   -
  42 +
42 43 /**
43 44 * Cassandra user constants.
44 45 */
... ... @@ -50,11 +51,11 @@ public class ModelConstants {
50 51 public static final String USER_FIRST_NAME_PROPERTY = "first_name";
51 52 public static final String USER_LAST_NAME_PROPERTY = "last_name";
52 53 public static final String USER_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY;
53   -
  54 +
54 55 public static final String USER_BY_EMAIL_COLUMN_FAMILY_NAME = "user_by_email";
55 56 public static final String USER_BY_TENANT_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "user_by_tenant_and_search_text";
56 57 public static final String USER_BY_CUSTOMER_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "user_by_customer_and_search_text";
57   -
  58 +
58 59 /**
59 60 * Cassandra user_credentials constants.
60 61 */
... ... @@ -64,20 +65,20 @@ public class ModelConstants {
64 65 public static final String USER_CREDENTIALS_PASSWORD_PROPERTY = "password";
65 66 public static final String USER_CREDENTIALS_ACTIVATE_TOKEN_PROPERTY = "activate_token";
66 67 public static final String USER_CREDENTIALS_RESET_TOKEN_PROPERTY = "reset_token";
67   -
  68 +
68 69 public static final String USER_CREDENTIALS_BY_USER_COLUMN_FAMILY_NAME = "user_credentials_by_user";
69 70 public static final String USER_CREDENTIALS_BY_ACTIVATE_TOKEN_COLUMN_FAMILY_NAME = "user_credentials_by_activate_token";
70 71 public static final String USER_CREDENTIALS_BY_RESET_TOKEN_COLUMN_FAMILY_NAME = "user_credentials_by_reset_token";
71   -
  72 +
72 73 /**
73 74 * Cassandra admin_settings constants.
74 75 */
75 76 public static final String ADMIN_SETTINGS_COLUMN_FAMILY_NAME = "admin_settings";
76 77 public static final String ADMIN_SETTINGS_KEY_PROPERTY = "key";
77 78 public static final String ADMIN_SETTINGS_JSON_VALUE_PROPERTY = "json_value";
78   -
  79 +
79 80 public static final String ADMIN_SETTINGS_BY_KEY_COLUMN_FAMILY_NAME = "admin_settings_by_key";
80   -
  81 +
81 82 /**
82 83 * Cassandra contact constants.
83 84 */
... ... @@ -97,9 +98,9 @@ public class ModelConstants {
97 98 public static final String TENANT_TITLE_PROPERTY = TITLE_PROPERTY;
98 99 public static final String TENANT_REGION_PROPERTY = "region";
99 100 public static final String TENANT_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY;
100   -
  101 +
101 102 public static final String TENANT_BY_REGION_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "tenant_by_region_and_search_text";
102   -
  103 +
103 104 /**
104 105 * Cassandra customer constants.
105 106 */
... ... @@ -107,9 +108,9 @@ public class ModelConstants {
107 108 public static final String CUSTOMER_TENANT_ID_PROPERTY = TENTANT_ID_PROPERTY;
108 109 public static final String CUSTOMER_TITLE_PROPERTY = TITLE_PROPERTY;
109 110 public static final String CUSTOMER_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY;
110   -
  111 +
111 112 public static final String CUSTOMER_BY_TENANT_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "customer_by_tenant_and_search_text";
112   -
  113 +
113 114 /**
114 115 * Cassandra device constants.
115 116 */
... ... @@ -118,12 +119,12 @@ public class ModelConstants {
118 119 public static final String DEVICE_CUSTOMER_ID_PROPERTY = CUSTOMER_ID_PROPERTY;
119 120 public static final String DEVICE_NAME_PROPERTY = "name";
120 121 public static final String DEVICE_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY;
121   -
  122 +
122 123 public static final String DEVICE_BY_TENANT_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "device_by_tenant_and_search_text";
123 124 public static final String DEVICE_BY_CUSTOMER_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "device_by_customer_and_search_text";
124 125 public static final String DEVICE_BY_TENANT_AND_NAME_VIEW_NAME = "device_by_tenant_and_name";
125 126
126   -
  127 +
127 128 /**
128 129 * Cassandra device_credentials constants.
129 130 */
... ... @@ -132,7 +133,7 @@ public class ModelConstants {
132 133 public static final String DEVICE_CREDENTIALS_CREDENTIALS_TYPE_PROPERTY = "credentials_type";
133 134 public static final String DEVICE_CREDENTIALS_CREDENTIALS_ID_PROPERTY = "credentials_id";
134 135 public static final String DEVICE_CREDENTIALS_CREDENTIALS_VALUE_PROPERTY = "credentials_value";
135   -
  136 +
136 137 public static final String DEVICE_CREDENTIALS_BY_DEVICE_COLUMN_FAMILY_NAME = "device_credentials_by_device";
137 138 public static final String DEVICE_CREDENTIALS_BY_CREDENTIALS_ID_COLUMN_FAMILY_NAME = "device_credentials_by_credentials_id";
138 139
... ... @@ -203,9 +204,9 @@ public class ModelConstants {
203 204 public static final String COMPONENT_DESCRIPTOR_BY_SCOPE_TYPE_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "component_desc_by_scope_type_search_text";
204 205 public static final String COMPONENT_DESCRIPTOR_BY_ID = "component_desc_by_id";
205 206
206   - /**
207   - * Cassandra rule metadata constants.
208   - */
  207 + /**
  208 + * Cassandra rule metadata constants.
  209 + */
209 210 public static final String RULE_COLUMN_FAMILY_NAME = "rule";
210 211 public static final String RULE_TENANT_ID_PROPERTY = TENTANT_ID_PROPERTY;
211 212 public static final String RULE_NAME_PROPERTY = "name";
... ... @@ -259,4 +260,31 @@ public class ModelConstants {
259 260 public static final String STRING_VALUE_COLUMN = "str_v";
260 261 public static final String LONG_VALUE_COLUMN = "long_v";
261 262 public static final String DOUBLE_VALUE_COLUMN = "dbl_v";
  263 +
  264 + public static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN)};
  265 +
  266 + public static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN,};
  267 + public static final String[] MIN_AGGREGATION_COLUMNS = ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS,
  268 + new String[]{min(LONG_VALUE_COLUMN), min(DOUBLE_VALUE_COLUMN), min(BOOLEAN_VALUE_COLUMN), min(STRING_VALUE_COLUMN)});
  269 + public static final String[] MAX_AGGREGATION_COLUMNS = ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS,
  270 + new String[]{max(LONG_VALUE_COLUMN), max(DOUBLE_VALUE_COLUMN), max(BOOLEAN_VALUE_COLUMN), max(STRING_VALUE_COLUMN)});
  271 + public static final String[] SUM_AGGREGATION_COLUMNS = ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS,
  272 + new String[]{sum(LONG_VALUE_COLUMN), sum(DOUBLE_VALUE_COLUMN)});
  273 + public static final String[] AVG_AGGREGATION_COLUMNS = ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS, SUM_AGGREGATION_COLUMNS);
  274 +
  275 + public static String min(String s) {
  276 + return "min(" + s + ")";
  277 + }
  278 +
  279 + public static String max(String s) {
  280 + return "max(" + s + ")";
  281 + }
  282 +
  283 + public static String sum(String s) {
  284 + return "sum(" + s + ")";
  285 + }
  286 +
  287 + public static String count(String s) {
  288 + return "count(" + s + ")";
  289 + }
262 290 }
... ...
  1 +package org.thingsboard.server.dao.timeseries;
  2 +
  3 +import com.datastax.driver.core.ResultSet;
  4 +import com.datastax.driver.core.Row;
  5 +import org.thingsboard.server.common.data.kv.*;
  6 +
  7 +import javax.annotation.Nullable;
  8 +import java.util.List;
  9 +import java.util.Optional;
  10 +
  11 +/**
  12 + * Created by ashvayka on 20.02.17.
  13 + */
  14 +public class AggregatePartitionsFunction implements com.google.common.base.Function<List<ResultSet>, Optional<TsKvEntry>> {
  15 +
  16 + private static final int LONG_CNT_POS = 0;
  17 + private static final int DOUBLE_CNT_POS = 1;
  18 + private static final int BOOL_CNT_POS = 2;
  19 + private static final int STR_CNT_POS = 3;
  20 + private static final int LONG_POS = 4;
  21 + private static final int DOUBLE_POS = 5;
  22 + private static final int BOOL_POS = 6;
  23 + private static final int STR_POS = 7;
  24 +
  25 + private final Aggregation aggregation;
  26 + private final String key;
  27 + private final long ts;
  28 +
  29 + public AggregatePartitionsFunction(Aggregation aggregation, String key, long ts) {
  30 + this.aggregation = aggregation;
  31 + this.key = key;
  32 + this.ts = ts;
  33 + }
  34 +
  35 + @Nullable
  36 + @Override
  37 + public Optional<TsKvEntry> apply(@Nullable List<ResultSet> rsList) {
  38 + if (rsList == null || rsList.size() == 0) {
  39 + return Optional.empty();
  40 + }
  41 + long count = 0;
  42 + DataType dataType = null;
  43 +
  44 + Boolean bValue = null;
  45 + String sValue = null;
  46 + Double dValue = null;
  47 + Long lValue = null;
  48 +
  49 + for (ResultSet rs : rsList) {
  50 + for (Row row : rs.all()) {
  51 + long curCount;
  52 +
  53 + Long curLValue = null;
  54 + Double curDValue = null;
  55 + Boolean curBValue = null;
  56 + String curSValue = null;
  57 +
  58 + long longCount = row.getLong(LONG_CNT_POS);
  59 + long doubleCount = row.getLong(DOUBLE_CNT_POS);
  60 + long boolCount = row.getLong(BOOL_CNT_POS);
  61 + long strCount = row.getLong(STR_CNT_POS);
  62 +
  63 + if (longCount > 0) {
  64 + dataType = DataType.LONG;
  65 + curCount = longCount;
  66 + curLValue = getLongValue(row);
  67 + } else if (doubleCount > 0) {
  68 + dataType = DataType.DOUBLE;
  69 + curCount = doubleCount;
  70 + curDValue = getDoubleValue(row);
  71 + } else if (boolCount > 0) {
  72 + dataType = DataType.BOOLEAN;
  73 + curCount = boolCount;
  74 + curBValue = getBooleanValue(row);
  75 + } else if (strCount > 0) {
  76 + dataType = DataType.STRING;
  77 + curCount = strCount;
  78 + curSValue = getStringValue(row);
  79 + } else {
  80 + continue;
  81 + }
  82 +
  83 + if (aggregation == Aggregation.COUNT) {
  84 + count += curCount;
  85 + } else if (aggregation == Aggregation.AVG || aggregation == Aggregation.SUM) {
  86 + count += curCount;
  87 + dValue = dValue == null ? curDValue : dValue + curDValue;
  88 + lValue = lValue == null ? curLValue : lValue + curLValue;
  89 + } else if (aggregation == Aggregation.MIN) {
  90 + if (curDValue != null) {
  91 + dValue = dValue == null ? curDValue : Math.min(dValue, curDValue);
  92 + } else if (curLValue != null) {
  93 + lValue = lValue == null ? curLValue : Math.min(lValue, curLValue);
  94 + } else if (curBValue != null) {
  95 + bValue = bValue == null ? curBValue : bValue && curBValue;
  96 + } else if (curSValue != null) {
  97 + if (sValue == null || curSValue.compareTo(sValue) < 0) {
  98 + sValue = curSValue;
  99 + }
  100 + }
  101 + } else if (aggregation == Aggregation.MAX) {
  102 + if (curDValue != null) {
  103 + dValue = dValue == null ? curDValue : Math.max(dValue, curDValue);
  104 + } else if (curLValue != null) {
  105 + lValue = lValue == null ? curLValue : Math.max(lValue, curLValue);
  106 + } else if (curBValue != null) {
  107 + bValue = bValue == null ? curBValue : bValue || curBValue;
  108 + } else if (curSValue != null) {
  109 + if (sValue == null || curSValue.compareTo(sValue) > 0) {
  110 + sValue = curSValue;
  111 + }
  112 + }
  113 + }
  114 + }
  115 + }
  116 + if (dataType == null) {
  117 + return Optional.empty();
  118 + } else if (aggregation == Aggregation.COUNT) {
  119 + return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, (long) count)));
  120 + } else if (aggregation == Aggregation.AVG || aggregation == Aggregation.SUM) {
  121 + if (count == 0 || (dataType == DataType.DOUBLE && dValue == null) || (dataType == DataType.LONG && lValue == null)) {
  122 + return Optional.empty();
  123 + } else if (dataType == DataType.DOUBLE) {
  124 + return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.SUM ? dValue : (dValue / count))));
  125 + } else if (dataType == DataType.LONG) {
  126 + return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, aggregation == Aggregation.SUM ? lValue : (lValue / count))));
  127 + }
  128 + } else if (aggregation == Aggregation.MIN || aggregation == Aggregation.MAX) {
  129 + if (dataType == DataType.DOUBLE) {
  130 + return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, dValue)));
  131 + } else if (dataType == DataType.LONG) {
  132 + return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, lValue)));
  133 + } else if (dataType == DataType.STRING) {
  134 + return Optional.of(new BasicTsKvEntry(ts, new StringDataEntry(key, sValue)));
  135 + } else {
  136 + return Optional.of(new BasicTsKvEntry(ts, new BooleanDataEntry(key, bValue)));
  137 + }
  138 + }
  139 + return null;
  140 + }
  141 +
  142 + private Boolean getBooleanValue(Row row) {
  143 + if (aggregation == Aggregation.MIN || aggregation == Aggregation.MAX) {
  144 + return row.getBool(BOOL_POS);
  145 + } else {
  146 + return null;
  147 + }
  148 + }
  149 +
  150 + private String getStringValue(Row row) {
  151 + if (aggregation == Aggregation.MIN || aggregation == Aggregation.MAX) {
  152 + return row.getString(STR_POS);
  153 + } else {
  154 + return null;
  155 + }
  156 + }
  157 +
  158 + private Long getLongValue(Row row) {
  159 + if (aggregation == Aggregation.MIN || aggregation == Aggregation.MAX
  160 + || aggregation == Aggregation.SUM || aggregation == Aggregation.AVG) {
  161 + return row.getLong(LONG_POS);
  162 + } else {
  163 + return null;
  164 + }
  165 + }
  166 +
  167 + private Double getDoubleValue(Row row) {
  168 + if (aggregation == Aggregation.MIN || aggregation == Aggregation.MAX
  169 + || aggregation == Aggregation.SUM || aggregation == Aggregation.AVG) {
  170 + return row.getDouble(DOUBLE_POS);
  171 + } else {
  172 + return null;
  173 + }
  174 + }
  175 +}
... ...
1 1 /**
2 2 * Copyright © 2016-2017 The Thingsboard Authors
3   - *
  3 + * <p>
4 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 5 * you may not use this file except in compliance with the License.
6 6 * You may obtain a copy of the License at
7   - *
8   - * http://www.apache.org/licenses/LICENSE-2.0
9   - *
  7 + * <p>
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + * <p>
10 10 * Unless required by applicable law or agreed to in writing, software
11 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
... ... @@ -18,6 +18,10 @@ package org.thingsboard.server.dao.timeseries;
18 18 import com.datastax.driver.core.*;
19 19 import com.datastax.driver.core.querybuilder.QueryBuilder;
20 20 import com.datastax.driver.core.querybuilder.Select;
  21 +import com.google.common.base.Function;
  22 +import com.google.common.util.concurrent.AsyncFunction;
  23 +import com.google.common.util.concurrent.Futures;
  24 +import com.google.common.util.concurrent.ListenableFuture;
21 25 import lombok.extern.slf4j.Slf4j;
22 26 import org.springframework.beans.factory.annotation.Value;
23 27 import org.springframework.stereotype.Component;
... ... @@ -26,7 +30,16 @@ import org.thingsboard.server.common.data.kv.DataType;
26 30 import org.thingsboard.server.dao.AbstractDao;
27 31 import org.thingsboard.server.dao.model.ModelConstants;
28 32
  33 +import javax.annotation.Nullable;
  34 +import javax.annotation.PostConstruct;
  35 +import javax.annotation.PreDestroy;
  36 +import java.time.Instant;
  37 +import java.time.LocalDateTime;
  38 +import java.time.ZoneOffset;
29 39 import java.util.*;
  40 +import java.util.concurrent.ExecutorService;
  41 +import java.util.concurrent.Executors;
  42 +import java.util.stream.Collectors;
30 43
31 44 import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
32 45 import static com.datastax.driver.core.querybuilder.QueryBuilder.select;
... ... @@ -41,48 +54,136 @@ public class BaseTimeseriesDao extends AbstractDao implements TimeseriesDao {
41 54 @Value("${cassandra.query.max_limit_per_request}")
42 55 protected Integer maxLimitPerRequest;
43 56
  57 + @Value("${cassandra.query.read_result_processing_threads}")
  58 + private int readResultsProcessingThreads;
  59 +
  60 + @Value("${cassandra.query.min_read_step}")
  61 + private int minReadStep;
  62 +
  63 + @Value("${cassandra.query.ts_key_value_partitioning}")
  64 + private String partitioning;
  65 +
  66 + private TsPartitionDate tsFormat;
  67 +
  68 + private ExecutorService readResultsProcessingExecutor;
  69 +
44 70 private PreparedStatement partitionInsertStmt;
45 71 private PreparedStatement[] latestInsertStmts;
46 72 private PreparedStatement[] saveStmts;
  73 + private PreparedStatement[] fetchStmts;
47 74 private PreparedStatement findLatestStmt;
48 75 private PreparedStatement findAllLatestStmt;
49 76
  77 + @PostConstruct
  78 + public void init() {
  79 + getFetchStmt(Aggregation.NONE);
  80 + readResultsProcessingExecutor = Executors.newFixedThreadPool(readResultsProcessingThreads);
  81 + Optional<TsPartitionDate> partition = TsPartitionDate.parse(partitioning);
  82 + if (partition.isPresent()) {
  83 + tsFormat = partition.get();
  84 + } else {
  85 + log.warn("Incorrect configuration of partitioning {}", partitioning);
  86 + throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!");
  87 + }
  88 + }
  89 +
  90 + @PreDestroy
  91 + public void stop() {
  92 + if (readResultsProcessingExecutor != null) {
  93 + readResultsProcessingExecutor.shutdownNow();
  94 + }
  95 + }
  96 +
50 97 @Override
51   - public List<TsKvEntry> find(String entityType, UUID entityId, TsKvQuery query, Optional<Long> minPartition, Optional<Long> maxPartition) {
52   - List<Row> rows = Collections.emptyList();
53   - Long[] parts = fetchPartitions(entityType, entityId, query.getKey(), minPartition, maxPartition);
54   - int partsLength = parts.length;
55   - if (parts != null && partsLength > 0) {
56   - int limit = maxLimitPerRequest;
57   - Optional<Integer> lim = query.getLimit();
58   - if (lim.isPresent() && lim.get() < maxLimitPerRequest) {
59   - limit = lim.get();
60   - }
  98 + public long toPartitionTs(long ts) {
  99 + LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC);
  100 + return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli();
  101 + }
61 102
62   - rows = new ArrayList<>(limit);
63   - int lastIdx = partsLength - 1;
64   - for (int i = 0; i < partsLength; i++) {
65   - int currentLimit;
66   - if (rows.size() >= limit) {
67   - break;
68   - } else {
69   - currentLimit = limit - rows.size();
  103 +
  104 + private static String[] getFetchColumnNames(Aggregation aggregation) {
  105 + switch (aggregation) {
  106 + case NONE:
  107 + return ModelConstants.NONE_AGGREGATION_COLUMNS;
  108 + case MIN:
  109 + return ModelConstants.MIN_AGGREGATION_COLUMNS;
  110 + case MAX:
  111 + return ModelConstants.MAX_AGGREGATION_COLUMNS;
  112 + case SUM:
  113 + return ModelConstants.SUM_AGGREGATION_COLUMNS;
  114 + case COUNT:
  115 + return ModelConstants.COUNT_AGGREGATION_COLUMNS;
  116 + case AVG:
  117 + return ModelConstants.AVG_AGGREGATION_COLUMNS;
  118 + default:
  119 + throw new RuntimeException("Aggregation type: " + aggregation + " is not supported!");
  120 + }
  121 + }
  122 +
  123 + @Override
  124 + public ListenableFuture<List<TsKvEntry>> findAllAsync(String entityType, UUID entityId, TsKvQuery query, long minPartition, long maxPartition) {
  125 + if (query.getAggregation() == Aggregation.NONE) {
  126 + //TODO:
  127 + return null;
  128 + } else {
  129 + long step = Math.max((query.getEndTs() - query.getStartTs()) / query.getLimit(), minReadStep);
  130 + long stepTs = query.getStartTs();
  131 + List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>();
  132 + while (stepTs < query.getEndTs()) {
  133 + long startTs = stepTs;
  134 + long endTs = stepTs + step;
  135 + TsKvQuery subQuery = new BaseTsKvQuery(query.getKey(), startTs, endTs, 1, query.getAggregation());
  136 + futures.add(findAndAggregateAsync(entityType, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs)));
  137 + stepTs = endTs;
  138 + }
  139 + ListenableFuture<List<Optional<TsKvEntry>>> future = Futures.allAsList(futures);
  140 + return Futures.transform(future, new Function<List<Optional<TsKvEntry>>, List<TsKvEntry>>() {
  141 + @Nullable
  142 + @Override
  143 + public List<TsKvEntry> apply(@Nullable List<Optional<TsKvEntry>> input) {
  144 + return input.stream().filter(v -> v.isPresent()).map(v -> v.get()).collect(Collectors.toList());
70 145 }
71   - Long partition = parts[i];
72   - Select.Where where = select().from(ModelConstants.TS_KV_CF).where(eq(ModelConstants.ENTITY_TYPE_COLUMN, entityType))
73   - .and(eq(ModelConstants.ENTITY_ID_COLUMN, entityId))
74   - .and(eq(ModelConstants.KEY_COLUMN, query.getKey()))
75   - .and(eq(ModelConstants.PARTITION_COLUMN, partition));
76   - if (i == 0 && query.getStartTs().isPresent()) {
77   - where.and(QueryBuilder.gt(ModelConstants.TS_COLUMN, query.getStartTs().get()));
78   - } else if (i == lastIdx && query.getEndTs().isPresent()) {
79   - where.and(QueryBuilder.lte(ModelConstants.TS_COLUMN, query.getEndTs().get()));
  146 + });
  147 + }
  148 + }
  149 +
  150 + private ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(String entityType, UUID entityId, TsKvQuery query, long minPartition, long maxPartition) {
  151 + final Aggregation aggregation = query.getAggregation();
  152 + final long startTs = query.getStartTs();
  153 + final long endTs = query.getEndTs();
  154 + final long ts = startTs + (endTs - startTs) / 2;
  155 +
  156 + ResultSetFuture partitionsFuture = fetchPartitions(entityType, entityId, query.getKey(), minPartition, maxPartition);
  157 + com.google.common.base.Function<ResultSet, List<Long>> toArrayFunction = rows -> rows.all().stream()
  158 + .map(row -> row.getLong(ModelConstants.PARTITION_COLUMN)).collect(Collectors.toList());
  159 +
  160 + ListenableFuture<List<Long>> partitionsListFuture = Futures.transform(partitionsFuture, toArrayFunction, readResultsProcessingExecutor);
  161 +
  162 + AsyncFunction<List<Long>, List<ResultSet>> fetchChunksFunction = partitions -> {
  163 + try {
  164 + PreparedStatement proto = getFetchStmt(aggregation);
  165 + List<ResultSetFuture> futures = new ArrayList<>(partitions.size());
  166 + for (Long partition : partitions) {
  167 + BoundStatement stmt = proto.bind();
  168 + stmt.setString(0, entityType);
  169 + stmt.setUUID(1, entityId);
  170 + stmt.setString(2, query.getKey());
  171 + stmt.setLong(3, partition);
  172 + stmt.setLong(4, startTs);
  173 + stmt.setLong(5, endTs);
  174 + log.debug("Generated query [{}] for entityType {} and entityId {}", stmt, entityType, entityId);
  175 + futures.add(executeAsyncRead(stmt));
80 176 }
81   - where.limit(currentLimit);
82   - rows.addAll(executeRead(where).all());
  177 + return Futures.allAsList(futures);
  178 + } catch (Throwable e) {
  179 + log.error("Failed to fetch data", e);
  180 + throw e;
83 181 }
84   - }
85   - return convertResultToTsKvEntryList(rows);
  182 + };
  183 +
  184 + ListenableFuture<List<ResultSet>> aggregationChunks = Futures.transform(partitionsListFuture, fetchChunksFunction, readResultsProcessingExecutor);
  185 +
  186 + return Futures.transform(aggregationChunks, new AggregatePartitionsFunction(aggregation, query.getKey(), ts), readResultsProcessingExecutor);
86 187 }
87 188
88 189 @Override
... ... @@ -190,13 +291,12 @@ public class BaseTimeseriesDao extends AbstractDao implements TimeseriesDao {
190 291 * Select existing partitions from the table
191 292 * <code>{@link ModelConstants#TS_KV_PARTITIONS_CF}</code> for the given entity
192 293 */
193   - private Long[] fetchPartitions(String entityType, UUID entityId, String key, Optional<Long> minPartition, Optional<Long> maxPartition) {
  294 + private ResultSetFuture fetchPartitions(String entityType, UUID entityId, String key, long minPartition, long maxPartition) {
194 295 Select.Where select = QueryBuilder.select(ModelConstants.PARTITION_COLUMN).from(ModelConstants.TS_KV_PARTITIONS_CF).where(eq(ModelConstants.ENTITY_TYPE_COLUMN, entityType))
195 296 .and(eq(ModelConstants.ENTITY_ID_COLUMN, entityId)).and(eq(ModelConstants.KEY_COLUMN, key));
196   - minPartition.ifPresent(startTs -> select.and(QueryBuilder.gte(ModelConstants.PARTITION_COLUMN, minPartition.get())));
197   - maxPartition.ifPresent(endTs -> select.and(QueryBuilder.lte(ModelConstants.PARTITION_COLUMN, maxPartition.get())));
198   - ResultSet resultSet = executeRead(select);
199   - return resultSet.all().stream().map(row -> row.getLong(ModelConstants.PARTITION_COLUMN)).toArray(Long[]::new);
  297 + select.and(QueryBuilder.gte(ModelConstants.PARTITION_COLUMN, minPartition));
  298 + select.and(QueryBuilder.lte(ModelConstants.PARTITION_COLUMN, maxPartition));
  299 + return executeAsyncRead(select);
200 300 }
201 301
202 302 private PreparedStatement getSaveStmt(DataType dataType) {
... ... @@ -216,6 +316,23 @@ public class BaseTimeseriesDao extends AbstractDao implements TimeseriesDao {
216 316 return saveStmts[dataType.ordinal()];
217 317 }
218 318
  319 + private PreparedStatement getFetchStmt(Aggregation aggType) {
  320 + if (fetchStmts == null) {
  321 + fetchStmts = new PreparedStatement[Aggregation.values().length];
  322 + for (Aggregation type : Aggregation.values()) {
  323 + fetchStmts[type.ordinal()] = getSession().prepare("SELECT " +
  324 + String.join(", ", getFetchColumnNames(type)) + " FROM " + ModelConstants.TS_KV_CF
  325 + + " WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + " = ? "
  326 + + "AND " + ModelConstants.ENTITY_ID_COLUMN + " = ? "
  327 + + "AND " + ModelConstants.KEY_COLUMN + " = ? "
  328 + + "AND " + ModelConstants.PARTITION_COLUMN + " = ? "
  329 + + "AND " + ModelConstants.TS_COLUMN + " > ? "
  330 + + "AND " + ModelConstants.TS_COLUMN + " <= ?");
  331 + }
  332 + }
  333 + return fetchStmts[aggType.ordinal()];
  334 + }
  335 +
219 336 private PreparedStatement getLatestStmt(DataType dataType) {
220 337 if (latestInsertStmts == null) {
221 338 latestInsertStmts = new PreparedStatement[DataType.values().length];
... ...
1 1 /**
2 2 * Copyright © 2016-2017 The Thingsboard Authors
3   - *
  3 + * <p>
4 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 5 * you may not use this file except in compliance with the License.
6 6 * You may obtain a copy of the License at
7   - *
8   - * http://www.apache.org/licenses/LICENSE-2.0
9   - *
  7 + * <p>
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + * <p>
10 10 * Unless required by applicable law or agreed to in writing, software
11 11 * distributed under the License is distributed on an "AS IS" BASIS,
12 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
... ... @@ -23,21 +23,23 @@ import com.google.common.util.concurrent.Futures;
23 23 import com.google.common.util.concurrent.ListenableFuture;
24 24 import lombok.extern.slf4j.Slf4j;
25 25 import org.thingsboard.server.common.data.id.UUIDBased;
  26 +import org.thingsboard.server.common.data.kv.BaseTsKvQuery;
26 27 import org.thingsboard.server.common.data.kv.TsKvEntry;
27 28 import org.thingsboard.server.common.data.kv.TsKvQuery;
28 29 import org.thingsboard.server.dao.exception.IncorrectParameterException;
29   -import org.slf4j.Logger;
30   -import org.slf4j.LoggerFactory;
31 30 import org.springframework.beans.factory.annotation.Autowired;
32 31 import org.springframework.beans.factory.annotation.Value;
33 32 import org.springframework.stereotype.Service;
34 33 import org.thingsboard.server.dao.service.Validator;
35 34
36 35 import javax.annotation.PostConstruct;
  36 +import javax.annotation.PreDestroy;
37 37 import java.time.Instant;
38 38 import java.time.LocalDateTime;
39 39 import java.time.ZoneOffset;
40 40 import java.util.*;
  41 +import java.util.concurrent.ExecutorService;
  42 +import java.util.concurrent.Executors;
41 43
42 44 import static org.apache.commons.lang3.StringUtils.isBlank;
43 45
... ... @@ -50,38 +52,14 @@ public class BaseTimeseriesService implements TimeseriesService {
50 52
51 53 public static final int INSERTS_PER_ENTRY = 3;
52 54
53   - @Value("${cassandra.query.ts_key_value_partitioning}")
54   - private String partitioning;
55   -
56 55 @Autowired
57 56 private TimeseriesDao timeseriesDao;
58 57
59   - private TsPartitionDate tsFormat;
60   -
61   - @PostConstruct
62   - public void init() {
63   - Optional<TsPartitionDate> partition = TsPartitionDate.parse(partitioning);
64   - if (partition.isPresent()) {
65   - tsFormat = partition.get();
66   - } else {
67   - log.warn("Incorrect configuration of partitioning {}", partitioning);
68   - throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!");
69   - }
70   - }
71   -
72 58 @Override
73   - public List<TsKvEntry> find(String entityType, UUIDBased entityId, TsKvQuery query) {
  59 + public ListenableFuture<List<TsKvEntry>> findAll(String entityType, UUIDBased entityId, TsKvQuery query) {
74 60 validate(entityType, entityId);
75 61 validate(query);
76   - return timeseriesDao.find(entityType, entityId.getId(), query, toPartitionTs(query.getStartTs()), toPartitionTs(query.getEndTs()));
77   - }
78   -
79   - private Optional<Long> toPartitionTs(Optional<Long> ts) {
80   - if (ts.isPresent()) {
81   - return Optional.of(toPartitionTs(ts.get()));
82   - } else {
83   - return Optional.empty();
84   - }
  62 + return timeseriesDao.findAllAsync(entityType, entityId.getId(), query, timeseriesDao.toPartitionTs(query.getStartTs()), timeseriesDao.toPartitionTs(query.getEndTs()));
85 63 }
86 64
87 65 @Override
... ... @@ -106,7 +84,7 @@ public class BaseTimeseriesService implements TimeseriesService {
106 84 throw new IncorrectParameterException("Key value entry can't be null");
107 85 }
108 86 UUID uid = entityId.getId();
109   - long partitionTs = toPartitionTs(tsKvEntry.getTs());
  87 + long partitionTs = timeseriesDao.toPartitionTs(tsKvEntry.getTs());
110 88
111 89 List<ResultSetFuture> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY);
112 90 saveAndRegisterFutures(futures, entityType, tsKvEntry, uid, partitionTs);
... ... @@ -122,7 +100,7 @@ public class BaseTimeseriesService implements TimeseriesService {
122 100 throw new IncorrectParameterException("Key value entry can't be null");
123 101 }
124 102 UUID uid = entityId.getId();
125   - long partitionTs = toPartitionTs(tsKvEntry.getTs());
  103 + long partitionTs = timeseriesDao.toPartitionTs(tsKvEntry.getTs());
126 104 saveAndRegisterFutures(futures, entityType, tsKvEntry, uid, partitionTs);
127 105 }
128 106 return Futures.allAsList(futures);
... ... @@ -144,14 +122,6 @@ public class BaseTimeseriesService implements TimeseriesService {
144 122 futures.add(timeseriesDao.save(entityType, uid, partitionTs, tsKvEntry));
145 123 }
146 124
147   - private long toPartitionTs(long ts) {
148   - LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC);
149   -
150   - LocalDateTime parititonTime = tsFormat.truncatedTo(time);
151   -
152   - return parititonTime.toInstant(ZoneOffset.UTC).toEpochMilli();
153   - }
154   -
155 125 private static void validate(String entityType, UUIDBased entityId) {
156 126 Validator.validateString(entityType, "Incorrect entityType " + entityType);
157 127 Validator.validateId(entityId, "Incorrect entityId " + entityId);
... ... @@ -163,5 +133,6 @@ public class BaseTimeseriesService implements TimeseriesService {
163 133 } else if (isBlank(query.getKey())) {
164 134 throw new IncorrectParameterException("Incorrect TsKvQuery. Key can't be empty");
165 135 }
  136 + //TODO: add validation of all params
166 137 }
167 138 }
... ...
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.timeseries;
17 17
18 18 import com.datastax.driver.core.ResultSetFuture;
19 19 import com.datastax.driver.core.Row;
  20 +import com.google.common.util.concurrent.ListenableFuture;
20 21 import org.thingsboard.server.common.data.kv.TsKvEntry;
21 22 import org.thingsboard.server.common.data.kv.TsKvQuery;
22 23
... ... @@ -30,7 +31,11 @@ import java.util.UUID;
30 31 */
31 32 public interface TimeseriesDao {
32 33
33   - List<TsKvEntry> find(String entityType, UUID entityId, TsKvQuery query, Optional<Long> minPartition, Optional<Long> maxPartition);
  34 + long toPartitionTs(long ts);
  35 +
  36 + ListenableFuture<List<TsKvEntry>> findAllAsync(String entityType, UUID entityId, TsKvQuery query, long minPartition, long maxPartition);
  37 +
  38 +// List<TsKvEntry> find(String entityType, UUID entityId, TsKvQuery query, Optional<Long> minPartition, Optional<Long> maxPartition);
34 39
35 40 ResultSetFuture findLatest(String entityType, UUID entityId, String key);
36 41
... ...
... ... @@ -19,6 +19,7 @@ import com.datastax.driver.core.ResultSet;
19 19 import com.datastax.driver.core.ResultSetFuture;
20 20 import com.datastax.driver.core.Row;
21 21 import com.google.common.util.concurrent.ListenableFuture;
  22 +import org.thingsboard.server.common.data.id.DeviceId;
22 23 import org.thingsboard.server.common.data.id.UUIDBased;
23 24 import org.thingsboard.server.common.data.kv.TsKvEntry;
24 25 import org.thingsboard.server.common.data.kv.TsKvQuery;
... ... @@ -32,8 +33,7 @@ import java.util.Set;
32 33 */
33 34 public interface TimeseriesService {
34 35
35   - //TODO: Replace this with async operation
36   - List<TsKvEntry> find(String entityType, UUIDBased entityId, TsKvQuery query);
  36 + ListenableFuture<List<TsKvEntry>> findAll(String entityType, UUIDBased entityId, TsKvQuery query);
37 37
38 38 ListenableFuture<List<ResultSet>> findLatest(String entityType, UUIDBased entityId, Collection<String> keys);
39 39
... ...
... ... @@ -25,11 +25,11 @@ import java.util.Arrays;
25 25
26 26 @RunWith(ClasspathSuite.class)
27 27 @ClassnameFilters({
28   - "org.thingsboard.server.dao.service.*Test",
29   - "org.thingsboard.server.dao.kv.*Test",
30   - "org.thingsboard.server.dao.plugin.*Test",
31   - "org.thingsboard.server.dao.rule.*Test",
32   - "org.thingsboard.server.dao.attributes.*Test",
  28 +// "org.thingsboard.server.dao.service.*Test",
  29 +// "org.thingsboard.server.dao.kv.*Test",
  30 +// "org.thingsboard.server.dao.plugin.*Test",
  31 +// "org.thingsboard.server.dao.rule.*Test",
  32 +// "org.thingsboard.server.dao.attributes.*Test",
33 33 "org.thingsboard.server.dao.timeseries.*Test"
34 34 })
35 35 public class DaoTestSuite {
... ...
... ... @@ -116,14 +116,36 @@ public class TimeseriesServiceTest extends AbstractServiceTest {
116 116 entries.add(tsKvEntry);
117 117 }
118 118 log.debug("Saved all records {}", localDateTime);
119   - List<TsKvEntry> list = tsService.find(DataConstants.DEVICE, deviceId, new BaseTsKvQuery(STRING_KEY, entries.get(599).getTs(),
120   - LocalDateTime.now(ZoneOffset.UTC).toInstant(ZoneOffset.UTC).toEpochMilli()));
  119 + List<TsKvEntry> list = tsService.findAll(DataConstants.DEVICE, deviceId, new BaseTsKvQuery(STRING_KEY, entries.get(599).getTs(),
  120 + LocalDateTime.now(ZoneOffset.UTC).toInstant(ZoneOffset.UTC).toEpochMilli(), PARTITION_MINUTES - 599, Aggregation.MIN)).get();
121 121 log.debug("Fetched records {}", localDateTime);
122 122 List<TsKvEntry> expected = entries.subList(600, PARTITION_MINUTES);
123 123 assertEquals(expected.size(), list.size());
124 124 assertEquals(expected, list);
125 125 }
126 126
  127 +// @Test
  128 +// public void testFindDeviceTsDataByQuery() throws Exception {
  129 +// DeviceId deviceId = new DeviceId(UUIDs.timeBased());
  130 +// LocalDateTime localDateTime = LocalDateTime.now(ZoneOffset.UTC).minusMinutes(PARTITION_MINUTES);
  131 +// log.debug("Start event time is {}", localDateTime);
  132 +// List<TsKvEntry> entries = new ArrayList<>(PARTITION_MINUTES);
  133 +//
  134 +// for (int i = 0; i < PARTITION_MINUTES; i++) {
  135 +// long time = localDateTime.plusMinutes(i).toInstant(ZoneOffset.UTC).toEpochMilli();
  136 +// BasicTsKvEntry tsKvEntry = new BasicTsKvEntry(time, stringKvEntry);
  137 +// tsService.save(DataConstants.DEVICE, deviceId, tsKvEntry).get();
  138 +// entries.add(tsKvEntry);
  139 +// }
  140 +// log.debug("Saved all records {}", localDateTime);
  141 +// List<TsKvEntry> list = tsService.findAll(DataConstants.DEVICE, deviceId, new BaseTsKvQuery(STRING_KEY, entries.get(599).getTs(),
  142 +// LocalDateTime.now(ZoneOffset.UTC).toInstant(ZoneOffset.UTC).toEpochMilli(), PARTITION_MINUTES - 599, Aggregation.MIN)).get();
  143 +// log.debug("Fetched records {}", localDateTime);
  144 +// List<TsKvEntry> expected = entries.subList(600, PARTITION_MINUTES);
  145 +// assertEquals(expected.size(), list.size());
  146 +// assertEquals(expected, list);
  147 +// }
  148 +
127 149
128 150 private void saveEntries(DeviceId deviceId, long ts) throws ExecutionException, InterruptedException {
129 151 tsService.save(DataConstants.DEVICE, deviceId, toTsEntry(ts, stringKvEntry)).get();
... ...
... ... @@ -2,7 +2,7 @@ cassandra.cluster_name=Thingsboard Cluster
2 2
3 3 cassandra.keyspace_name=thingsboard
4 4
5   -cassandra.url=127.0.0.1:9142
  5 +cassandra.url=127.0.0.1:9042
6 6
7 7 cassandra.ssl=false
8 8
... ... @@ -47,3 +47,7 @@ cassandra.query.default_fetch_size=2000
47 47 cassandra.query.ts_key_value_partitioning=HOURS
48 48
49 49 cassandra.query.max_limit_per_request=1000
  50 +
  51 +cassandra.query.read_result_processing_threads=3
  52 +
  53 +cassandra.query.min_read_step=100
\ No newline at end of file
... ...
... ... @@ -84,6 +84,8 @@ public interface PluginContext {
84 84
85 85 List<TsKvEntry> loadTimeseries(DeviceId deviceId, TsKvQuery query);
86 86
  87 + void loadTimeseries(DeviceId deviceId, TsKvQuery query, PluginCallback<List<TsKvEntry>> callback);
  88 +
87 89 void loadLatestTimeseries(DeviceId deviceId, Collection<String> keys, PluginCallback<List<TsKvEntry>> callback);
88 90
89 91 void loadLatestTimeseries(DeviceId deviceId, PluginCallback<List<TsKvEntry>> callback);
... ...
... ... @@ -95,8 +95,9 @@ public class TelemetryRestMsgHandler extends DefaultRestMsgHandler {
95 95 Optional<Integer> limit = request.getIntParamValue("limit");
96 96 Map<String, List<TsData>> data = new LinkedHashMap<>();
97 97 for (String key : keys.split(",")) {
98   - List<TsKvEntry> entries = ctx.loadTimeseries(deviceId, new BaseTsKvQuery(key, startTs, endTs, limit));
99   - data.put(key, entries.stream().map(v -> new TsData(v.getTs(), v.getValueAsString())).collect(Collectors.toList()));
  98 + //TODO: refactoring
  99 +// List<TsKvEntry> entries = ctx.loadTimeseries(deviceId, new BaseTsKvQuery(key, startTs, endTs, limit));
  100 +// data.put(key, entries.stream().map(v -> new TsData(v.getTs(), v.getValueAsString())).collect(Collectors.toList()));
100 101 }
101 102 msg.getResponseHolder().setResult(new ResponseEntity<>(data, HttpStatus.OK));
102 103 } else if ("attributes".equals(entity)) {
... ...