Commit 682c15cfe1d533b5bb9526a6081063aeefb92896

Authored by Andrii Shvaika
1 parent 374cf2a1

Test Fix

@@ -272,6 +272,8 @@ sql: @@ -272,6 +272,8 @@ sql:
272 batch_threads: "${SQL_TS_LATEST_BATCH_THREADS:4}" 272 batch_threads: "${SQL_TS_LATEST_BATCH_THREADS:4}"
273 # Specify whether to remove null characters from strValue of attributes and timeseries before insert 273 # Specify whether to remove null characters from strValue of attributes and timeseries before insert
274 remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" 274 remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}"
  275 + # Specify whether to log database queries and their parameters generated by entity query repository
  276 + log_entity_queries: "${SQL_LOG_ENTITY_QUERIES:false}"
275 postgres: 277 postgres:
276 # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE. 278 # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE.
277 ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}" 279 ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}"
@@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.query; @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.query;
18 import lombok.extern.slf4j.Slf4j; 18 import lombok.extern.slf4j.Slf4j;
19 import org.apache.commons.lang3.StringUtils; 19 import org.apache.commons.lang3.StringUtils;
20 import org.springframework.beans.factory.annotation.Autowired; 20 import org.springframework.beans.factory.annotation.Autowired;
  21 +import org.springframework.beans.factory.annotation.Value;
21 import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; 22 import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
22 import org.springframework.stereotype.Repository; 23 import org.springframework.stereotype.Repository;
23 import org.springframework.transaction.TransactionStatus; 24 import org.springframework.transaction.TransactionStatus;
@@ -41,6 +42,7 @@ import org.thingsboard.server.dao.model.ModelConstants; @@ -41,6 +42,7 @@ import org.thingsboard.server.dao.model.ModelConstants;
41 import org.thingsboard.server.dao.util.SqlDao; 42 import org.thingsboard.server.dao.util.SqlDao;
42 43
43 import java.util.ArrayList; 44 import java.util.ArrayList;
  45 +import java.util.Arrays;
44 import java.util.Collection; 46 import java.util.Collection;
45 import java.util.HashMap; 47 import java.util.HashMap;
46 import java.util.HashSet; 48 import java.util.HashSet;
@@ -114,6 +116,9 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { @@ -114,6 +116,9 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository {
114 protected final NamedParameterJdbcTemplate jdbcTemplate; 116 protected final NamedParameterJdbcTemplate jdbcTemplate;
115 private final TransactionTemplate transactionTemplate; 117 private final TransactionTemplate transactionTemplate;
116 118
  119 + @Value("${sql.log_entity_queries:false}")
  120 + private boolean logSqlQueries;
  121 +
117 public DefaultAlarmQueryRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) { 122 public DefaultAlarmQueryRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) {
118 this.jdbcTemplate = jdbcTemplate; 123 this.jdbcTemplate = jdbcTemplate;
119 this.transactionTemplate = transactionTemplate; 124 this.transactionTemplate = transactionTemplate;
@@ -237,6 +242,10 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { @@ -237,6 +242,10 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository {
237 dataQuery = String.format("%s limit %s offset %s", dataQuery, pageLink.getPageSize(), startIndex); 242 dataQuery = String.format("%s limit %s offset %s", dataQuery, pageLink.getPageSize(), startIndex);
238 } 243 }
239 List<Map<String, Object>> rows = jdbcTemplate.queryForList(dataQuery, ctx); 244 List<Map<String, Object>> rows = jdbcTemplate.queryForList(dataQuery, ctx);
  245 + if (logSqlQueries) {
  246 + log.error("QUERY: {}", dataQuery);
  247 + Arrays.asList(ctx.getParameterNames()).forEach(param -> log.error("QUERY PARAM: {}->{}", param, ctx.getValue(param)));
  248 + }
240 return AlarmDataAdapter.createAlarmData(pageLink, rows, totalElements, orderedEntityIds); 249 return AlarmDataAdapter.createAlarmData(pageLink, rows, totalElements, orderedEntityIds);
241 }); 250 });
242 } 251 }
@@ -5,7 +5,7 @@ @@ -5,7 +5,7 @@
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at 6 * You may obtain a copy of the License at
7 * 7 *
8 - * http://www.apache.org/licenses/LICENSE-2.0 8 + * http://www.apache.org/licenses/LICENSE-2.0
9 * 9 *
10 * Unless required by applicable law or agreed to in writing, software 10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS, 11 * distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.query; @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.query;
18 import lombok.extern.slf4j.Slf4j; 18 import lombok.extern.slf4j.Slf4j;
19 import org.apache.commons.lang3.StringUtils; 19 import org.apache.commons.lang3.StringUtils;
20 import org.springframework.beans.factory.annotation.Autowired; 20 import org.springframework.beans.factory.annotation.Autowired;
  21 +import org.springframework.beans.factory.annotation.Value;
21 import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; 22 import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
22 import org.springframework.stereotype.Repository; 23 import org.springframework.stereotype.Repository;
23 import org.springframework.transaction.TransactionStatus; 24 import org.springframework.transaction.TransactionStatus;
@@ -223,6 +224,9 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { @@ -223,6 +224,9 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
223 private final NamedParameterJdbcTemplate jdbcTemplate; 224 private final NamedParameterJdbcTemplate jdbcTemplate;
224 private final TransactionTemplate transactionTemplate; 225 private final TransactionTemplate transactionTemplate;
225 226
  227 + @Value("${sql.log_entity_queries:false}")
  228 + private boolean logSqlQueries;
  229 +
226 public DefaultEntityQueryRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) { 230 public DefaultEntityQueryRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) {
227 this.jdbcTemplate = jdbcTemplate; 231 this.jdbcTemplate = jdbcTemplate;
228 this.transactionTemplate = transactionTemplate; 232 this.transactionTemplate = transactionTemplate;
@@ -236,8 +240,10 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { @@ -236,8 +240,10 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
236 ctx.append(addEntityTableQuery(ctx, query.getEntityFilter())); 240 ctx.append(addEntityTableQuery(ctx, query.getEntityFilter()));
237 ctx.append(" e where "); 241 ctx.append(" e where ");
238 ctx.append(buildEntityWhere(ctx, query.getEntityFilter(), Collections.emptyList())); 242 ctx.append(buildEntityWhere(ctx, query.getEntityFilter(), Collections.emptyList()));
239 -// log.error("QUERY: {}", ctx.getQuery());  
240 -// Arrays.asList(ctx.getParameterNames()).forEach(param -> log.error("QUERY PARAM: {}->{}", param, ctx.getValue(param))); 243 + if (logSqlQueries) {
  244 + log.info("QUERY: {}", ctx.getQuery());
  245 + Arrays.asList(ctx.getParameterNames()).forEach(param -> log.info("QUERY PARAM: {}->{}", param, ctx.getValue(param)));
  246 + }
241 return transactionTemplate.execute(status -> jdbcTemplate.queryForObject(ctx.getQuery(), ctx, Long.class)); 247 return transactionTemplate.execute(status -> jdbcTemplate.queryForObject(ctx.getQuery(), ctx, Long.class));
242 } 248 }
243 249
@@ -331,8 +337,10 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { @@ -331,8 +337,10 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
331 if (pageLink.getPageSize() > 0) { 337 if (pageLink.getPageSize() > 0) {
332 dataQuery = String.format("%s limit %s offset %s", dataQuery, pageLink.getPageSize(), startIndex); 338 dataQuery = String.format("%s limit %s offset %s", dataQuery, pageLink.getPageSize(), startIndex);
333 } 339 }
334 -// log.error("QUERY: {}", dataQuery);  
335 -// Arrays.asList(ctx.getParameterNames()).forEach(param -> log.error("QUERY PARAM: {}->{}", param, ctx.getValue(param))); 340 + if (logSqlQueries) {
  341 + log.error("QUERY: {}", dataQuery);
  342 + Arrays.asList(ctx.getParameterNames()).forEach(param -> log.error("QUERY PARAM: {}->{}", param, ctx.getValue(param)));
  343 + }
336 List<Map<String, Object>> rows = jdbcTemplate.queryForList(dataQuery, ctx); 344 List<Map<String, Object>> rows = jdbcTemplate.queryForList(dataQuery, ctx);
337 return EntityDataAdapter.createEntityData(pageLink, selectionMapping, rows, totalElements); 345 return EntityDataAdapter.createEntityData(pageLink, selectionMapping, rows, totalElements);
338 }); 346 });
@@ -245,13 +245,13 @@ public class EntityKeyMapping { @@ -245,13 +245,13 @@ public class EntityKeyMapping {
245 filterQuery = " AND (" + filterQuery + ")"; 245 filterQuery = " AND (" + filterQuery + ")";
246 } 246 }
247 if (entityKey.getType().equals(EntityKeyType.TIME_SERIES)) { 247 if (entityKey.getType().equals(EntityKeyType.TIME_SERIES)) {
248 - String join = hasFilter() ? "left join" : "left outer join"; 248 + String join = hasFilter() ? "inner join" : "left join";
249 return String.format("%s ts_kv_latest %s ON %s.entity_id=entities.id AND %s.key = (select key_id from ts_kv_dictionary where key = :%s_key_id) %s", 249 return String.format("%s ts_kv_latest %s ON %s.entity_id=entities.id AND %s.key = (select key_id from ts_kv_dictionary where key = :%s_key_id) %s",
250 join, alias, alias, alias, alias, filterQuery); 250 join, alias, alias, alias, alias, filterQuery);
251 } else { 251 } else {
252 String query; 252 String query;
253 if (!entityKey.getType().equals(EntityKeyType.ATTRIBUTE)) { 253 if (!entityKey.getType().equals(EntityKeyType.ATTRIBUTE)) {
254 - String join = hasFilter() ? "left join" : "left outer join"; 254 + String join = hasFilter() ? "inner join" : "left join";
255 query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.entity_type=%s AND %s.attribute_key=:%s_key_id ", 255 query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.entity_type=%s AND %s.attribute_key=:%s_key_id ",
256 join, alias, alias, alias, entityTypeStr, alias, alias); 256 join, alias, alias, alias, entityTypeStr, alias, alias);
257 String scope; 257 String scope;
@@ -46,4 +46,6 @@ queue.rule-engine.queues[0].poll-interval=25 @@ -46,4 +46,6 @@ queue.rule-engine.queues[0].poll-interval=25
46 queue.rule-engine.queues[0].partitions=3 46 queue.rule-engine.queues[0].partitions=3
47 queue.rule-engine.queues[0].pack-processing-timeout=3000 47 queue.rule-engine.queues[0].pack-processing-timeout=3000
48 queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES 48 queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES
49 -queue.rule-engine.queues[0].submit-strategy.type=BURST  
  49 +queue.rule-engine.queues[0].submit-strategy.type=BURST
  50 +
  51 +sql.log_entity_queries=true