Commit 9c44920fe749b07fdb089a6a9eb1075789fc53ed

Authored by Viacheslav Kukhtyn
2 parents 9829dd17 6814c21e

Merge branch 'master' into feature/log-telemetry-updated

Showing 34 changed files with 810 additions and 255 deletions
... ... @@ -16,18 +16,23 @@
16 16 package org.thingsboard.server.controller;
17 17
18 18 import org.springframework.beans.factory.annotation.Autowired;
  19 +import org.springframework.http.ResponseEntity;
19 20 import org.springframework.security.access.prepost.PreAuthorize;
20 21 import org.springframework.web.bind.annotation.RequestBody;
21 22 import org.springframework.web.bind.annotation.RequestMapping;
22 23 import org.springframework.web.bind.annotation.RequestMethod;
  24 +import org.springframework.web.bind.annotation.RequestParam;
23 25 import org.springframework.web.bind.annotation.ResponseBody;
24 26 import org.springframework.web.bind.annotation.RestController;
  27 +import org.springframework.web.context.request.async.DeferredResult;
25 28 import org.thingsboard.server.common.data.exception.ThingsboardException;
  29 +import org.thingsboard.server.common.data.id.TenantId;
26 30 import org.thingsboard.server.common.data.page.PageData;
27 31 import org.thingsboard.server.common.data.query.AlarmData;
28 32 import org.thingsboard.server.common.data.query.AlarmDataQuery;
29 33 import org.thingsboard.server.common.data.query.EntityCountQuery;
30 34 import org.thingsboard.server.common.data.query.EntityData;
  35 +import org.thingsboard.server.common.data.query.EntityDataPageLink;
31 36 import org.thingsboard.server.common.data.query.EntityDataQuery;
32 37 import org.thingsboard.server.queue.util.TbCoreComponent;
33 38 import org.thingsboard.server.service.query.EntityQueryService;
... ... @@ -40,6 +45,7 @@ public class EntityQueryController extends BaseController {
40 45 @Autowired
41 46 private EntityQueryService entityQueryService;
42 47
  48 + private static final int MAX_PAGE_SIZE = 100;
43 49
44 50 @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
45 51 @RequestMapping(value = "/entitiesQuery/count", method = RequestMethod.POST)
... ... @@ -76,4 +82,24 @@ public class EntityQueryController extends BaseController {
76 82 throw handleException(e);
77 83 }
78 84 }
  85 +
  86 + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
  87 + @RequestMapping(value = "/entitiesQuery/find/keys", method = RequestMethod.POST)
  88 + @ResponseBody
  89 + public DeferredResult<ResponseEntity> findEntityTimeseriesAndAttributesKeysByQuery(@RequestBody EntityDataQuery query,
  90 + @RequestParam("timeseries") boolean isTimeseries,
  91 + @RequestParam("attributes") boolean isAttributes) throws ThingsboardException {
  92 + TenantId tenantId = getTenantId();
  93 + checkNotNull(query);
  94 + try {
  95 + EntityDataPageLink pageLink = query.getPageLink();
  96 + if (pageLink.getPageSize() > MAX_PAGE_SIZE) {
  97 + pageLink.setPageSize(MAX_PAGE_SIZE);
  98 + }
  99 + return entityQueryService.getKeysByQuery(getCurrentUser(), tenantId, query, isTimeseries, isAttributes);
  100 + } catch (Exception e) {
  101 + throw handleException(e);
  102 + }
  103 + }
  104 +
79 105 }
... ...
... ... @@ -15,11 +15,23 @@
15 15 */
16 16 package org.thingsboard.server.service.query;
17 17
  18 +import com.fasterxml.jackson.databind.node.ArrayNode;
  19 +import com.fasterxml.jackson.databind.node.ObjectNode;
  20 +import com.google.common.util.concurrent.FutureCallback;
  21 +import com.google.common.util.concurrent.Futures;
  22 +import com.google.common.util.concurrent.ListenableFuture;
18 23 import lombok.extern.slf4j.Slf4j;
  24 +import org.checkerframework.checker.nullness.qual.Nullable;
19 25 import org.springframework.beans.factory.annotation.Autowired;
20 26 import org.springframework.beans.factory.annotation.Value;
  27 +import org.springframework.http.HttpStatus;
  28 +import org.springframework.http.ResponseEntity;
21 29 import org.springframework.stereotype.Service;
  30 +import org.springframework.util.CollectionUtils;
  31 +import org.springframework.web.context.request.async.DeferredResult;
  32 +import org.thingsboard.server.common.data.EntityType;
22 33 import org.thingsboard.server.common.data.id.EntityId;
  34 +import org.thingsboard.server.common.data.id.TenantId;
23 35 import org.thingsboard.server.common.data.page.PageData;
24 36 import org.thingsboard.server.common.data.query.AlarmData;
25 37 import org.thingsboard.server.common.data.query.AlarmDataQuery;
... ... @@ -31,12 +43,25 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder;
31 43 import org.thingsboard.server.common.data.query.EntityKey;
32 44 import org.thingsboard.server.common.data.query.EntityKeyType;
33 45 import org.thingsboard.server.dao.alarm.AlarmService;
  46 +import org.thingsboard.server.dao.attributes.AttributesService;
34 47 import org.thingsboard.server.dao.entity.EntityService;
35 48 import org.thingsboard.server.dao.model.ModelConstants;
  49 +import org.thingsboard.server.dao.timeseries.TimeseriesService;
  50 +import org.thingsboard.server.dao.util.mapping.JacksonUtil;
36 51 import org.thingsboard.server.queue.util.TbCoreComponent;
  52 +import org.thingsboard.server.service.executors.DbCallbackExecutorService;
  53 +import org.thingsboard.server.service.security.AccessValidator;
37 54 import org.thingsboard.server.service.security.model.SecurityUser;
38 55
  56 +import java.util.ArrayList;
  57 +import java.util.Collection;
  58 +import java.util.Collections;
39 59 import java.util.LinkedHashMap;
  60 +import java.util.List;
  61 +import java.util.Map;
  62 +import java.util.Set;
  63 +import java.util.function.Consumer;
  64 +import java.util.stream.Collectors;
40 65
41 66 @Service
42 67 @Slf4j
... ... @@ -52,6 +77,15 @@ public class DefaultEntityQueryService implements EntityQueryService {
52 77 @Value("${server.ws.max_entities_per_alarm_subscription:1000}")
53 78 private int maxEntitiesPerAlarmSubscription;
54 79
  80 + @Autowired
  81 + private DbCallbackExecutorService dbCallbackExecutor;
  82 +
  83 + @Autowired
  84 + private TimeseriesService timeseriesService;
  85 +
  86 + @Autowired
  87 + private AttributesService attributesService;
  88 +
55 89 @Override
56 90 public long countEntitiesByQuery(SecurityUser securityUser, EntityCountQuery query) {
57 91 return entityService.countEntitiesByQuery(securityUser.getTenantId(), securityUser.getCustomerId(), query);
... ... @@ -100,4 +134,103 @@ public class DefaultEntityQueryService implements EntityQueryService {
100 134 EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null, entitiesSortOrder);
101 135 return new EntityDataQuery(query.getEntityFilter(), edpl, query.getEntityFields(), query.getLatestValues(), query.getKeyFilters());
102 136 }
  137 +
  138 + @Override
  139 + public DeferredResult<ResponseEntity> getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query,
  140 + boolean isTimeseries, boolean isAttributes) {
  141 + final DeferredResult<ResponseEntity> response = new DeferredResult<>();
  142 + if (!isAttributes && !isTimeseries) {
  143 + replyWithEmptyResponse(response);
  144 + return response;
  145 + }
  146 +
  147 + List<EntityId> ids = this.findEntityDataByQuery(securityUser, query).getData().stream()
  148 + .map(EntityData::getEntityId)
  149 + .collect(Collectors.toList());
  150 + if (ids.isEmpty()) {
  151 + replyWithEmptyResponse(response);
  152 + return response;
  153 + }
  154 +
  155 + Set<EntityType> types = ids.stream().map(EntityId::getEntityType).collect(Collectors.toSet());
  156 + final ListenableFuture<List<String>> timeseriesKeysFuture;
  157 + final ListenableFuture<List<String>> attributesKeysFuture;
  158 +
  159 + if (isTimeseries) {
  160 + timeseriesKeysFuture = dbCallbackExecutor.submit(() -> timeseriesService.findAllKeysByEntityIds(tenantId, ids));
  161 + } else {
  162 + timeseriesKeysFuture = null;
  163 + }
  164 +
  165 + if (isAttributes) {
  166 + Map<EntityType, List<EntityId>> typesMap = ids.stream().collect(Collectors.groupingBy(EntityId::getEntityType));
  167 + List<ListenableFuture<List<String>>> futures = new ArrayList<>(typesMap.size());
  168 + typesMap.forEach((type, entityIds) -> futures.add(dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIds(tenantId, type, entityIds))));
  169 + attributesKeysFuture = Futures.transform(Futures.allAsList(futures), lists -> {
  170 + if (CollectionUtils.isEmpty(lists)) {
  171 + return Collections.emptyList();
  172 + }
  173 + return lists.stream().flatMap(List::stream).distinct().sorted().collect(Collectors.toList());
  174 + }, dbCallbackExecutor);
  175 + } else {
  176 + attributesKeysFuture = null;
  177 + }
  178 +
  179 + if (isTimeseries && isAttributes) {
  180 + Futures.whenAllComplete(timeseriesKeysFuture, attributesKeysFuture).run(() -> {
  181 + try {
  182 + replyWithResponse(response, types, timeseriesKeysFuture.get(), attributesKeysFuture.get());
  183 + } catch (Exception e) {
  184 + log.error("Failed to fetch timeseries and attributes keys!", e);
  185 + AccessValidator.handleError(e, response, HttpStatus.INTERNAL_SERVER_ERROR);
  186 + }
  187 + }, dbCallbackExecutor);
  188 + } else if (isTimeseries) {
  189 + addCallback(timeseriesKeysFuture, keys -> replyWithResponse(response, types, keys, null),
  190 + error -> {
  191 + log.error("Failed to fetch timeseries keys!", error);
  192 + AccessValidator.handleError(error, response, HttpStatus.INTERNAL_SERVER_ERROR);
  193 + });
  194 + } else {
  195 + addCallback(attributesKeysFuture, keys -> replyWithResponse(response, types, null, keys),
  196 + error -> {
  197 + log.error("Failed to fetch attributes keys!", error);
  198 + AccessValidator.handleError(error, response, HttpStatus.INTERNAL_SERVER_ERROR);
  199 + });
  200 + }
  201 + return response;
  202 + }
  203 +
  204 + private void replyWithResponse(DeferredResult<ResponseEntity> response, Set<EntityType> types, List<String> timeseriesKeys, List<String> attributesKeys) {
  205 + ObjectNode json = JacksonUtil.newObjectNode();
  206 + addItemsToArrayNode(json.putArray("entityTypes"), types);
  207 + addItemsToArrayNode(json.putArray("timeseries"), timeseriesKeys);
  208 + addItemsToArrayNode(json.putArray("attribute"), attributesKeys);
  209 + response.setResult(new ResponseEntity(json, HttpStatus.OK));
  210 + }
  211 +
  212 + private void replyWithEmptyResponse(DeferredResult<ResponseEntity> response) {
  213 + replyWithResponse(response, Collections.emptySet(), Collections.emptyList(), Collections.emptyList());
  214 + }
  215 +
  216 + private void addItemsToArrayNode(ArrayNode arrayNode, Collection<?> collection) {
  217 + if (!CollectionUtils.isEmpty(collection)) {
  218 + collection.forEach(item -> arrayNode.add(item.toString()));
  219 + }
  220 + }
  221 +
  222 + private void addCallback(ListenableFuture<List<String>> future, Consumer<List<String>> success, Consumer<Throwable> error) {
  223 + Futures.addCallback(future, new FutureCallback<List<String>>() {
  224 + @Override
  225 + public void onSuccess(@Nullable List<String> keys) {
  226 + success.accept(keys);
  227 + }
  228 +
  229 + @Override
  230 + public void onFailure(Throwable t) {
  231 + error.accept(t);
  232 + }
  233 + }, dbCallbackExecutor);
  234 + }
  235 +
103 236 }
... ...
... ... @@ -15,6 +15,9 @@
15 15 */
16 16 package org.thingsboard.server.service.query;
17 17
  18 +import org.springframework.http.ResponseEntity;
  19 +import org.springframework.web.context.request.async.DeferredResult;
  20 +import org.thingsboard.server.common.data.id.TenantId;
18 21 import org.thingsboard.server.common.data.page.PageData;
19 22 import org.thingsboard.server.common.data.query.AlarmData;
20 23 import org.thingsboard.server.common.data.query.AlarmDataQuery;
... ... @@ -31,4 +34,7 @@ public interface EntityQueryService {
31 34
32 35 PageData<AlarmData> findAlarmDataByQuery(SecurityUser securityUser, AlarmDataQuery query);
33 36
  37 + DeferredResult<ResponseEntity> getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query,
  38 + boolean isTimeseries, boolean isAttributes);
  39 +
34 40 }
... ...
... ... @@ -192,8 +192,9 @@ cassandra:
192 192 read_consistency_level: "${CASSANDRA_READ_CONSISTENCY_LEVEL:ONE}"
193 193 write_consistency_level: "${CASSANDRA_WRITE_CONSISTENCY_LEVEL:ONE}"
194 194 default_fetch_size: "${CASSANDRA_DEFAULT_FETCH_SIZE:2000}"
195   - # Specify partitioning size for timestamp key-value storage. Example: MINUTES, HOURS, DAYS, MONTHS,INDEFINITE
  195 + # Specify partitioning size for timestamp key-value storage. Example: MINUTES, HOURS, DAYS, MONTHS, INDEFINITE
196 196 ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}"
  197 + ts_key_value_partitions_max_cache_size: "${TS_KV_PARTITIONS_MAX_CACHE_SIZE:100000}"
197 198 ts_key_value_ttl: "${TS_KV_TTL:0}"
198 199 events_ttl: "${TS_EVENTS_TTL:0}"
199 200 # Specify TTL of debug log in seconds. The current value corresponds to one week
... ...
... ... @@ -16,6 +16,7 @@
16 16 package org.thingsboard.server.dao.attributes;
17 17
18 18 import com.google.common.util.concurrent.ListenableFuture;
  19 +import org.thingsboard.server.common.data.EntityType;
19 20 import org.thingsboard.server.common.data.id.DeviceProfileId;
20 21 import org.thingsboard.server.common.data.id.EntityId;
21 22 import org.thingsboard.server.common.data.id.TenantId;
... ... @@ -42,4 +43,6 @@ public interface AttributesService {
42 43
43 44 List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId);
44 45
  46 + List<String> findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List<EntityId> entityIds);
  47 +
45 48 }
... ...
... ... @@ -50,4 +50,6 @@ public interface TimeseriesService {
50 50 ListenableFuture<Collection<String>> removeAllLatest(TenantId tenantId, EntityId entityId);
51 51
52 52 List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId);
  53 +
  54 + List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds);
53 55 }
... ...
... ... @@ -462,26 +462,26 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
462 462 String clientId = msg.payload().clientIdentifier();
463 463 if (DataConstants.PROVISION.equals(userName) || DataConstants.PROVISION.equals(clientId)) {
464 464 deviceSessionCtx.setProvisionOnly(true);
465   - ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED));
  465 + ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED, msg));
466 466 } else {
467 467 X509Certificate cert;
468 468 if (sslHandler != null && (cert = getX509Certificate()) != null) {
469   - processX509CertConnect(ctx, cert);
  469 + processX509CertConnect(ctx, cert, msg);
470 470 } else {
471 471 processAuthTokenConnect(ctx, msg);
472 472 }
473 473 }
474 474 }
475 475
476   - private void processAuthTokenConnect(ChannelHandlerContext ctx, MqttConnectMessage msg) {
477   - String userName = msg.payload().userName();
  476 + private void processAuthTokenConnect(ChannelHandlerContext ctx, MqttConnectMessage connectMessage) {
  477 + String userName = connectMessage.payload().userName();
478 478 log.info("[{}] Processing connect msg for client with user name: {}!", sessionId, userName);
479 479 TransportProtos.ValidateBasicMqttCredRequestMsg.Builder request = TransportProtos.ValidateBasicMqttCredRequestMsg.newBuilder()
480   - .setClientId(msg.payload().clientIdentifier());
  480 + .setClientId(connectMessage.payload().clientIdentifier());
481 481 if (userName != null) {
482 482 request.setUserName(userName);
483 483 }
484   - String password = msg.payload().password();
  484 + String password = connectMessage.payload().password();
485 485 if (password != null) {
486 486 request.setPassword(password);
487 487 }
... ... @@ -489,19 +489,19 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
489 489 new TransportServiceCallback<ValidateDeviceCredentialsResponse>() {
490 490 @Override
491 491 public void onSuccess(ValidateDeviceCredentialsResponse msg) {
492   - onValidateDeviceResponse(msg, ctx);
  492 + onValidateDeviceResponse(msg, ctx, connectMessage);
493 493 }
494 494
495 495 @Override
496 496 public void onError(Throwable e) {
497 497 log.trace("[{}] Failed to process credentials: {}", address, userName, e);
498   - ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE));
  498 + ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE, connectMessage));
499 499 ctx.close();
500 500 }
501 501 });
502 502 }
503 503
504   - private void processX509CertConnect(ChannelHandlerContext ctx, X509Certificate cert) {
  504 + private void processX509CertConnect(ChannelHandlerContext ctx, X509Certificate cert, MqttConnectMessage connectMessage) {
505 505 try {
506 506 if (!context.isSkipValidityCheckForClientCert()) {
507 507 cert.checkValidity();
... ... @@ -512,18 +512,18 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
512 512 new TransportServiceCallback<ValidateDeviceCredentialsResponse>() {
513 513 @Override
514 514 public void onSuccess(ValidateDeviceCredentialsResponse msg) {
515   - onValidateDeviceResponse(msg, ctx);
  515 + onValidateDeviceResponse(msg, ctx, connectMessage);
516 516 }
517 517
518 518 @Override
519 519 public void onError(Throwable e) {
520 520 log.trace("[{}] Failed to process credentials: {}", address, sha3Hash, e);
521   - ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE));
  521 + ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE, connectMessage));
522 522 ctx.close();
523 523 }
524 524 });
525 525 } catch (Exception e) {
526   - ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_REFUSED_NOT_AUTHORIZED));
  526 + ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_REFUSED_NOT_AUTHORIZED, connectMessage));
527 527 ctx.close();
528 528 }
529 529 }
... ... @@ -547,11 +547,11 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
547 547 doDisconnect();
548 548 }
549 549
550   - private MqttConnAckMessage createMqttConnAckMsg(MqttConnectReturnCode returnCode) {
  550 + private MqttConnAckMessage createMqttConnAckMsg(MqttConnectReturnCode returnCode, MqttConnectMessage msg) {
551 551 MqttFixedHeader mqttFixedHeader =
552 552 new MqttFixedHeader(CONNACK, false, AT_MOST_ONCE, false, 0);
553 553 MqttConnAckVariableHeader mqttConnAckVariableHeader =
554   - new MqttConnAckVariableHeader(returnCode, true);
  554 + new MqttConnAckVariableHeader(returnCode, !msg.variableHeader().isCleanSession());
555 555 return new MqttConnAckMessage(mqttFixedHeader, mqttConnAckVariableHeader);
556 556 }
557 557
... ... @@ -627,9 +627,10 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
627 627 }
628 628 }
629 629
630   - private void onValidateDeviceResponse(ValidateDeviceCredentialsResponse msg, ChannelHandlerContext ctx) {
  630 +
  631 + private void onValidateDeviceResponse(ValidateDeviceCredentialsResponse msg, ChannelHandlerContext ctx, MqttConnectMessage connectMessage) {
631 632 if (!msg.hasDeviceInfo()) {
632   - ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_REFUSED_NOT_AUTHORIZED));
  633 + ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_REFUSED_NOT_AUTHORIZED, connectMessage));
633 634 ctx.close();
634 635 } else {
635 636 deviceSessionCtx.setDeviceInfo(msg.getDeviceInfo());
... ... @@ -640,7 +641,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
640 641 public void onSuccess(Void msg) {
641 642 transportService.registerAsyncSession(deviceSessionCtx.getSessionInfo(), MqttTransportHandler.this);
642 643 checkGatewaySession();
643   - ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED));
  644 + ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED, connectMessage));
644 645 log.info("[{}] Client connected!", sessionId);
645 646 }
646 647
... ... @@ -651,7 +652,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
651 652 } else {
652 653 log.warn("[{}] Failed to submit session event", sessionId, e);
653 654 }
654   - ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE));
  655 + ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE, connectMessage));
655 656 ctx.close();
656 657 }
657 658 });
... ...
... ... @@ -16,6 +16,7 @@
16 16 package org.thingsboard.server.dao.attributes;
17 17
18 18 import com.google.common.util.concurrent.ListenableFuture;
  19 +import org.thingsboard.server.common.data.EntityType;
19 20 import org.thingsboard.server.common.data.id.DeviceProfileId;
20 21 import org.thingsboard.server.common.data.id.EntityId;
21 22 import org.thingsboard.server.common.data.id.TenantId;
... ... @@ -41,4 +42,6 @@ public interface AttributesDao {
41 42 ListenableFuture<List<Void>> removeAll(TenantId tenantId, EntityId entityId, String attributeType, List<String> keys);
42 43
43 44 List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId);
  45 +
  46 + List<String> findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List<EntityId> entityIds);
44 47 }
... ...
... ... @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.Futures;
20 20 import com.google.common.util.concurrent.ListenableFuture;
21 21 import org.springframework.beans.factory.annotation.Autowired;
22 22 import org.springframework.stereotype.Service;
  23 +import org.thingsboard.server.common.data.EntityType;
23 24 import org.thingsboard.server.common.data.id.DeviceProfileId;
24 25 import org.thingsboard.server.common.data.id.EntityId;
25 26 import org.thingsboard.server.common.data.id.TenantId;
... ... @@ -66,6 +67,11 @@ public class BaseAttributesService implements AttributesService {
66 67 }
67 68
68 69 @Override
  70 + public List<String> findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List<EntityId> entityIds) {
  71 + return attributesDao.findAllKeysByEntityIds(tenantId, entityType, entityIds);
  72 + }
  73 +
  74 + @Override
69 75 public ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes) {
70 76 validate(entityId, scope);
71 77 attributes.forEach(attribute -> validate(attribute));
... ...
... ... @@ -56,5 +56,8 @@ public interface AttributeKvRepository extends CrudRepository<AttributeKvEntity,
56 56 "AND entity_id in (SELECT id FROM device WHERE tenant_id = :tenantId limit 100) ORDER BY attribute_key", nativeQuery = true)
57 57 List<String> findAllKeysByTenantId(@Param("tenantId") UUID tenantId);
58 58
  59 + @Query(value = "SELECT DISTINCT attribute_key FROM attribute_kv WHERE entity_type = :entityType " +
  60 + "AND entity_id in :entityIds ORDER BY attribute_key", nativeQuery = true)
  61 + List<String> findAllKeysByEntityIds(@Param("entityType") String entityType, @Param("entityIds") List<UUID> entityIds);
59 62 }
60 63
... ...
... ... @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j;
22 22 import org.springframework.beans.factory.annotation.Autowired;
23 23 import org.springframework.beans.factory.annotation.Value;
24 24 import org.springframework.stereotype.Component;
  25 +import org.thingsboard.server.common.data.EntityType;
25 26 import org.thingsboard.server.common.data.id.DeviceProfileId;
26 27 import org.thingsboard.server.common.data.id.EntityId;
27 28 import org.thingsboard.server.common.data.id.TenantId;
... ... @@ -146,6 +147,12 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl
146 147 }
147 148
148 149 @Override
  150 + public List<String> findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List<EntityId> entityIds) {
  151 + return attributeKvRepository
  152 + .findAllKeysByEntityIds(entityType.name(), entityIds.stream().map(EntityId::getId).collect(Collectors.toList()));
  153 + }
  154 +
  155 + @Override
149 156 public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, String attributeType, AttributeKvEntry attribute) {
150 157 AttributeKvEntity entity = new AttributeKvEntity();
151 158 entity.setId(new AttributeKvCompositeKey(entityId.getEntityType(), entityId.getId(), attributeType, attribute.getKey()));
... ...
... ... @@ -61,6 +61,7 @@ import java.util.Optional;
61 61 import java.util.UUID;
62 62 import java.util.concurrent.ExecutionException;
63 63 import java.util.function.Function;
  64 +import java.util.stream.Collectors;
64 65
65 66 @Slf4j
66 67 @Component
... ... @@ -169,6 +170,11 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme
169 170 }
170 171 }
171 172
  173 + @Override
  174 + public List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds) {
  175 + return tsKvLatestRepository.findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).collect(Collectors.toList()));
  176 + }
  177 +
172 178 private ListenableFuture<Void> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
173 179 ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(tenantId, entityId, query);
174 180 return Futures.transformAsync(future, entryList -> {
... ...
... ... @@ -36,4 +36,9 @@ public interface TsKvLatestRepository extends CrudRepository<TsKvLatestEntity, T
36 36 "WHERE ts_kv_latest.entity_id IN (SELECT id FROM device WHERE tenant_id = :tenant_id limit 100) ORDER BY ts_kv_dictionary.key", nativeQuery = true)
37 37 List<String> getKeysByTenantId(@Param("tenant_id") UUID tenantId);
38 38
  39 + @Query(value = "SELECT DISTINCT ts_kv_dictionary.key AS strKey FROM ts_kv_latest " +
  40 + "INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id " +
  41 + "WHERE ts_kv_latest.entity_id IN :entityIds ORDER BY ts_kv_dictionary.key", nativeQuery = true)
  42 + List<String> findAllKeysByEntityIds(@Param("entityIds") List<UUID> entityIds);
  43 +
39 44 }
... ...
... ... @@ -122,6 +122,11 @@ public class BaseTimeseriesService implements TimeseriesService {
122 122 }
123 123
124 124 @Override
  125 + public List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds) {
  126 + return timeseriesLatestDao.findAllKeysByEntityIds(tenantId, entityIds);
  127 + }
  128 +
  129 + @Override
125 130 public ListenableFuture<Integer> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
126 131 validate(entityId);
127 132 if (tsKvEntry == null) {
... ...
... ... @@ -79,12 +79,17 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
79 79
80 80 protected static List<Long> FIXED_PARTITION = Arrays.asList(new Long[]{0L});
81 81
  82 + private CassandraTsPartitionsCache cassandraTsPartitionsCache;
  83 +
82 84 @Autowired
83 85 private Environment environment;
84 86
85 87 @Value("${cassandra.query.ts_key_value_partitioning}")
86 88 private String partitioning;
87 89
  90 + @Value("${cassandra.query.ts_key_value_partitions_max_cache_size:100000}")
  91 + private long partitionsCacheSize;
  92 +
88 93 @Value("${cassandra.query.ts_key_value_ttl}")
89 94 private long systemTtl;
90 95
... ... @@ -111,13 +116,16 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
111 116 super.startExecutor();
112 117 if (!isInstall()) {
113 118 getFetchStmt(Aggregation.NONE, DESC_ORDER);
114   - }
115   - Optional<NoSqlTsPartitionDate> partition = NoSqlTsPartitionDate.parse(partitioning);
116   - if (partition.isPresent()) {
117   - tsFormat = partition.get();
118   - } else {
119   - log.warn("Incorrect configuration of partitioning {}", partitioning);
120   - throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!");
  119 + Optional<NoSqlTsPartitionDate> partition = NoSqlTsPartitionDate.parse(partitioning);
  120 + if (partition.isPresent()) {
  121 + tsFormat = partition.get();
  122 + if (!isFixedPartitioning() && partitionsCacheSize > 0) {
  123 + cassandraTsPartitionsCache = new CassandraTsPartitionsCache(partitionsCacheSize);
  124 + }
  125 + } else {
  126 + log.warn("Incorrect configuration of partitioning {}", partitioning);
  127 + throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!");
  128 + }
121 129 }
122 130 }
123 131
... ... @@ -175,17 +183,18 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
175 183 }
176 184 ttl = computeTtl(ttl);
177 185 long partition = toPartitionTs(tsKvEntryTs);
178   - log.debug("Saving partition {} for the entity [{}-{}] and key {}", partition, entityId.getEntityType(), entityId.getId(), key);
179   - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder((ttl == 0 ? getPartitionInsertStmt() : getPartitionInsertTtlStmt()).bind());
180   - stmtBuilder.setString(0, entityId.getEntityType().name())
181   - .setUuid(1, entityId.getId())
182   - .setLong(2, partition)
183   - .setString(3, key);
184   - if (ttl > 0) {
185   - stmtBuilder.setInt(4, (int) ttl);
  186 + if (cassandraTsPartitionsCache == null) {
  187 + return doSavePartition(tenantId, entityId, key, ttl, partition);
  188 + } else {
  189 + CassandraPartitionCacheKey partitionSearchKey = new CassandraPartitionCacheKey(entityId, key, partition);
  190 + if (!cassandraTsPartitionsCache.has(partitionSearchKey)) {
  191 + ListenableFuture<Integer> result = doSavePartition(tenantId, entityId, key, ttl, partition);
  192 + Futures.addCallback(result, new CacheCallback<>(partitionSearchKey), MoreExecutors.directExecutor());
  193 + return result;
  194 + } else {
  195 + return Futures.immediateFuture(0);
  196 + }
186 197 }
187   - BoundStatement stmt = stmtBuilder.build();
188   - return getFuture(executeAsyncWrite(tenantId, stmt), rs -> 0);
189 198 }
190 199
191 200 @Override
... ... @@ -461,6 +470,38 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
461 470 return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null);
462 471 }
463 472
  473 + private ListenableFuture<Integer> doSavePartition(TenantId tenantId, EntityId entityId, String key, long ttl, long partition) {
  474 + log.debug("Saving partition {} for the entity [{}-{}] and key {}", partition, entityId.getEntityType(), entityId.getId(), key);
  475 + PreparedStatement preparedStatement = ttl == 0 ? getPartitionInsertStmt() : getPartitionInsertTtlStmt();
  476 + BoundStatement stmt = preparedStatement.bind();
  477 + stmt = stmt.setString(0, entityId.getEntityType().name())
  478 + .setUuid(1, entityId.getId())
  479 + .setLong(2, partition)
  480 + .setString(3, key);
  481 + if (ttl > 0) {
  482 + stmt = stmt.setInt(4, (int) ttl);
  483 + }
  484 + return getFuture(executeAsyncWrite(tenantId, stmt), rs -> 0);
  485 + }
  486 +
  487 + private class CacheCallback<Void> implements FutureCallback<Void> {
  488 + private final CassandraPartitionCacheKey key;
  489 +
  490 + private CacheCallback(CassandraPartitionCacheKey key) {
  491 + this.key = key;
  492 + }
  493 +
  494 + @Override
  495 + public void onSuccess(Void result) {
  496 + cassandraTsPartitionsCache.put(key);
  497 + }
  498 +
  499 + @Override
  500 + public void onFailure(Throwable t) {
  501 +
  502 + }
  503 + }
  504 +
464 505 private long computeTtl(long ttl) {
465 506 if (systemTtl > 0) {
466 507 if (ttl == 0) {
... ...
... ... @@ -87,6 +87,11 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes
87 87 }
88 88
89 89 @Override
  90 + public List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds) {
  91 + return Collections.emptyList();
  92 + }
  93 +
  94 + @Override
90 95 public ListenableFuture<Void> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
91 96 BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind());
92 97 stmtBuilder.setString(0, entityId.getEntityType().name())
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.timeseries;
  17 +
  18 +import lombok.AllArgsConstructor;
  19 +import lombok.Data;
  20 +import org.thingsboard.server.common.data.id.EntityId;
  21 +
  22 +@Data
  23 +@AllArgsConstructor
  24 +public class CassandraPartitionCacheKey {
  25 +
  26 + private EntityId entityId;
  27 + private String key;
  28 + private long partition;
  29 +
  30 +}
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.timeseries;
  17 +
  18 +import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
  19 +import com.github.benmanes.caffeine.cache.Caffeine;
  20 +
  21 +import java.util.concurrent.CompletableFuture;
  22 +
  23 +public class CassandraTsPartitionsCache {
  24 +
  25 + private AsyncLoadingCache<CassandraPartitionCacheKey, Boolean> partitionsCache;
  26 +
  27 + public CassandraTsPartitionsCache(long maxCacheSize) {
  28 + this.partitionsCache = Caffeine.newBuilder()
  29 + .maximumSize(maxCacheSize)
  30 + .buildAsync(key -> {
  31 + throw new IllegalStateException("'get' methods calls are not supported!");
  32 + });
  33 + }
  34 +
  35 + public boolean has(CassandraPartitionCacheKey key) {
  36 + return partitionsCache.getIfPresent(key) != null;
  37 + }
  38 +
  39 + public void put(CassandraPartitionCacheKey key) {
  40 + partitionsCache.put(key, CompletableFuture.completedFuture(true));
  41 + }
  42 +}
... ...
... ... @@ -35,4 +35,6 @@ public interface TimeseriesLatestDao {
35 35 ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query);
36 36
37 37 List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId);
  38 +
  39 + List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds);
38 40 }
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.nosql;
  17 +
  18 +import com.datastax.oss.driver.api.core.ConsistencyLevel;
  19 +import com.datastax.oss.driver.api.core.cql.BoundStatement;
  20 +import com.datastax.oss.driver.api.core.cql.PreparedStatement;
  21 +import com.datastax.oss.driver.api.core.cql.Statement;
  22 +import com.google.common.util.concurrent.Futures;
  23 +import org.junit.Before;
  24 +import org.junit.Test;
  25 +import org.junit.runner.RunWith;
  26 +import org.mockito.Mock;
  27 +import org.mockito.Spy;
  28 +import org.mockito.runners.MockitoJUnitRunner;
  29 +import org.springframework.core.env.Environment;
  30 +import org.springframework.test.util.ReflectionTestUtils;
  31 +import org.thingsboard.server.common.data.id.TenantId;
  32 +import org.thingsboard.server.dao.cassandra.CassandraCluster;
  33 +import org.thingsboard.server.dao.cassandra.guava.GuavaSession;
  34 +import org.thingsboard.server.dao.timeseries.CassandraBaseTimeseriesDao;
  35 +
  36 +import java.util.UUID;
  37 +
  38 +import static org.mockito.Matchers.any;
  39 +import static org.mockito.Matchers.anyInt;
  40 +import static org.mockito.Matchers.anyString;
  41 +import static org.mockito.Mockito.doReturn;
  42 +import static org.mockito.Mockito.times;
  43 +import static org.mockito.Mockito.verify;
  44 +import static org.mockito.Mockito.when;
  45 +
  46 +@RunWith(MockitoJUnitRunner.class)
  47 +public class CassandraPartitionsCacheTest {
  48 +
  49 + @Spy
  50 + private CassandraBaseTimeseriesDao cassandraBaseTimeseriesDao;
  51 +
  52 + @Mock
  53 + private PreparedStatement preparedStatement;
  54 +
  55 + @Mock
  56 + private BoundStatement boundStatement;
  57 +
  58 + @Mock
  59 + private Environment environment;
  60 +
  61 + @Mock
  62 + private CassandraBufferedRateExecutor rateLimiter;
  63 +
  64 + @Mock
  65 + private CassandraCluster cluster;
  66 +
  67 + @Mock
  68 + private GuavaSession session;
  69 +
  70 + @Before
  71 + public void setUp() throws Exception {
  72 + ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "partitioning", "MONTHS");
  73 + ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "partitionsCacheSize", 100000);
  74 + ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "systemTtl", 0);
  75 + ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "setNullValuesEnabled", false);
  76 + ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "environment", environment);
  77 + ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "rateLimiter", rateLimiter);
  78 + ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "cluster", cluster);
  79 +
  80 + when(cluster.getDefaultReadConsistencyLevel()).thenReturn(ConsistencyLevel.ONE);
  81 + when(cluster.getDefaultWriteConsistencyLevel()).thenReturn(ConsistencyLevel.ONE);
  82 + when(cluster.getSession()).thenReturn(session);
  83 + when(session.prepare(anyString())).thenReturn(preparedStatement);
  84 +
  85 + when(preparedStatement.bind()).thenReturn(boundStatement);
  86 +
  87 + when(boundStatement.setString(anyInt(), anyString())).thenReturn(boundStatement);
  88 + when(boundStatement.setUuid(anyInt(), any(UUID.class))).thenReturn(boundStatement);
  89 + when(boundStatement.setLong(anyInt(), any(Long.class))).thenReturn(boundStatement);
  90 +
  91 + doReturn(Futures.immediateFuture(0)).when(cassandraBaseTimeseriesDao).getFuture(any(TbResultSetFuture.class), any());
  92 + }
  93 +
  94 + @Test
  95 + public void testPartitionSave() throws Exception {
  96 + cassandraBaseTimeseriesDao.init();
  97 +
  98 + UUID id = UUID.randomUUID();
  99 + TenantId tenantId = new TenantId(id);
  100 + long tsKvEntryTs = System.currentTimeMillis();
  101 +
  102 + for (int i = 0; i < 50000; i++) {
  103 + cassandraBaseTimeseriesDao.savePartition(tenantId, tenantId, tsKvEntryTs, "test" + i, 0);
  104 + }
  105 + for (int i = 0; i < 60000; i++) {
  106 + cassandraBaseTimeseriesDao.savePartition(tenantId, tenantId, tsKvEntryTs, "test" + i, 0);
  107 + }
  108 + verify(cassandraBaseTimeseriesDao, times(60000)).executeAsyncWrite(any(TenantId.class), any(Statement.class));
  109 + }
  110 +}
... ...
... ... @@ -54,6 +54,8 @@ cassandra.query.default_fetch_size=2000
54 54
55 55 cassandra.query.ts_key_value_partitioning=HOURS
56 56
  57 +cassandra.query.ts_key_value_partitions_max_cache_size=100000
  58 +
57 59 cassandra.query.ts_key_value_ttl=0
58 60
59 61 cassandra.query.debug_events_ttl=604800
... ...
... ... @@ -41,10 +41,16 @@ import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.
41 41 import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils';
42 42 import { RuleChainService } from '@core/http/rule-chain.service';
43 43 import { AliasInfo, StateParams, SubscriptionInfo } from '@core/api/widget-api.models';
44   -import { Datasource, DatasourceType, KeyInfo } from '@app/shared/models/widget.models';
  44 +import { DataKey, Datasource, DatasourceType, KeyInfo } from '@app/shared/models/widget.models';
45 45 import { UtilsService } from '@core/services/utils.service';
46 46 import { AliasFilterType, EntityAlias, EntityAliasFilter, EntityAliasFilterResult } from '@shared/models/alias.models';
47   -import { entityFields, EntityInfo, ImportEntitiesResultInfo, ImportEntityData } from '@shared/models/entity.models';
  47 +import {
  48 + EntitiesKeysByQuery,
  49 + entityFields,
  50 + EntityInfo,
  51 + ImportEntitiesResultInfo,
  52 + ImportEntityData
  53 +} from '@shared/models/entity.models';
48 54 import { EntityRelationService } from '@core/http/entity-relation.service';
49 55 import { deepClone, isDefined, isDefinedAndNotNull } from '@core/utils';
50 56 import { Asset } from '@shared/models/asset.models';
... ... @@ -376,6 +382,13 @@ export class EntityService {
376 382 return this.http.post<PageData<EntityData>>('/api/entitiesQuery/find', query, defaultHttpOptionsFromConfig(config));
377 383 }
378 384
  385 + public findEntityKeysByQuery(query: EntityDataQuery, attributes = true, timeseries = true,
  386 + config?: RequestConfig): Observable<EntitiesKeysByQuery> {
  387 + return this.http.post<EntitiesKeysByQuery>(
  388 + `/api/entitiesQuery/find/keys?attributes=${attributes}&timeseries=${timeseries}`,
  389 + query, defaultHttpOptionsFromConfig(config));
  390 + }
  391 +
379 392 public findAlarmDataByQuery(query: AlarmDataQuery, config?: RequestConfig): Observable<PageData<AlarmData>> {
380 393 return this.http.post<PageData<AlarmData>>('/api/alarmsQuery/find', query, defaultHttpOptionsFromConfig(config));
381 394 }
... ... @@ -595,7 +608,7 @@ export class EntityService {
595 608 return entityTypes;
596 609 }
597 610
598   - private getEntityFieldKeys(entityType: EntityType, searchText: string): Array<string> {
  611 + private getEntityFieldKeys(entityType: EntityType, searchText: string = ''): Array<string> {
599 612 const entityFieldKeys: string[] = [entityFields.createdTime.keyName];
600 613 const query = searchText.toLowerCase();
601 614 switch (entityType) {
... ... @@ -637,7 +650,7 @@ export class EntityService {
637 650 return query ? entityFieldKeys.filter((entityField) => entityField.toLowerCase().indexOf(query) === 0) : entityFieldKeys;
638 651 }
639 652
640   - private getAlarmKeys(searchText: string): Array<string> {
  653 + private getAlarmKeys(searchText: string = ''): Array<string> {
641 654 const alarmKeys: string[] = Object.keys(alarmFields);
642 655 const query = searchText.toLowerCase();
643 656 return query ? alarmKeys.filter((alarmField) => alarmField.toLowerCase().indexOf(query) === 0) : alarmKeys;
... ... @@ -672,6 +685,59 @@ export class EntityService {
672 685 );
673 686 }
674 687
  688 + public getEntityKeysByEntityFilter(filter: EntityFilter, types: DataKeyType[], config?: RequestConfig): Observable<Array<DataKey>> {
  689 + if (!types.length) {
  690 + return of([]);
  691 + }
  692 + let entitiesKeysByQuery$: Observable<EntitiesKeysByQuery>;
  693 + if (filter !== null && types.some(type => [DataKeyType.timeseries, DataKeyType.attribute].includes(type))) {
  694 + const dataQuery = {
  695 + entityFilter: filter,
  696 + pageLink: createDefaultEntityDataPageLink(100),
  697 + };
  698 + entitiesKeysByQuery$ = this.findEntityKeysByQuery(dataQuery, types.includes(DataKeyType.attribute),
  699 + types.includes(DataKeyType.timeseries), config);
  700 + } else {
  701 + entitiesKeysByQuery$ = of({
  702 + attribute: [],
  703 + timeseries: [],
  704 + entityTypes: [],
  705 + });
  706 + }
  707 + return entitiesKeysByQuery$.pipe(
  708 + map((entitiesKeys) => {
  709 + const dataKeys: Array<DataKey> = [];
  710 + types.forEach(type => {
  711 + let keys: Array<string>;
  712 + switch (type) {
  713 + case DataKeyType.entityField:
  714 + if (entitiesKeys.entityTypes.length) {
  715 + const entitiesFields = [];
  716 + entitiesKeys.entityTypes.forEach(entityType => entitiesFields.push(...this.getEntityFieldKeys(entityType)));
  717 + keys = Array.from(new Set(entitiesFields));
  718 + }
  719 + break;
  720 + case DataKeyType.alarm:
  721 + keys = this.getAlarmKeys();
  722 + break;
  723 + case DataKeyType.attribute:
  724 + case DataKeyType.timeseries:
  725 + if (entitiesKeys[type].length) {
  726 + keys = entitiesKeys[type];
  727 + }
  728 + break;
  729 + }
  730 + if (keys) {
  731 + dataKeys.push(...keys.map(key => {
  732 + return {name: key, type};
  733 + }));
  734 + }
  735 + });
  736 + return dataKeys;
  737 + })
  738 + );
  739 + }
  740 +
675 741 public createDatasourcesFromSubscriptionsInfo(subscriptionsInfo: Array<SubscriptionInfo>): Array<Datasource> {
676 742 const datasources = subscriptionsInfo.map(subscriptionInfo => this.createDatasourceFromSubscriptionInfo(subscriptionInfo));
677 743 this.utils.generateColors(datasources);
... ...
... ... @@ -33,87 +33,91 @@
33 33 </button>
34 34 </div>
35 35 </mat-expansion-panel-header>
36   - <div fxLayout="column" fxLayoutGap="0.5em">
37   - <mat-divider></mat-divider>
38   - <mat-form-field fxFlex floatLabel="always">
39   - <mat-label>{{'device-profile.alarm-type' | translate}}</mat-label>
40   - <input required matInput formControlName="alarmType" placeholder="Enter alarm type">
41   - <mat-error *ngIf="alarmFormGroup.get('alarmType').hasError('required')">
42   - {{ 'device-profile.alarm-type-required' | translate }}
43   - </mat-error>
44   - <mat-error *ngIf="alarmFormGroup.get('alarmType').hasError('unique')">
45   - {{ 'device-profile.alarm-type-unique' | translate }}
46   - </mat-error>
47   - </mat-form-field>
48   - </div>
49   - <mat-expansion-panel class="advanced-settings" [expanded]="false">
50   - <mat-expansion-panel-header>
51   - <mat-panel-title>
52   - <div fxFlex fxLayout="row" fxLayoutAlign="end center">
53   - <div class="tb-small" translate>device-profile.advanced-settings</div>
54   - </div>
55   - </mat-panel-title>
56   - </mat-expansion-panel-header>
57   - <mat-checkbox formControlName="propagate" style="display: block; padding-bottom: 16px;">
58   - {{ 'device-profile.propagate-alarm' | translate }}
59   - </mat-checkbox>
60   - <section *ngIf="alarmFormGroup.get('propagate').value === true" style="padding-bottom: 1em;">
61   - <mat-form-field floatLabel="always" class="mat-block">
62   - <mat-label translate>device-profile.alarm-rule-relation-types-list</mat-label>
63   - <mat-chip-list #relationTypesChipList [disabled]="disabled">
64   - <mat-chip
65   - *ngFor="let key of alarmFormGroup.get('propagateRelationTypes').value;"
66   - (removed)="removeRelationType(key)">
67   - {{key}}
68   - <mat-icon matChipRemove>close</mat-icon>
69   - </mat-chip>
70   - <input matInput type="text" placeholder="{{'device-profile.alarm-rule-relation-types-list' | translate}}"
71   - style="max-width: 200px;"
72   - [matChipInputFor]="relationTypesChipList"
73   - [matChipInputSeparatorKeyCodes]="separatorKeysCodes"
74   - (matChipInputTokenEnd)="addRelationType($event)"
75   - [matChipInputAddOnBlur]="true">
76   - </mat-chip-list>
77   - <mat-hint innerHTML="{{ 'device-profile.alarm-rule-relation-types-list-hint' | translate }}"></mat-hint>
  36 + <ng-template matExpansionPanelContent>
  37 + <div fxLayout="column" fxLayoutGap="0.5em">
  38 + <mat-divider></mat-divider>
  39 + <mat-form-field fxFlex floatLabel="always">
  40 + <mat-label>{{'device-profile.alarm-type' | translate}}</mat-label>
  41 + <input required matInput formControlName="alarmType" placeholder="Enter alarm type">
  42 + <mat-error *ngIf="alarmFormGroup.get('alarmType').hasError('required')">
  43 + {{ 'device-profile.alarm-type-required' | translate }}
  44 + </mat-error>
  45 + <mat-error *ngIf="alarmFormGroup.get('alarmType').hasError('unique')">
  46 + {{ 'device-profile.alarm-type-unique' | translate }}
  47 + </mat-error>
78 48 </mat-form-field>
79   - </section>
80   - </mat-expansion-panel>
81   - <div fxFlex fxLayout="column">
82   - <div translate class="tb-small" style="padding-bottom: 8px;">device-profile.create-alarm-rules</div>
83   - <tb-create-alarm-rules formControlName="createRules"
84   - style="padding-bottom: 16px;"
85   - [deviceProfileId]="deviceProfileId">
86   - </tb-create-alarm-rules>
87   - <div translate class="tb-small" style="padding-bottom: 8px;">device-profile.clear-alarm-rule</div>
88   - <div fxLayout="row" fxLayoutGap="8px;" fxLayoutAlign="start center"
89   - [fxShow]="alarmFormGroup.get('clearRule').value"
90   - style="padding-bottom: 8px;">
91   - <div class="clear-alarm-rule" fxFlex fxLayout="row">
92   - <tb-alarm-rule formControlName="clearRule" fxFlex [deviceProfileId]="deviceProfileId">
93   - </tb-alarm-rule>
94   - </div>
95   - <button *ngIf="!disabled"
96   - mat-icon-button color="primary" style="min-width: 40px;"
97   - type="button"
98   - (click)="removeClearAlarmRule()"
99   - matTooltip="{{ 'action.remove' | translate }}"
100   - matTooltipPosition="above">
101   - <mat-icon>remove_circle_outline</mat-icon>
102   - </button>
103 49 </div>
104   - <div *ngIf="disabled && !alarmFormGroup.get('clearRule').value">
105   - <span translate fxLayoutAlign="center center" style="margin: 16px 0"
106   - class="tb-prompt">device-profile.no-clear-alarm-rule</span>
107   - </div>
108   - <div *ngIf="!disabled" [fxShow]="!alarmFormGroup.get('clearRule').value">
109   - <button mat-stroked-button color="primary"
110   - type="button"
111   - (click)="addClearAlarmRule()"
112   - matTooltip="{{ 'device-profile.add-clear-alarm-rule' | translate }}"
113   - matTooltipPosition="above">
114   - <mat-icon class="button-icon">add_circle_outline</mat-icon>
115   - {{ 'device-profile.add-clear-alarm-rule' | translate }}
116   - </button>
  50 + <mat-expansion-panel class="advanced-settings" [expanded]="false">
  51 + <mat-expansion-panel-header>
  52 + <mat-panel-title>
  53 + <div fxFlex fxLayout="row" fxLayoutAlign="end center">
  54 + <div class="tb-small" translate>device-profile.advanced-settings</div>
  55 + </div>
  56 + </mat-panel-title>
  57 + </mat-expansion-panel-header>
  58 + <ng-template matExpansionPanelContent>
  59 + <mat-checkbox formControlName="propagate" style="display: block; padding-bottom: 16px;">
  60 + {{ 'device-profile.propagate-alarm' | translate }}
  61 + </mat-checkbox>
  62 + <section *ngIf="alarmFormGroup.get('propagate').value === true" style="padding-bottom: 1em;">
  63 + <mat-form-field floatLabel="always" class="mat-block">
  64 + <mat-label translate>device-profile.alarm-rule-relation-types-list</mat-label>
  65 + <mat-chip-list #relationTypesChipList [disabled]="disabled">
  66 + <mat-chip
  67 + *ngFor="let key of alarmFormGroup.get('propagateRelationTypes').value;"
  68 + (removed)="removeRelationType(key)">
  69 + {{key}}
  70 + <mat-icon matChipRemove>close</mat-icon>
  71 + </mat-chip>
  72 + <input matInput type="text" placeholder="{{'device-profile.alarm-rule-relation-types-list' | translate}}"
  73 + style="max-width: 200px;"
  74 + [matChipInputFor]="relationTypesChipList"
  75 + [matChipInputSeparatorKeyCodes]="separatorKeysCodes"
  76 + (matChipInputTokenEnd)="addRelationType($event)"
  77 + [matChipInputAddOnBlur]="true">
  78 + </mat-chip-list>
  79 + <mat-hint innerHTML="{{ 'device-profile.alarm-rule-relation-types-list-hint' | translate }}"></mat-hint>
  80 + </mat-form-field>
  81 + </section>
  82 + </ng-template>
  83 + </mat-expansion-panel>
  84 + <div fxFlex fxLayout="column">
  85 + <div translate class="tb-small" style="padding-bottom: 8px;">device-profile.create-alarm-rules</div>
  86 + <tb-create-alarm-rules formControlName="createRules"
  87 + style="padding-bottom: 16px;"
  88 + [deviceProfileId]="deviceProfileId">
  89 + </tb-create-alarm-rules>
  90 + <div translate class="tb-small" style="padding-bottom: 8px;">device-profile.clear-alarm-rule</div>
  91 + <div fxLayout="row" fxLayoutGap="8px;" fxLayoutAlign="start center"
  92 + [fxShow]="alarmFormGroup.get('clearRule').value"
  93 + style="padding-bottom: 8px;">
  94 + <div class="clear-alarm-rule" fxFlex fxLayout="row">
  95 + <tb-alarm-rule formControlName="clearRule" fxFlex [deviceProfileId]="deviceProfileId">
  96 + </tb-alarm-rule>
  97 + </div>
  98 + <button *ngIf="!disabled"
  99 + mat-icon-button color="primary" style="min-width: 40px;"
  100 + type="button"
  101 + (click)="removeClearAlarmRule()"
  102 + matTooltip="{{ 'action.remove' | translate }}"
  103 + matTooltipPosition="above">
  104 + <mat-icon>remove_circle_outline</mat-icon>
  105 + </button>
  106 + </div>
  107 + <div *ngIf="disabled && !alarmFormGroup.get('clearRule').value">
  108 + <span translate fxLayoutAlign="center center" style="margin: 16px 0"
  109 + class="tb-prompt">device-profile.no-clear-alarm-rule</span>
  110 + </div>
  111 + <div *ngIf="!disabled" [fxShow]="!alarmFormGroup.get('clearRule').value">
  112 + <button mat-stroked-button color="primary"
  113 + type="button"
  114 + (click)="addClearAlarmRule()"
  115 + matTooltip="{{ 'device-profile.add-clear-alarm-rule' | translate }}"
  116 + matTooltipPosition="above">
  117 + <mat-icon class="button-icon">add_circle_outline</mat-icon>
  118 + {{ 'device-profile.add-clear-alarm-rule' | translate }}
  119 + </button>
  120 + </div>
117 121 </div>
118   - </div>
  122 + </ng-template>
119 123 </mat-expansion-panel>
... ...
... ... @@ -91,10 +91,12 @@
91 91 <div translate>device-profile.profile-configuration</div>
92 92 </mat-panel-title>
93 93 </mat-expansion-panel-header>
94   - <tb-device-profile-configuration
95   - formControlName="configuration"
96   - required>
97   - </tb-device-profile-configuration>
  94 + <ng-template matExpansionPanelContent>
  95 + <tb-device-profile-configuration
  96 + formControlName="configuration"
  97 + required>
  98 + </tb-device-profile-configuration>
  99 + </ng-template>
98 100 </mat-expansion-panel>
99 101 <mat-expansion-panel *ngIf="displayTransportConfiguration" [expanded]="true">
100 102 <mat-expansion-panel-header>
... ... @@ -102,10 +104,12 @@
102 104 <div translate>device-profile.transport-configuration</div>
103 105 </mat-panel-title>
104 106 </mat-expansion-panel-header>
105   - <tb-device-profile-transport-configuration
106   - formControlName="transportConfiguration"
107   - required>
108   - </tb-device-profile-transport-configuration>
  107 + <ng-template matExpansionPanelContent>
  108 + <tb-device-profile-transport-configuration
  109 + formControlName="transportConfiguration"
  110 + required>
  111 + </tb-device-profile-transport-configuration>
  112 + </ng-template>
109 113 </mat-expansion-panel>
110 114 <mat-expansion-panel [expanded]="false">
111 115 <mat-expansion-panel-header>
... ... @@ -115,10 +119,12 @@
115 119 entityForm.get('profileData.alarms').value.length : 0} }}</div>
116 120 </mat-panel-title>
117 121 </mat-expansion-panel-header>
118   - <tb-device-profile-alarms
119   - formControlName="alarms"
120   - [deviceProfileId]="deviceProfileId">
121   - </tb-device-profile-alarms>
  122 + <ng-template matExpansionPanelContent>
  123 + <tb-device-profile-alarms
  124 + formControlName="alarms"
  125 + [deviceProfileId]="deviceProfileId">
  126 + </tb-device-profile-alarms>
  127 + </ng-template>
122 128 </mat-expansion-panel>
123 129 <mat-expansion-panel [expanded]="true">
124 130 <mat-expansion-panel-header>
... ... @@ -126,9 +132,11 @@
126 132 <div translate>device-profile.device-provisioning</div>
127 133 </mat-panel-title>
128 134 </mat-expansion-panel-header>
129   - <tb-device-profile-provision-configuration
130   - formControlName="provisionConfiguration">
131   - </tb-device-profile-provision-configuration>
  135 + <ng-template matExpansionPanelContent>
  136 + <tb-device-profile-provision-configuration
  137 + formControlName="provisionConfiguration">
  138 + </tb-device-profile-provision-configuration>
  139 + </ng-template>
132 140 </mat-expansion-panel>
133 141 </mat-accordion>
134 142 </div>
... ...
... ... @@ -22,9 +22,11 @@
22 22 <div translate>tenant-profile.profile-configuration</div>
23 23 </mat-panel-title>
24 24 </mat-expansion-panel-header>
25   - <tb-tenant-profile-configuration
26   - formControlName="configuration"
27   - required>
28   - </tb-tenant-profile-configuration>
  25 + <ng-template matExpansionPanelContent>
  26 + <tb-tenant-profile-configuration
  27 + formControlName="configuration"
  28 + required>
  29 + </tb-tenant-profile-configuration>
  30 + </ng-template>
29 31 </mat-expansion-panel>
30 32 </form>
... ...
... ... @@ -36,7 +36,7 @@ import { EntityService } from '@core/http/entity.service';
36 36 import { DataKeysCallbacks } from '@home/components/widget/data-keys.component.models';
37 37 import { DataKeyType } from '@shared/models/telemetry/telemetry.models';
38 38 import { Observable, of } from 'rxjs';
39   -import { map, mergeMap, tap } from 'rxjs/operators';
  39 +import { map, mergeMap, publishReplay, refCount, tap } from 'rxjs/operators';
40 40 import { alarmFields } from '@shared/models/alarm.models';
41 41 import { JsFuncComponent } from '@shared/components/js-func.component';
42 42 import { JsonFormComponentData } from '@shared/components/json-form/json-form-component.models';
... ... @@ -95,6 +95,7 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con
95 95
96 96 filteredKeys: Observable<Array<string>>;
97 97 private latestKeySearchResult: Array<string> = null;
  98 + private fetchObservable$: Observable<Array<string>> = null;
98 99
99 100 keySearchText = '';
100 101
... ... @@ -205,31 +206,42 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con
205 206 }
206 207
207 208 private fetchKeys(searchText?: string): Observable<Array<string>> {
208   - if (this.latestKeySearchResult === null || this.keySearchText !== searchText) {
  209 + if (this.keySearchText !== searchText || this.latestKeySearchResult === null) {
209 210 this.keySearchText = searchText;
210   - let fetchObservable: Observable<Array<DataKey>> = null;
  211 + const dataKeyFilter = this.createKeyFilter(this.keySearchText);
  212 + return this.getKeys().pipe(
  213 + map(name => name.filter(dataKeyFilter)),
  214 + tap(res => this.latestKeySearchResult = res)
  215 + );
  216 + }
  217 + return of(this.latestKeySearchResult);
  218 + }
  219 +
  220 + private getKeys() {
  221 + if (this.fetchObservable$ === null) {
  222 + let fetchObservable: Observable<Array<DataKey>>;
211 223 if (this.modelValue.type === DataKeyType.alarm) {
212   - const dataKeyFilter = this.createDataKeyFilter(this.keySearchText);
213   - fetchObservable = of(this.alarmKeys.filter(dataKeyFilter));
  224 + fetchObservable = of(this.alarmKeys);
214 225 } else {
215 226 if (this.entityAliasId) {
216 227 const dataKeyTypes = [this.modelValue.type];
217   - fetchObservable = this.callbacks.fetchEntityKeys(this.entityAliasId, this.keySearchText, dataKeyTypes);
  228 + fetchObservable = this.callbacks.fetchEntityKeys(this.entityAliasId, dataKeyTypes);
218 229 } else {
219 230 fetchObservable = of([]);
220 231 }
221 232 }
222   - return fetchObservable.pipe(
  233 + this.fetchObservable$ = fetchObservable.pipe(
223 234 map((dataKeys) => dataKeys.map((dataKey) => dataKey.name)),
224   - tap(res => this.latestKeySearchResult = res)
  235 + publishReplay(1),
  236 + refCount()
225 237 );
226 238 }
227   - return of(this.latestKeySearchResult);
  239 + return this.fetchObservable$;
228 240 }
229 241
230   - private createDataKeyFilter(query: string): (key: DataKey) => boolean {
  242 + private createKeyFilter(query: string): (key: string) => boolean {
231 243 const lowercaseQuery = query.toLowerCase();
232   - return key => key.name.toLowerCase().indexOf(lowercaseQuery) === 0;
  244 + return key => key.toLowerCase().startsWith(lowercaseQuery);
233 245 }
234 246
235 247 public validateOnSubmit() {
... ...
... ... @@ -20,5 +20,5 @@ import { Observable } from 'rxjs';
20 20
21 21 export interface DataKeysCallbacks {
22 22 generateDataKey: (chip: any, type: DataKeyType) => DataKey;
23   - fetchEntityKeys: (entityAliasId: string, query: string, types: Array<DataKeyType>) => Observable<Array<DataKey>>;
  23 + fetchEntityKeys: (entityAliasId: string, types: Array<DataKeyType>) => Observable<Array<DataKey>>;
24 24 }
... ...
... ... @@ -38,7 +38,7 @@ import {
38 38 Validators
39 39 } from '@angular/forms';
40 40 import { Observable, of } from 'rxjs';
41   -import { filter, map, mergeMap, share, tap } from 'rxjs/operators';
  41 +import { filter, map, mergeMap, publishReplay, refCount, share, tap } from 'rxjs/operators';
42 42 import { Store } from '@ngrx/store';
43 43 import { AppState } from '@app/core/core.state';
44 44 import { TranslateService } from '@ngx-translate/core';
... ... @@ -142,6 +142,7 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, AfterVie
142 142
143 143 searchText = '';
144 144 private latestSearchTextResult: Array<DataKey> = null;
  145 + private fetchObservable$: Observable<Array<DataKey>> = null;
145 146
146 147 private dirty = false;
147 148
... ... @@ -260,6 +261,7 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, AfterVie
260 261 if (!change.firstChange && change.currentValue !== change.previousValue) {
261 262 if (propName === 'entityAliasId') {
262 263 this.searchText = '';
  264 + this.fetchObservable$ = null;
263 265 this.latestSearchTextResult = null;
264 266 this.dirty = true;
265 267 } else if (['widgetType', 'datasourceType'].includes(propName)) {
... ... @@ -405,14 +407,24 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, AfterVie
405 407 return key ? key.name : undefined;
406 408 }
407 409
408   - fetchKeys(searchText?: string): Observable<Array<DataKey>> {
409   - if (this.latestSearchTextResult === null || this.searchText !== searchText) {
  410 + private fetchKeys(searchText?: string): Observable<Array<DataKey>> {
  411 + if (this.searchText !== searchText || this.latestSearchTextResult === null) {
410 412 this.searchText = searchText;
411   - let fetchObservable: Observable<Array<DataKey>> = null;
  413 + const dataKeyFilter = this.createDataKeyFilter(this.searchText);
  414 + return this.getKeys().pipe(
  415 + map(name => name.filter(dataKeyFilter)),
  416 + tap(res => this.latestSearchTextResult = res)
  417 + );
  418 + }
  419 + return of(this.latestSearchTextResult);
  420 + }
  421 +
  422 + private getKeys(): Observable<Array<DataKey>> {
  423 + if (this.fetchObservable$ === null) {
  424 + let fetchObservable: Observable<Array<DataKey>>;
412 425 if (this.datasourceType === DatasourceType.function) {
413   - const dataKeyFilter = this.createDataKeyFilter(this.searchText);
414 426 const targetKeysList = this.widgetType === widgetType.alarm ? this.alarmKeys : this.functionTypeKeys;
415   - fetchObservable = of(targetKeysList.filter(dataKeyFilter));
  427 + fetchObservable = of(targetKeysList);
416 428 } else {
417 429 if (this.entityAliasId) {
418 430 const dataKeyTypes = [DataKeyType.timeseries];
... ... @@ -420,24 +432,25 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, AfterVie
420 432 dataKeyTypes.push(DataKeyType.attribute);
421 433 dataKeyTypes.push(DataKeyType.entityField);
422 434 if (this.widgetType === widgetType.alarm) {
423   - dataKeyTypes.push(DataKeyType.alarm);
  435 + dataKeyTypes.push(DataKeyType.alarm);
424 436 }
425 437 }
426   - fetchObservable = this.callbacks.fetchEntityKeys(this.entityAliasId, this.searchText, dataKeyTypes);
  438 + fetchObservable = this.callbacks.fetchEntityKeys(this.entityAliasId, dataKeyTypes);
427 439 } else {
428 440 fetchObservable = of([]);
429 441 }
430 442 }
431   - return fetchObservable.pipe(
432   - tap(res => this.latestSearchTextResult = res)
  443 + this.fetchObservable$ = fetchObservable.pipe(
  444 + publishReplay(1),
  445 + refCount()
433 446 );
434 447 }
435   - return of(this.latestSearchTextResult);
  448 + return this.fetchObservable$;
436 449 }
437 450
438 451 private createDataKeyFilter(query: string): (key: DataKey) => boolean {
439 452 const lowercaseQuery = query.toLowerCase();
440   - return key => key.name.toLowerCase().indexOf(lowercaseQuery) === 0;
  453 + return key => key.name.toLowerCase().startsWith(lowercaseQuery);
441 454 }
442 455
443 456 textIsNotEmpty(text: string): boolean {
... ...
... ... @@ -608,26 +608,32 @@ export default abstract class LeafletMap {
608 608 return polygon;
609 609 }
610 610
611   - updatePoints(pointsData: FormattedData[], getTooltip: (point: FormattedData, setTooltip?: boolean) => string) {
  611 + updatePoints(pointsData: FormattedData[][], getTooltip: (point: FormattedData) => string) {
  612 + if(pointsData.length) {
612 613 if (this.points) {
613   - this.map.removeLayer(this.points);
  614 + this.map.removeLayer(this.points);
614 615 }
615 616 this.points = new FeatureGroup();
616   - pointsData.filter(pdata => !!this.convertPosition(pdata)).forEach(data => {
617   - const point = L.circleMarker(this.convertPosition(data), {
618   - color: this.options.pointColor,
619   - radius: this.options.pointSize
620   - });
621   - if (!this.options.pointTooltipOnRightPanel) {
622   - point.on('click', () => getTooltip(data));
623   - }
624   - else {
625   - createTooltip(point, this.options, data.$datasource, getTooltip(data, false));
626   - }
627   - this.points.addLayer(point);
  617 + }
  618 + for(let i = 0; i < pointsData.length; i++) {
  619 + const pointsList = pointsData[i];
  620 + pointsList.filter(pdata => !!this.convertPosition(pdata)).forEach(data => {
  621 + const point = L.circleMarker(this.convertPosition(data), {
  622 + color: this.options.pointColor,
  623 + radius: this.options.pointSize
  624 + });
  625 + if (!this.options.pointTooltipOnRightPanel) {
  626 + point.on('click', () => getTooltip(data));
  627 + } else {
  628 + createTooltip(point, this.options, data.$datasource, getTooltip(data));
  629 + }
  630 + this.points.addLayer(point);
628 631 });
  632 + }
  633 + if(pointsData.length) {
629 634 this.map.addLayer(this.points);
630 635 }
  636 + }
631 637
632 638 // Polyline
633 639
... ...
... ... @@ -28,8 +28,12 @@
28 28 </button>
29 29 </div>
30 30 <div class="trip-animation-tooltip md-whiteframe-z4" fxLayout="column"
31   - [ngClass]="{'trip-animation-tooltip-hidden':!visibleTooltip}" [innerHTML]="mainTooltip"
32   - [ngStyle]="{'background-color': settings.tooltipColor, 'opacity': settings.tooltipOpacity, 'color': settings.tooltipFontColor}">
  31 + [ngClass]="{'trip-animation-tooltip-hidden':!visibleTooltip}"
  32 + [ngStyle]="{'background-color': settings.tooltipColor, 'opacity': settings.tooltipOpacity, 'color': settings.tooltipFontColor}">
  33 + <div *ngFor="let mainTooltip of mainTooltips"
  34 + [innerHTML]="mainTooltip"
  35 + style="padding: 10px 0">
  36 + </div>
33 37 </div>
34 38 </div>
35 39 <tb-history-selector *ngIf="historicalData"
... ...
... ... @@ -73,6 +73,9 @@
73 73 }
74 74
75 75 .trip-animation-tooltip {
  76 + display: flex;
  77 + overflow: auto;
  78 + max-height: 90%;
76 79 position: absolute;
77 80 top: 30px;
78 81 right: 0;
... ... @@ -86,4 +89,4 @@
86 89 }
87 90 }
88 91 }
89   -}
\ No newline at end of file
  92 +}
... ...
... ... @@ -47,6 +47,9 @@ import moment from 'moment';
47 47 import { isUndefined } from '@core/utils';
48 48 import { ResizeObserver } from '@juggle/resize-observer';
49 49
  50 +interface dataMap {
  51 + [key: string] : FormattedData
  52 +}
50 53
51 54 @Component({
52 55 // tslint:disable-next-line:component-selector
... ... @@ -70,7 +73,7 @@ export class TripAnimationComponent implements OnInit, AfterViewInit, OnDestroy
70 73 interpolatedTimeData = [];
71 74 widgetConfig: WidgetConfig;
72 75 settings: TripAnimationSettings;
73   - mainTooltip = '';
  76 + mainTooltips = [];
74 77 visibleTooltip = false;
75 78 activeTrip: FormattedData;
76 79 label: string;
... ... @@ -115,7 +118,7 @@ export class TripAnimationComponent implements OnInit, AfterViewInit, OnDestroy
115 118 this.historicalData = parseArray(this.ctx.data).filter(arr => arr.length);
116 119 if (this.historicalData.length) {
117 120 this.calculateIntervals();
118   - this.timeUpdated(this.currentTime && this.currentTime > this.minTime ? this.currentTime : this.minTime);
  121 + this.timeUpdated(this.minTime);
119 122 }
120 123 this.mapWidget.map.map?.invalidateSize();
121 124 this.cd.detectChanges();
... ... @@ -140,32 +143,39 @@ export class TripAnimationComponent implements OnInit, AfterViewInit, OnDestroy
140 143 this.currentTime = time;
141 144 const currentPosition = this.interpolatedTimeData
142 145 .map(dataSource => dataSource[time])
143   - .filter(ds => ds);
144   - if (isUndefined(currentPosition[0])) {
145   - const timePoints = Object.keys(this.interpolatedTimeData[0]).map(item => parseInt(item, 10));
146   - for (let i = 1; i < timePoints.length; i++) {
147   - if (timePoints[i - 1] < time && timePoints[i] > time) {
148   - const beforePosition = this.interpolatedTimeData[0][timePoints[i - 1]];
149   - const afterPosition = this.interpolatedTimeData[0][timePoints[i]];
150   - const ratio = getRatio(timePoints[i - 1], timePoints[i], time);
151   - currentPosition[0] = {
152   - ...beforePosition,
153   - time,
154   - ...interpolateOnLineSegment(beforePosition, afterPosition, this.settings.latKeyName, this.settings.lngKeyName, ratio)
  146 + for(let j = 0; j < this.interpolatedTimeData.length; j++) {
  147 + if (isUndefined(currentPosition[j])) {
  148 + const timePoints = Object.keys(this.interpolatedTimeData[j]).map(item => parseInt(item, 10));
  149 + for (let i = 1; i < timePoints.length; i++) {
  150 + if (timePoints[i - 1] < time && timePoints[i] > time) {
  151 + const beforePosition = this.interpolatedTimeData[j][timePoints[i - 1]];
  152 + const afterPosition = this.interpolatedTimeData[j][timePoints[i]];
  153 + const ratio = getRatio(timePoints[i - 1], timePoints[i], time);
  154 + currentPosition[j] = {
  155 + ...beforePosition,
  156 + time,
  157 + ...interpolateOnLineSegment(beforePosition, afterPosition, this.settings.latKeyName, this.settings.lngKeyName, ratio)
  158 + }
  159 + break;
155 160 }
156   - break;
157 161 }
158 162 }
159 163 }
  164 + for(let j = 0; j < this.interpolatedTimeData.length; j++) {
  165 + if (isUndefined(currentPosition[j])) {
  166 + currentPosition[j] = this.calculateLastPoints(this.interpolatedTimeData[j], time);
  167 + }
  168 + }
160 169 this.calcLabel();
161   - this.calcTooltip(currentPosition.find(position => position.entityName === this.activeTrip.entityName));
  170 + this.calcMainTooltip(currentPosition);
162 171 if (this.mapWidget && this.mapWidget.map && this.mapWidget.map.map) {
163   - this.mapWidget.map.updatePolylines(this.interpolatedTimeData.map(ds => _.values(ds)), true, this.activeTrip);
  172 + const formattedInterpolatedTimeData = this.interpolatedTimeData.map(ds => _.values(ds));
  173 + this.mapWidget.map.updatePolylines(formattedInterpolatedTimeData, true);
164 174 if (this.settings.showPolygon) {
165 175 this.mapWidget.map.updatePolygons(this.interpolatedTimeData);
166 176 }
167 177 if (this.settings.showPoints) {
168   - this.mapWidget.map.updatePoints(_.values(_.union(this.interpolatedTimeData)[0]), this.calcTooltip);
  178 + this.mapWidget.map.updatePoints(formattedInterpolatedTimeData.map(ds => _.union(ds)), this.calcTooltip);
169 179 }
170 180 this.mapWidget.map.updateMarkers(currentPosition, true, (trip) => {
171 181 this.activeTrip = trip;
... ... @@ -177,6 +187,23 @@ export class TripAnimationComponent implements OnInit, AfterViewInit, OnDestroy
177 187 setActiveTrip() {
178 188 }
179 189
  190 + private calculateLastPoints(dataSource: dataMap, time: number): FormattedData {
  191 + const timeArr = Object.keys(dataSource);
  192 + let index = timeArr.findIndex((dtime, index) => {
  193 + return Number(dtime) >= time;
  194 + });
  195 +
  196 + if(index !== -1) {
  197 + if(Number(timeArr[index]) !== time && index !== 0) {
  198 + index--;
  199 + }
  200 + } else {
  201 + index = timeArr.length - 1;
  202 + }
  203 +
  204 + return dataSource[timeArr[index]];
  205 + }
  206 +
180 207 calculateIntervals() {
181 208 this.historicalData.forEach((dataSource, index) => {
182 209 this.minTime = dataSource[0]?.time || Infinity;
... ... @@ -194,16 +221,19 @@ export class TripAnimationComponent implements OnInit, AfterViewInit, OnDestroy
194 221 }
195 222 }
196 223
197   - calcTooltip = (point?: FormattedData): string => {
  224 + calcTooltip = (point: FormattedData): string => {
198 225 const data = point ? point : this.activeTrip;
199 226 const tooltipPattern: string = this.settings.useTooltipFunction ?
200 227 safeExecute(this.settings.tooltipFunction, [data, this.historicalData, point.dsIndex]) : this.settings.tooltipPattern;
201   - const tooltipText = parseWithTranslation.parseTemplate(tooltipPattern, data, true);
202   - this.mainTooltip = this.sanitizer.sanitize(
203   - SecurityContext.HTML, tooltipText);
204   - this.cd.detectChanges();
205   - this.activeTrip = point;
206   - return tooltipText;
  228 + return parseWithTranslation.parseTemplate(tooltipPattern, data, true);
  229 + }
  230 +
  231 + private calcMainTooltip(points: FormattedData[]): void {
  232 + const tooltips = [];
  233 + for (let point of points) {
  234 + tooltips.push(this.sanitizer.sanitize(SecurityContext.HTML, this.calcTooltip(point)));
  235 + }
  236 + this.mainTooltips = tooltips;
207 237 }
208 238
209 239 calcLabel() {
... ...
... ... @@ -54,13 +54,13 @@ import { UtilsService } from '@core/services/utils.service';
54 54 import { DataKeyType } from '@shared/models/telemetry/telemetry.models';
55 55 import { TranslateService } from '@ngx-translate/core';
56 56 import { EntityType } from '@shared/models/entity-type.models';
57   -import { forkJoin, Observable, of, Subscription } from 'rxjs';
  57 +import { Observable, of, Subscription } from 'rxjs';
58 58 import { WidgetConfigCallbacks } from '@home/components/widget/widget-config.component.models';
59 59 import {
60 60 EntityAliasDialogComponent,
61 61 EntityAliasDialogData
62 62 } from '@home/components/alias/entity-alias-dialog.component';
63   -import { catchError, map, mergeMap, tap } from 'rxjs/operators';
  63 +import { catchError, mergeMap, tap } from 'rxjs/operators';
64 64 import { MatDialog } from '@angular/material/dialog';
65 65 import { EntityService } from '@core/http/entity.service';
66 66 import { JsonFormComponentData } from '@shared/components/json-form/json-form-component.models';
... ... @@ -792,54 +792,16 @@ export class WidgetConfigComponent extends PageComponent implements OnInit, Cont
792 792 );
793 793 }
794 794
795   - private fetchEntityKeys(entityAliasId: string, query: string, dataKeyTypes: Array<DataKeyType>): Observable<Array<DataKey>> {
796   - return this.aliasController.resolveSingleEntityInfo(entityAliasId).pipe(
797   - mergeMap((entity) => {
798   - if (entity) {
799   - const fetchEntityTasks: Array<Observable<Array<DataKey>>> = [];
800   - for (const dataKeyType of dataKeyTypes) {
801   - fetchEntityTasks.push(
802   - this.entityService.getEntityKeys(
803   - {entityType: entity.entityType, id: entity.id},
804   - query,
805   - dataKeyType,
806   - {ignoreLoading: true, ignoreErrors: true}
807   - ).pipe(
808   - map((keys) => {
809   - const dataKeys: Array<DataKey> = [];
810   - for (const key of keys) {
811   - dataKeys.push({name: key, type: dataKeyType});
812   - }
813   - return dataKeys;
814   - }
815   - ),
816   - catchError(() => of([]))
817   - ));
818   - }
819   - return forkJoin(fetchEntityTasks).pipe(
820   - map(arrayOfDataKeys => {
821   - const result = new Array<DataKey>();
822   - arrayOfDataKeys.forEach((dataKeyArray) => {
823   - result.push(...dataKeyArray);
824   - });
825   - return result;
826   - }
827   - ));
828   - } else if (dataKeyTypes.includes(DataKeyType.alarm)) {
829   - return this.entityService.getEntityKeys(null, query, DataKeyType.alarm).pipe(
830   - map((keys) => {
831   - const dataKeys: Array<DataKey> = [];
832   - for (const key of keys) {
833   - dataKeys.push({name: key, type: DataKeyType.alarm});
834   - }
835   - return dataKeys;
836   - }
837   - ),
838   - catchError(() => of([]))
839   - );
840   - } else {
841   - return of([]);
842   - }
  795 + private fetchEntityKeys(entityAliasId: string, dataKeyTypes: Array<DataKeyType>): Observable<Array<DataKey>> {
  796 + return this.aliasController.getAliasInfo(entityAliasId).pipe(
  797 + mergeMap((aliasInfo) => {
  798 + return this.entityService.getEntityKeysByEntityFilter(
  799 + aliasInfo.entityFilter,
  800 + dataKeyTypes,
  801 + {ignoreLoading: true, ignoreErrors: true}
  802 + ).pipe(
  803 + catchError(() => of([]))
  804 + );
843 805 }),
844 806 catchError(() => of([] as Array<DataKey>))
845 807 );
... ...
... ... @@ -64,6 +64,12 @@ export interface EntityField {
64 64 time?: boolean;
65 65 }
66 66
  67 +export interface EntitiesKeysByQuery {
  68 + attribute: Array<string>;
  69 + timeseries: Array<string>;
  70 + entityTypes: EntityType[];
  71 +}
  72 +
67 73 export const entityFields: {[fieldName: string]: EntityField} = {
68 74 createdTime: {
69 75 keyName: 'createdTime',
... ...