Merge branch 'feature/entity-data-query' of github.com:thingsboard/thingsboard i…

…nto feature/entity-data-query

@@ -399,6 +399,14 @@ public abstract class AbstractControllerTest {
         return readResponse(doPost(urlTemplate, content, params).andExpect(status().isOk()), responseClass);
         return readResponse(doPost(urlTemplate, content, params).andExpect(status().isOk()), responseClass);
     }
     }
 
 
+    protected <T,R> R doPostWithResponse(String urlTemplate, T content, Class<R> responseClass, String... params) throws Exception {
+        return readResponse(doPost(urlTemplate, content, params).andExpect(status().isOk()), responseClass);
+    }
+
+    protected <T,R> R doPostWithTypedResponse(String urlTemplate, T content, TypeReference<R> responseType, String... params) throws Exception {
+        return readResponse(doPost(urlTemplate, content, params).andExpect(status().isOk()), responseType);
+    }
+
     protected <T> T doPostAsync(String urlTemplate, T content, Class<T> responseClass, ResultMatcher resultMatcher, String... params) throws Exception {
     protected <T> T doPostAsync(String urlTemplate, T content, Class<T> responseClass, ResultMatcher resultMatcher, String... params) throws Exception {
         return readResponse(doPostAsync(urlTemplate, content, DEFAULT_TIMEOUT, params).andExpect(resultMatcher), responseClass);
         return readResponse(doPostAsync(urlTemplate, content, DEFAULT_TIMEOUT, params).andExpect(resultMatcher), responseClass);
     }
     }

+/**
+ * Copyright © 2016-2020 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.controller;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.thingsboard.server.common.data.DataConstants;
+import org.thingsboard.server.common.data.Device;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.Tenant;
+import org.thingsboard.server.common.data.User;
+import org.thingsboard.server.common.data.id.DeviceId;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.page.PageData;
+import org.thingsboard.server.common.data.query.DeviceTypeFilter;
+import org.thingsboard.server.common.data.query.EntityCountQuery;
+import org.thingsboard.server.common.data.query.EntityData;
+import org.thingsboard.server.common.data.query.EntityDataPageLink;
+import org.thingsboard.server.common.data.query.EntityDataQuery;
+import org.thingsboard.server.common.data.query.EntityDataSortOrder;
+import org.thingsboard.server.common.data.query.EntityKey;
+import org.thingsboard.server.common.data.query.EntityKeyType;
+import org.thingsboard.server.common.data.query.EntityListFilter;
+import org.thingsboard.server.common.data.query.KeyFilter;
+import org.thingsboard.server.common.data.query.NumericFilterPredicate;
+import org.thingsboard.server.common.data.security.Authority;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+public abstract class BaseEntityQueryControllerTest extends AbstractControllerTest {
+
+    private Tenant savedTenant;
+    private User tenantAdmin;
+
+    @Before
+    public void beforeTest() throws Exception {
+        loginSysAdmin();
+
+        Tenant tenant = new Tenant();
+        tenant.setTitle("My tenant");
+        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        Assert.assertNotNull(savedTenant);
+
+        tenantAdmin = new User();
+        tenantAdmin.setAuthority(Authority.TENANT_ADMIN);
+        tenantAdmin.setTenantId(savedTenant.getId());
+        tenantAdmin.setEmail("tenant2@thingsboard.org");
+        tenantAdmin.setFirstName("Joe");
+        tenantAdmin.setLastName("Downs");
+
+        tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1");
+    }
+
+    @After
+    public void afterTest() throws Exception {
+        loginSysAdmin();
+
+        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
+                .andExpect(status().isOk());
+    }
+
+    @Test
+    public void testCountEntitiesByQuery() throws Exception {
+        List<Device> devices = new ArrayList<>();
+        for (int i = 0; i < 97; i++) {
+            Device device = new Device();
+            device.setName("Device" + i);
+            device.setType("default");
+            device.setLabel("testLabel" + (int) (Math.random() * 1000));
+            devices.add(doPost("/api/device", device, Device.class));
+        }
+        DeviceTypeFilter filter = new DeviceTypeFilter();
+        filter.setDeviceType("default");
+        filter.setDeviceNameFilter("");
+
+        EntityCountQuery countQuery = new EntityCountQuery(filter);
+
+        Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
+        Assert.assertEquals(97, count.longValue());
+
+        filter.setDeviceType("unknown");
+        count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
+        Assert.assertEquals(0, count.longValue());
+
+        filter.setDeviceType("default");
+        filter.setDeviceNameFilter("Device1");
+
+        count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
+        Assert.assertEquals(11, count.longValue());
+
+        EntityListFilter entityListFilter = new EntityListFilter();
+        entityListFilter.setEntityType(EntityType.DEVICE);
+        entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList()));
+
+        countQuery = new EntityCountQuery(entityListFilter);
+
+        count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
+        Assert.assertEquals(97, count.longValue());
+    }
+
+    @Test
+    public void testSimpleFindEntityDataByQuery() throws Exception {
+        List<Device> devices = new ArrayList<>();
+        for (int i = 0; i < 97; i++) {
+            Device device = new Device();
+            device.setName("Device" + i);
+            device.setType("default");
+            device.setLabel("testLabel" + (int) (Math.random() * 1000));
+            devices.add(doPost("/api/device", device, Device.class));
+        }
+
+        DeviceTypeFilter filter = new DeviceTypeFilter();
+        filter.setDeviceType("default");
+        filter.setDeviceNameFilter("");
+
+        EntityDataSortOrder sortOrder = new EntityDataSortOrder(
+                new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC
+        );
+        EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
+        List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
+
+        EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
+
+        PageData<EntityData> data =
+                doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
+                });
+
+        Assert.assertEquals(97, data.getTotalElements());
+        Assert.assertEquals(10, data.getTotalPages());
+        Assert.assertTrue(data.hasNext());
+        Assert.assertEquals(10, data.getData().size());
+
+        List<EntityData> loadedEntities = new ArrayList<>(data.getData());
+        while (data.hasNext()) {
+            query = query.next();
+            data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
+            });
+            loadedEntities.addAll(data.getData());
+        }
+        Assert.assertEquals(97, loadedEntities.size());
+
+        List<EntityId> loadedIds = loadedEntities.stream().map(EntityData::getEntityId).collect(Collectors.toList());
+        List<EntityId> deviceIds = devices.stream().map(Device::getId).collect(Collectors.toList());
+
+        Assert.assertEquals(deviceIds, loadedIds);
+
+        List<String> loadedNames = loadedEntities.stream().map(entityData ->
+                entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).collect(Collectors.toList());
+        List<String> deviceNames = devices.stream().map(Device::getName).collect(Collectors.toList());
+
+        Assert.assertEquals(deviceNames, loadedNames);
+
+        sortOrder = new EntityDataSortOrder(
+                new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), EntityDataSortOrder.Direction.DESC
+        );
+
+        pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder);
+        query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
+        data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
+        });
+        Assert.assertEquals(11, data.getTotalElements());
+        Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue());
+
+    }
+
+    @Test
+    public void testFindEntityDataByQueryWithAttributes() throws Exception {
+
+        List<Device> devices = new ArrayList<>();
+        List<Long> temperatures = new ArrayList<>();
+        List<Long> highTemperatures = new ArrayList<>();
+        for (int i=0;i<67;i++) {
+            Device device = new Device();
+            String name = "Device"+i;
+            device.setName(name);
+            device.setType("default");
+            device.setLabel("testLabel"+(int)(Math.random()*1000));
+            devices.add(doPost("/api/device?accessToken="+name, device, Device.class));
+            long temperature = (long)(Math.random()*100);
+            temperatures.add(temperature);
+            if (temperature > 45) {
+                highTemperatures.add(temperature);
+            }
+        }
+        for (int i=0;i<devices.size();i++) {
+            Device device = devices.get(i);
+            String payload = "{\"temperature\":"+temperatures.get(i)+"}";
+            doPost("/api/plugins/telemetry/"+device.getId()+"/"+ DataConstants.SHARED_SCOPE, payload, String.class, status().isOk());
+        }
+        Thread.sleep(1000);
+
+        DeviceTypeFilter filter = new DeviceTypeFilter();
+        filter.setDeviceType("default");
+        filter.setDeviceNameFilter("");
+
+        EntityDataSortOrder sortOrder = new EntityDataSortOrder(
+                new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC
+        );
+        EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
+        List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
+        List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
+
+        EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null);
+        PageData<EntityData> data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
+        });
+
+        List<EntityData> loadedEntities = new ArrayList<>(data.getData());
+        while (data.hasNext()) {
+            query = query.next();
+            data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
+            });
+            loadedEntities.addAll(data.getData());
+        }
+        Assert.assertEquals(67, loadedEntities.size());
+
+        List<String> loadedTemperatures = loadedEntities.stream().map(entityData ->
+                entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
+        List<String> deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
+        Assert.assertEquals(deviceTemperatures, loadedTemperatures);
+
+        pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
+        KeyFilter highTemperatureFilter = new KeyFilter();
+        highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
+        NumericFilterPredicate predicate = new NumericFilterPredicate();
+        predicate.setValue(45);
+        predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
+        highTemperatureFilter.setPredicate(predicate);
+        List<KeyFilter> keyFilters = Collections.singletonList(highTemperatureFilter);
+
+        query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
+
+        data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
+        });
+        loadedEntities = new ArrayList<>(data.getData());
+        while (data.hasNext()) {
+            query = query.next();
+            data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
+            });
+            loadedEntities.addAll(data.getData());
+        }
+        Assert.assertEquals(highTemperatures.size(), loadedEntities.size());
+
+        List<String> loadedHighTemperatures = loadedEntities.stream().map(entityData ->
+                entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
+        List<String> deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
+
+        Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures);
+
+    }
+}

+/**
+ * Copyright © 2016-2020 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.controller.sql;
+
+import org.thingsboard.server.controller.BaseEntityQueryControllerTest;
+import org.thingsboard.server.dao.service.DaoSqlTest;
+
+@DaoSqlTest
+public class EntityQueryControllerSqlTest extends BaseEntityQueryControllerTest {
+}

@@ -20,7 +20,9 @@ import lombok.Getter;
 public class EntityCountQuery {
 public class EntityCountQuery {
 
 
     @Getter
     @Getter
-    private final EntityFilter entityFilter;
+    private EntityFilter entityFilter;
+
+    public EntityCountQuery() {}
 
 
     public EntityCountQuery(EntityFilter entityFilter) {
     public EntityCountQuery(EntityFilter entityFilter) {
         this.entityFilter = entityFilter;
         this.entityFilter = entityFilter;

@@ -16,15 +16,20 @@
 package org.thingsboard.server.common.data.query;
 package org.thingsboard.server.common.data.query;
 
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.Data;
 
 
 @Data
 @Data
+@AllArgsConstructor
 public class EntityDataPageLink {
 public class EntityDataPageLink {
 
 
-    private final int pageSize;
-    private final int page;
-    private final String textSearch;
-    private final EntityDataSortOrder sortOrder;
+    private int pageSize;
+    private int page;
+    private String textSearch;
+    private EntityDataSortOrder sortOrder;
+
+    public EntityDataPageLink() {
+    }
 
 
     @JsonIgnore
     @JsonIgnore
     public EntityDataPageLink nextPageLink() {
     public EntityDataPageLink nextPageLink() {

@@ -23,13 +23,21 @@ import java.util.List;
 public class EntityDataQuery extends EntityCountQuery {
 public class EntityDataQuery extends EntityCountQuery {
 
 
     @Getter
     @Getter
-    private final EntityDataPageLink pageLink;
+    private EntityDataPageLink pageLink;
     @Getter
     @Getter
-    private final List<EntityKey> entityFields;
+    private List<EntityKey> entityFields;
     @Getter
     @Getter
-    private final List<EntityKey> latestValues;
+    private List<EntityKey> latestValues;
     @Getter
     @Getter
-    private final List<KeyFilter> keyFilters;
+    private List<KeyFilter> keyFilters;
+
+    public EntityDataQuery() {
+        super();
+    }
+
+    public EntityDataQuery(EntityFilter entityFilter) {
+        super(entityFilter);
+    }
 
 
     public EntityDataQuery(EntityFilter entityFilter,
     public EntityDataQuery(EntityFilter entityFilter,
                            EntityDataPageLink pageLink,
                            EntityDataPageLink pageLink,

@@ -20,8 +20,10 @@ import lombok.Data;
 @Data
 @Data
 public class EntityDataSortOrder {
 public class EntityDataSortOrder {
 
 
-    private final EntityKey key;
-    private final Direction direction;
+    private EntityKey key;
+    private Direction direction;
+
+    public EntityDataSortOrder() {}
 
 
     public EntityDataSortOrder(EntityKey key) {
     public EntityDataSortOrder(EntityKey key) {
         this(key, Direction.ASC);
         this(key, Direction.ASC);

@@ -15,7 +15,27 @@
  */
  */
 package org.thingsboard.server.common.data.query;
 package org.thingsboard.server.common.data.query;
 
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+@JsonTypeInfo(
+        use = JsonTypeInfo.Id.NAME,
+        include = JsonTypeInfo.As.PROPERTY,
+        property = "type")
+@JsonSubTypes({
+        @JsonSubTypes.Type(value = SingleEntityFilter.class, name = "singleEntity"),
+        @JsonSubTypes.Type(value = EntityListFilter.class, name = "entityList"),
+        @JsonSubTypes.Type(value = EntityNameFilter.class, name = "entityName"),
+        @JsonSubTypes.Type(value = AssetTypeFilter.class, name = "assetType"),
+        @JsonSubTypes.Type(value = DeviceTypeFilter.class, name = "deviceType"),
+        @JsonSubTypes.Type(value = EntityViewTypeFilter.class, name = "entityViewType"),
+        @JsonSubTypes.Type(value = RelationsQueryFilter.class, name = "relationsQuery"),
+        @JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"),
+        @JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"),
+        @JsonSubTypes.Type(value = EntityViewSearchQueryFilter.class, name = "entityViewSearchQuery")})
 public interface EntityFilter {
 public interface EntityFilter {
 
 
+    @JsonIgnore
     EntityFilterType getType();
     EntityFilterType getType();
 }
 }

@@ -15,8 +15,22 @@
  */
  */
 package org.thingsboard.server.common.data.query;
 package org.thingsboard.server.common.data.query;
 
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+@JsonTypeInfo(
+        use = JsonTypeInfo.Id.NAME,
+        include = JsonTypeInfo.As.PROPERTY,
+        property = "type")
+@JsonSubTypes({
+        @JsonSubTypes.Type(value = StringFilterPredicate.class, name = "STRING"),
+        @JsonSubTypes.Type(value = NumericFilterPredicate.class, name = "NUMERIC"),
+        @JsonSubTypes.Type(value = BooleanFilterPredicate.class, name = "BOOLEAN"),
+        @JsonSubTypes.Type(value = ComplexFilterPredicate.class, name = "COMPLEX")})
 public interface KeyFilterPredicate {
 public interface KeyFilterPredicate {
 
 
+    @JsonIgnore
     FilterPredicateType getType();
     FilterPredicateType getType();
 
 
 }
 }

@@ -67,7 +67,7 @@ public class EntityDataAdapter {
                 } else {
                 } else {
                     strValue = convertValue(value);
                     strValue = convertValue(value);
                     Object tsObject = row[mapping.getIndex() + 1];
                     Object tsObject = row[mapping.getIndex() + 1];
-                    ts = Long.parseLong(tsObject.toString());
+                    ts = tsObject != null ? Long.parseLong(tsObject.toString()) : 0;
                 }
                 }
                 TsValue tsValue = new TsValue(ts, strValue);
                 TsValue tsValue = new TsValue(ts, strValue);
                 latest.computeIfAbsent(entityKey.getType(), entityKeyType -> new HashMap<>()).put(entityKey.getKey(), tsValue);
                 latest.computeIfAbsent(entityKey.getType(), entityKeyType -> new HashMap<>()).put(entityKey.getKey(), tsValue);

@@ -42,7 +42,6 @@ import org.thingsboard.server.common.data.query.EntityData;
 import org.thingsboard.server.common.data.query.EntityDataPageLink;
 import org.thingsboard.server.common.data.query.EntityDataPageLink;
 import org.thingsboard.server.common.data.query.EntityDataQuery;
 import org.thingsboard.server.common.data.query.EntityDataQuery;
 import org.thingsboard.server.common.data.query.EntityDataSortOrder;
 import org.thingsboard.server.common.data.query.EntityDataSortOrder;
-import org.thingsboard.server.common.data.query.EntityFilter;
 import org.thingsboard.server.common.data.query.EntityKey;
 import org.thingsboard.server.common.data.query.EntityKey;
 import org.thingsboard.server.common.data.query.EntityKeyType;
 import org.thingsboard.server.common.data.query.EntityKeyType;
 import org.thingsboard.server.common.data.query.EntityListFilter;
 import org.thingsboard.server.common.data.query.EntityListFilter;