Commit 8f2438d6ab57ca22b940a001e9a05e68225e6c69

Authored by Viacheslav Klimov
Committed by GitHub
1 parent f673b094

SNMP devices balancing (#4254)

* Fix merge errors

* Implement SNMP transports balancing

* Refactor; implement transport device cache

* Refactor

* Finish up device lifecycle handling implementing; refactor

* Refactor

* Change base image to thingsboard/openjdk11 for msa snmp transport

* Refactor

* Change transport services names to upper-case
Showing 67 changed files with 1831 additions and 461 deletions
... ... @@ -25,7 +25,6 @@ import org.springframework.stereotype.Service;
25 25 import org.thingsboard.common.util.ThingsBoardThreadFactory;
26 26 import org.thingsboard.server.actors.ActorSystemContext;
27 27 import org.thingsboard.server.actors.DefaultTbActorSystem;
28   -import org.thingsboard.server.actors.TbActorId;
29 28 import org.thingsboard.server.actors.TbActorRef;
30 29 import org.thingsboard.server.actors.TbActorSystem;
31 30 import org.thingsboard.server.actors.TbActorSystemSettings;
... ... @@ -33,14 +32,13 @@ import org.thingsboard.server.actors.app.AppActor;
33 32 import org.thingsboard.server.actors.app.AppInitMsg;
34 33 import org.thingsboard.server.actors.stats.StatsActor;
35 34 import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
36   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
37 35 import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
  36 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
38 37
39 38 import javax.annotation.PostConstruct;
40 39 import javax.annotation.PreDestroy;
41 40 import java.util.concurrent.ExecutorService;
42 41 import java.util.concurrent.Executors;
43   -import java.util.concurrent.ScheduledExecutorService;
44 42
45 43 @Service
46 44 @Slf4j
... ...
... ... @@ -52,7 +52,7 @@ import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
52 52 import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
53 53 import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto;
54 54 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
55   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  55 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
56 56 import org.thingsboard.server.queue.discovery.PartitionService;
57 57 import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
58 58 import org.thingsboard.server.queue.scheduler.SchedulerComponent;
... ...
... ... @@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.id.TenantProfileId;
22 22 import org.thingsboard.server.common.msg.queue.TbCallback;
23 23 import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
24 24 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
25   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  25 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
26 26
27 27 public interface TbApiUsageStateService extends ApplicationListener<PartitionChangeEvent> {
28 28
... ...
... ... @@ -52,7 +52,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM
52 52 import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
53 53 import org.thingsboard.server.queue.TbQueueConsumer;
54 54 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
55   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  55 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
56 56 import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
57 57 import org.thingsboard.server.queue.util.TbCoreComponent;
58 58 import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
... ...
... ... @@ -38,7 +38,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
38 38 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
39 39 import org.thingsboard.server.queue.TbQueueConsumer;
40 40 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
41   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  41 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
42 42 import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
43 43 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
44 44 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
... ...
... ... @@ -16,7 +16,7 @@
16 16 package org.thingsboard.server.service.queue;
17 17
18 18 import org.springframework.context.ApplicationListener;
19   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  19 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
20 20
21 21 public interface TbCoreConsumerService extends ApplicationListener<PartitionChangeEvent> {
22 22
... ...
... ... @@ -16,7 +16,7 @@
16 16 package org.thingsboard.server.service.queue;
17 17
18 18 import org.springframework.context.ApplicationListener;
19   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  19 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
20 20
21 21 public interface TbRuleEngineConsumerService extends ApplicationListener<PartitionChangeEvent> {
22 22
... ...
... ... @@ -34,7 +34,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
34 34 import org.thingsboard.server.common.msg.queue.TbCallback;
35 35 import org.thingsboard.server.queue.TbQueueConsumer;
36 36 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
37   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  37 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
38 38 import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
39 39 import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
40 40 import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
... ...
... ... @@ -54,7 +54,7 @@ import org.thingsboard.server.dao.tenant.TenantService;
54 54 import org.thingsboard.server.dao.timeseries.TimeseriesService;
55 55 import org.thingsboard.common.util.JacksonUtil;
56 56 import org.thingsboard.server.gen.transport.TransportProtos;
57   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  57 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
58 58 import org.thingsboard.server.queue.discovery.PartitionService;
59 59 import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
60 60 import org.thingsboard.server.queue.util.TbCoreComponent;
... ...
... ... @@ -18,7 +18,7 @@ package org.thingsboard.server.service.state;
18 18 import org.springframework.context.ApplicationListener;
19 19 import org.thingsboard.server.common.data.Device;
20 20 import org.thingsboard.server.common.data.id.DeviceId;
21   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  21 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
22 22 import org.thingsboard.server.gen.transport.TransportProtos;
23 23 import org.thingsboard.server.common.msg.queue.TbCallback;
24 24
... ...
... ... @@ -46,7 +46,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdate
46 46 import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateValueListProto;
47 47 import org.thingsboard.server.queue.TbQueueProducer;
48 48 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
49   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  49 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
50 50 import org.thingsboard.server.queue.discovery.PartitionService;
51 51 import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
52 52 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
... ...
... ... @@ -20,10 +20,9 @@ import org.springframework.beans.factory.annotation.Autowired;
20 20 import org.springframework.context.annotation.Lazy;
21 21 import org.springframework.context.event.EventListener;
22 22 import org.springframework.stereotype.Service;
23   -import org.thingsboard.common.util.ThingsBoardThreadFactory;
24 23 import org.thingsboard.server.gen.transport.TransportProtos;
25   -import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent;
26   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  24 +import org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent;
  25 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
27 26 import org.thingsboard.server.queue.discovery.PartitionService;
28 27 import org.thingsboard.server.common.msg.queue.ServiceType;
29 28 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
... ...
... ... @@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.id.TenantId;
22 22 import org.thingsboard.server.common.data.kv.AttributeKvEntry;
23 23 import org.thingsboard.server.common.data.kv.TsKvEntry;
24 24 import org.thingsboard.server.common.msg.queue.TbCallback;
25   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  25 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
26 26
27 27 import java.util.List;
28 28
... ...
... ... @@ -15,8 +15,8 @@
15 15 */
16 16 package org.thingsboard.server.service.subscription;
17 17
18   -import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent;
19   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  18 +import org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent;
  19 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
20 20 import org.thingsboard.server.common.msg.queue.TbCallback;
21 21 import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate;
22 22 import org.thingsboard.server.service.telemetry.sub.TelemetrySubscriptionUpdate;
... ...
... ... @@ -22,35 +22,18 @@ import lombok.extern.slf4j.Slf4j;
22 22 import org.springframework.beans.factory.annotation.Autowired;
23 23 import org.springframework.context.ApplicationListener;
24 24 import org.springframework.context.event.EventListener;
25   -import org.springframework.stereotype.Service;
26 25 import org.thingsboard.common.util.ThingsBoardThreadFactory;
27   -import org.thingsboard.server.common.data.id.EntityId;
28   -import org.thingsboard.server.common.data.id.TenantId;
29   -import org.thingsboard.server.common.data.kv.AttributeKvEntry;
30   -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
31   -import org.thingsboard.server.common.data.kv.BooleanDataEntry;
32   -import org.thingsboard.server.common.data.kv.DoubleDataEntry;
33   -import org.thingsboard.server.common.data.kv.LongDataEntry;
34   -import org.thingsboard.server.common.data.kv.StringDataEntry;
35   -import org.thingsboard.server.common.data.kv.TsKvEntry;
36 26 import org.thingsboard.server.common.msg.queue.ServiceType;
37   -import org.thingsboard.server.common.msg.queue.TbCallback;
38 27 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
39   -import org.thingsboard.server.dao.attributes.AttributesService;
40   -import org.thingsboard.server.dao.timeseries.TimeseriesService;
41   -import org.thingsboard.server.gen.transport.TransportProtos;
42   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  28 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
43 29 import org.thingsboard.server.queue.discovery.PartitionService;
44 30 import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
45 31 import org.thingsboard.server.service.queue.TbClusterService;
46 32 import org.thingsboard.server.service.subscription.SubscriptionManagerService;
47   -import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
48 33
49 34 import javax.annotation.Nullable;
50 35 import javax.annotation.PostConstruct;
51 36 import javax.annotation.PreDestroy;
52   -import java.util.Collections;
53   -import java.util.List;
54 37 import java.util.Optional;
55 38 import java.util.Set;
56 39 import java.util.concurrent.ConcurrentHashMap;
... ...
... ... @@ -17,8 +17,7 @@ package org.thingsboard.server.service.telemetry;
17 17
18 18 import org.springframework.context.ApplicationListener;
19 19 import org.thingsboard.rule.engine.api.RuleEngineAlarmService;
20   -import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
21   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  20 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
22 21
23 22 /**
24 23 * Created by ashvayka on 27.03.18.
... ...
... ... @@ -22,9 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture;
22 22 import lombok.extern.slf4j.Slf4j;
23 23 import org.checkerframework.checker.nullness.qual.Nullable;
24 24 import org.springframework.beans.factory.annotation.Autowired;
25   -import org.springframework.context.event.EventListener;
26 25 import org.springframework.stereotype.Service;
27   -import org.thingsboard.common.util.ThingsBoardThreadFactory;
28 26 import org.thingsboard.server.common.data.alarm.Alarm;
29 27 import org.thingsboard.server.common.data.alarm.AlarmInfo;
30 28 import org.thingsboard.server.common.data.alarm.AlarmQuery;
... ... @@ -35,43 +33,22 @@ import org.thingsboard.server.common.data.id.AlarmId;
35 33 import org.thingsboard.server.common.data.id.CustomerId;
36 34 import org.thingsboard.server.common.data.id.EntityId;
37 35 import org.thingsboard.server.common.data.id.TenantId;
38   -import org.thingsboard.server.common.data.kv.AttributeKvEntry;
39   -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
40   -import org.thingsboard.server.common.data.kv.BooleanDataEntry;
41   -import org.thingsboard.server.common.data.kv.DoubleDataEntry;
42   -import org.thingsboard.server.common.data.kv.LongDataEntry;
43   -import org.thingsboard.server.common.data.kv.StringDataEntry;
44   -import org.thingsboard.server.common.data.kv.TsKvEntry;
45 36 import org.thingsboard.server.common.data.page.PageData;
46 37 import org.thingsboard.server.common.data.query.AlarmData;
47   -import org.thingsboard.server.common.data.query.AlarmDataPageLink;
48 38 import org.thingsboard.server.common.data.query.AlarmDataQuery;
49 39 import org.thingsboard.server.common.msg.queue.ServiceType;
50 40 import org.thingsboard.server.common.msg.queue.TbCallback;
51 41 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
52 42 import org.thingsboard.server.dao.alarm.AlarmOperationResult;
53 43 import org.thingsboard.server.dao.alarm.AlarmService;
54   -import org.thingsboard.server.dao.attributes.AttributesService;
55   -import org.thingsboard.server.dao.timeseries.TimeseriesService;
56 44 import org.thingsboard.server.gen.transport.TransportProtos;
57   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
58 45 import org.thingsboard.server.queue.discovery.PartitionService;
59 46 import org.thingsboard.server.service.queue.TbClusterService;
60 47 import org.thingsboard.server.service.subscription.SubscriptionManagerService;
61 48 import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
62   -import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate;
63 49
64   -import javax.annotation.PostConstruct;
65   -import javax.annotation.PreDestroy;
66 50 import java.util.Collection;
67   -import java.util.Collections;
68   -import java.util.List;
69 51 import java.util.Optional;
70   -import java.util.Set;
71   -import java.util.concurrent.ConcurrentHashMap;
72   -import java.util.concurrent.ExecutorService;
73   -import java.util.concurrent.Executors;
74   -import java.util.function.Consumer;
75 52
76 53 /**
77 54 * Created by ashvayka on 27.03.18.
... ...
... ... @@ -16,8 +16,7 @@
16 16 package org.thingsboard.server.service.telemetry;
17 17
18 18 import org.springframework.context.ApplicationListener;
19   -import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
20   -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  19 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
21 20
22 21 /**
23 22 * Created by ashvayka on 27.03.18.
... ...
... ... @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.ApiUsageState;
30 30 import org.thingsboard.server.common.data.DataConstants;
31 31 import org.thingsboard.server.common.data.Device;
32 32 import org.thingsboard.server.common.data.DeviceProfile;
  33 +import org.thingsboard.server.common.data.DeviceTransportType;
33 34 import org.thingsboard.server.common.data.EntityType;
34 35 import org.thingsboard.server.common.data.TenantProfile;
35 36 import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials;
... ... @@ -61,11 +62,15 @@ import org.thingsboard.server.dao.resource.ResourceService;
61 62 import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
62 63 import org.thingsboard.server.gen.transport.TransportProtos;
63 64 import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto;
  65 +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceCredentialsRequestMsg;
  66 +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceRequestMsg;
64 67 import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg;
65 68 import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg;
66 69 import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg;
67 70 import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg;
68 71 import org.thingsboard.server.gen.transport.TransportProtos.GetResourcesRequestMsg;
  72 +import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesRequestMsg;
  73 +import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesResponseMsg;
69 74 import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg;
70 75 import org.thingsboard.server.gen.transport.TransportProtos.ProvisionResponseStatus;
71 76 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
... ... @@ -84,6 +89,7 @@ import org.thingsboard.server.service.state.DeviceStateService;
84 89
85 90 import java.util.Collections;
86 91 import java.util.List;
  92 +import java.util.Optional;
87 93 import java.util.UUID;
88 94 import java.util.concurrent.ConcurrentHashMap;
89 95 import java.util.concurrent.ConcurrentMap;
... ... @@ -139,40 +145,41 @@ public class DefaultTransportApiService implements TransportApiService {
139 145 @Override
140 146 public ListenableFuture<TbProtoQueueMsg<TransportApiResponseMsg>> handle(TbProtoQueueMsg<TransportApiRequestMsg> tbProtoQueueMsg) {
141 147 TransportApiRequestMsg transportApiRequestMsg = tbProtoQueueMsg.getValue();
  148 + ListenableFuture<TransportApiResponseMsg> result = null;
  149 +
142 150 if (transportApiRequestMsg.hasValidateTokenRequestMsg()) {
143 151 ValidateDeviceTokenRequestMsg msg = transportApiRequestMsg.getValidateTokenRequestMsg();
144   - return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN),
145   - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
  152 + result = validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN);
146 153 } else if (transportApiRequestMsg.hasValidateBasicMqttCredRequestMsg()) {
147 154 TransportProtos.ValidateBasicMqttCredRequestMsg msg = transportApiRequestMsg.getValidateBasicMqttCredRequestMsg();
148   - return Futures.transform(validateCredentials(msg),
149   - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
  155 + result = validateCredentials(msg);
150 156 } else if (transportApiRequestMsg.hasValidateX509CertRequestMsg()) {
151 157 ValidateDeviceX509CertRequestMsg msg = transportApiRequestMsg.getValidateX509CertRequestMsg();
152   - return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE),
153   - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
  158 + result = validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE);
154 159 } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) {
155   - return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()),
156   - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
  160 + result = handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg());
157 161 } else if (transportApiRequestMsg.hasEntityProfileRequestMsg()) {
158   - return Futures.transform(handle(transportApiRequestMsg.getEntityProfileRequestMsg()),
159   - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
  162 + result = handle(transportApiRequestMsg.getEntityProfileRequestMsg());
160 163 } else if (transportApiRequestMsg.hasLwM2MRequestMsg()) {
161   - return Futures.transform(handle(transportApiRequestMsg.getLwM2MRequestMsg()),
162   - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
  164 + result = handle(transportApiRequestMsg.getLwM2MRequestMsg());
163 165 } else if (transportApiRequestMsg.hasValidateDeviceLwM2MCredentialsRequestMsg()) {
164 166 ValidateDeviceLwM2MCredentialsRequestMsg msg = transportApiRequestMsg.getValidateDeviceLwM2MCredentialsRequestMsg();
165   - return Futures.transform(validateCredentials(msg.getCredentialsId(), DeviceCredentialsType.LWM2M_CREDENTIALS),
166   - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
  167 + result = validateCredentials(msg.getCredentialsId(), DeviceCredentialsType.LWM2M_CREDENTIALS);
167 168 } else if (transportApiRequestMsg.hasProvisionDeviceRequestMsg()) {
168   - return Futures.transform(handle(transportApiRequestMsg.getProvisionDeviceRequestMsg()),
169   - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
  169 + result = handle(transportApiRequestMsg.getProvisionDeviceRequestMsg());
170 170 } else if (transportApiRequestMsg.hasResourcesRequestMsg()) {
171   - return Futures.transform(handle(transportApiRequestMsg.getResourcesRequestMsg()),
172   - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
  171 + result = handle(transportApiRequestMsg.getResourcesRequestMsg());
  172 + } else if (transportApiRequestMsg.hasSnmpDevicesRequestMsg()) {
  173 + result = handle(transportApiRequestMsg.getSnmpDevicesRequestMsg());
  174 + } else if (transportApiRequestMsg.hasDeviceRequestMsg()) {
  175 + result = handle(transportApiRequestMsg.getDeviceRequestMsg());
  176 + } else if (transportApiRequestMsg.hasDeviceCredentialsRequestMsg()) {
  177 + result = handle(transportApiRequestMsg.getDeviceCredentialsRequestMsg());
173 178 }
174   - return Futures.transform(getEmptyTransportApiResponseFuture(),
175   - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
  179 +
  180 + return Futures.transform(Optional.ofNullable(result).orElseGet(this::getEmptyTransportApiResponseFuture),
  181 + value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()),
  182 + MoreExecutors.directExecutor());
176 183 }
177 184
178 185 private ListenableFuture<TransportApiResponseMsg> validateCredentials(String credentialsId, DeviceCredentialsType credentialsType) {
... ... @@ -366,6 +373,39 @@ public class DefaultTransportApiService implements TransportApiService {
366 373 return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(builder).build());
367 374 }
368 375
  376 + private ListenableFuture<TransportApiResponseMsg> handle(GetDeviceRequestMsg requestMsg) {
  377 + DeviceId deviceId = new DeviceId(new UUID(requestMsg.getDeviceIdMSB(), requestMsg.getDeviceIdLSB()));
  378 + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId);
  379 +
  380 + TransportApiResponseMsg responseMsg;
  381 + if (device != null) {
  382 + UUID deviceProfileId = device.getDeviceProfileId().getId();
  383 + responseMsg = TransportApiResponseMsg.newBuilder()
  384 + .setDeviceResponseMsg(TransportProtos.GetDeviceResponseMsg.newBuilder()
  385 + .setDeviceProfileIdMSB(deviceProfileId.getMostSignificantBits())
  386 + .setDeviceProfileIdLSB(deviceProfileId.getLeastSignificantBits())
  387 + .setDeviceTransportConfiguration(ByteString.copyFrom(
  388 + dataDecodingEncodingService.encode(device.getDeviceData().getTransportConfiguration())
  389 + )))
  390 + .build();
  391 + } else {
  392 + responseMsg = TransportApiResponseMsg.getDefaultInstance();
  393 + }
  394 +
  395 + return Futures.immediateFuture(responseMsg);
  396 + }
  397 +
  398 + private ListenableFuture<TransportApiResponseMsg> handle(GetDeviceCredentialsRequestMsg requestMsg) {
  399 + DeviceId deviceId = new DeviceId(new UUID(requestMsg.getDeviceIdMSB(), requestMsg.getDeviceIdLSB()));
  400 + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(TenantId.SYS_TENANT_ID, deviceId);
  401 +
  402 + return Futures.immediateFuture(TransportApiResponseMsg.newBuilder()
  403 + .setDeviceCredentialsResponseMsg(TransportProtos.GetDeviceCredentialsResponseMsg.newBuilder()
  404 + .setDeviceCredentialsData(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceCredentials))))
  405 + .build());
  406 + }
  407 +
  408 +
369 409 private ListenableFuture<TransportApiResponseMsg> handle(GetResourcesRequestMsg requestMsg) {
370 410 TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB()));
371 411 TransportProtos.GetResourcesResponseMsg.Builder builder = TransportProtos.GetResourcesResponseMsg.newBuilder();
... ... @@ -400,6 +440,20 @@ public class DefaultTransportApiService implements TransportApiService {
400 440 .build();
401 441 }
402 442
  443 + // TODO: request snmp devices with pagination
  444 + private ListenableFuture<TransportApiResponseMsg> handle(GetSnmpDevicesRequestMsg requestMsg) {
  445 + List<UUID> result = deviceService.findDevicesIdsByDeviceProfileTransportType(DeviceTransportType.SNMP);
  446 + GetSnmpDevicesResponseMsg responseMsg = GetSnmpDevicesResponseMsg.newBuilder()
  447 + .addAllIds(result.stream()
  448 + .map(UUID::toString)
  449 + .collect(Collectors.toList()))
  450 + .build();
  451 +
  452 + return Futures.immediateFuture(TransportApiResponseMsg.newBuilder()
  453 + .setSnmpDevicesResponseMsg(responseMsg)
  454 + .build());
  455 + }
  456 +
403 457 private ListenableFuture<TransportApiResponseMsg> getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) {
404 458 return Futures.transform(deviceService.findDeviceByIdAsync(TenantId.SYS_TENANT_ID, deviceId), device -> {
405 459 if (device == null) {
... ...
... ... @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture;
19 19 import org.thingsboard.server.common.data.Device;
20 20 import org.thingsboard.server.common.data.DeviceInfo;
21 21 import org.thingsboard.server.common.data.DeviceProfile;
  22 +import org.thingsboard.server.common.data.DeviceTransportType;
22 23 import org.thingsboard.server.common.data.EntitySubtype;
23 24 import org.thingsboard.server.common.data.device.DeviceSearchQuery;
24 25 import org.thingsboard.server.common.data.id.CustomerId;
... ... @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
31 32 import org.thingsboard.server.dao.device.provision.ProvisionRequest;
32 33
33 34 import java.util.List;
  35 +import java.util.UUID;
34 36
35 37 public interface DeviceService {
36 38
... ... @@ -90,4 +92,5 @@ public interface DeviceService {
90 92
91 93 Device saveDevice(ProvisionRequest provisionRequest, DeviceProfile profile);
92 94
  95 + List<UUID> findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType);
93 96 }
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.data;
  17 +
  18 +public interface TbTransportService {
  19 + String getName();
  20 +}
... ...
... ... @@ -21,6 +21,8 @@ import com.fasterxml.jackson.annotation.JsonSubTypes;
21 21 import com.fasterxml.jackson.annotation.JsonTypeInfo;
22 22 import org.thingsboard.server.common.data.DeviceTransportType;
23 23
  24 +import java.io.Serializable;
  25 +
24 26 @JsonIgnoreProperties(ignoreUnknown = true)
25 27 @JsonTypeInfo(
26 28 use = JsonTypeInfo.Id.NAME,
... ... @@ -31,7 +33,7 @@ import org.thingsboard.server.common.data.DeviceTransportType;
31 33 @JsonSubTypes.Type(value = MqttDeviceTransportConfiguration.class, name = "MQTT"),
32 34 @JsonSubTypes.Type(value = Lwm2mDeviceTransportConfiguration.class, name = "LWM2M"),
33 35 @JsonSubTypes.Type(value = SnmpDeviceTransportConfiguration.class, name = "SNMP")})
34   -public interface DeviceTransportConfiguration {
  36 +public interface DeviceTransportConfiguration extends Serializable {
35 37
36 38 @JsonIgnore
37 39 DeviceTransportType getType();
... ...
... ... @@ -19,14 +19,14 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
19 19 import lombok.Data;
20 20 import org.thingsboard.server.common.data.DeviceTransportType;
21 21
  22 +import java.util.Collections;
22 23 import java.util.List;
23 24 import java.util.stream.Collectors;
24 25 import java.util.stream.Stream;
25 26
26 27 @Data
27 28 public class SnmpProfileTransportConfiguration implements DeviceProfileTransportConfiguration {
28   -
29   - private int poolPeriodMs;
  29 + private int pollPeriodMs;
30 30 private int timeoutMs;
31 31 private int retries;
32 32 private List<SnmpDeviceProfileKvMapping> attributes;
... ... @@ -39,6 +39,10 @@ public class SnmpProfileTransportConfiguration implements DeviceProfileTransport
39 39
40 40 @JsonIgnore
41 41 public List<SnmpDeviceProfileKvMapping> getKvMappings() {
42   - return Stream.concat(attributes.stream(), telemetry.stream()).collect(Collectors.toList());
  42 + if (attributes != null && telemetry != null) {
  43 + return Stream.concat(attributes.stream(), telemetry.stream()).collect(Collectors.toList());
  44 + } else {
  45 + return Collections.emptyList();
  46 + }
43 47 }
44 48 }
... ...
... ... @@ -19,8 +19,12 @@ import lombok.Getter;
19 19 import lombok.extern.slf4j.Slf4j;
20 20 import org.springframework.beans.factory.annotation.Autowired;
21 21 import org.springframework.beans.factory.annotation.Value;
  22 +import org.springframework.context.ApplicationContext;
  23 +import org.springframework.context.event.ContextRefreshedEvent;
  24 +import org.springframework.context.event.EventListener;
22 25 import org.springframework.stereotype.Component;
23 26 import org.springframework.util.StringUtils;
  27 +import org.thingsboard.server.common.data.TbTransportService;
24 28 import org.thingsboard.server.common.data.id.TenantId;
25 29 import org.thingsboard.server.common.msg.queue.ServiceType;
26 30 import org.thingsboard.server.gen.transport.TransportProtos;
... ... @@ -32,6 +36,7 @@ import javax.annotation.PostConstruct;
32 36 import java.net.InetAddress;
33 37 import java.net.UnknownHostException;
34 38 import java.util.Arrays;
  39 +import java.util.Collection;
35 40 import java.util.Collections;
36 41 import java.util.List;
37 42 import java.util.Optional;
... ... @@ -56,6 +61,8 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider {
56 61
57 62 @Autowired(required = false)
58 63 private TbQueueRuleEngineSettings ruleEngineSettings;
  64 + @Autowired
  65 + private ApplicationContext applicationContext;
59 66
60 67 private List<ServiceType> serviceTypes;
61 68 private ServiceInfo serviceInfo;
... ... @@ -102,6 +109,19 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider {
102 109 serviceInfo = builder.build();
103 110 }
104 111
  112 + @EventListener(ContextRefreshedEvent.class)
  113 + public void setTransports() {
  114 + serviceInfo = ServiceInfo.newBuilder(serviceInfo)
  115 + .addAllTransports(getTransportServices().stream()
  116 + .map(TbTransportService::getName)
  117 + .collect(Collectors.toSet()))
  118 + .build();
  119 + }
  120 +
  121 + private Collection<TbTransportService> getTransportServices() {
  122 + return applicationContext.getBeansOfType(TbTransportService.class).values();
  123 + }
  124 +
105 125 @Override
106 126 public ServiceInfo getServiceInfo() {
107 127 return serviceInfo;
... ...
... ... @@ -15,26 +15,27 @@
15 15 */
16 16 package org.thingsboard.server.queue.discovery;
17 17
18   -import com.google.common.hash.HashCode;
19 18 import com.google.common.hash.HashFunction;
  19 +import com.google.common.hash.Hasher;
20 20 import com.google.common.hash.Hashing;
21   -import lombok.Getter;
22 21 import lombok.extern.slf4j.Slf4j;
23 22 import org.springframework.beans.factory.annotation.Value;
24 23 import org.springframework.context.ApplicationEventPublisher;
25 24 import org.springframework.stereotype.Service;
26 25 import org.thingsboard.server.common.data.id.EntityId;
27 26 import org.thingsboard.server.common.data.id.TenantId;
28   -import org.thingsboard.server.common.msg.queue.ServiceQueueKey;
29 27 import org.thingsboard.server.common.msg.queue.ServiceQueue;
  28 +import org.thingsboard.server.common.msg.queue.ServiceQueueKey;
30 29 import org.thingsboard.server.common.msg.queue.ServiceType;
31 30 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
32 31 import org.thingsboard.server.gen.transport.TransportProtos;
33 32 import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo;
  33 +import org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent;
  34 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
  35 +import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent;
34 36 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
35 37
36 38 import javax.annotation.PostConstruct;
37   -import java.nio.charset.StandardCharsets;
38 39 import java.util.ArrayList;
39 40 import java.util.Collections;
40 41 import java.util.Comparator;
... ... @@ -46,7 +47,6 @@ import java.util.Set;
46 47 import java.util.UUID;
47 48 import java.util.concurrent.ConcurrentHashMap;
48 49 import java.util.concurrent.ConcurrentMap;
49   -import java.util.concurrent.ConcurrentNavigableMap;
50 50 import java.util.stream.Collectors;
51 51
52 52 @Service
... ... @@ -186,6 +186,8 @@ public class HashPartitionService implements PartitionService {
186 186 applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes));
187 187 }
188 188 }
  189 +
  190 + applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService));
189 191 }
190 192
191 193 @Override
... ... @@ -219,6 +221,14 @@ public class HashPartitionService implements PartitionService {
219 221 }
220 222 }
221 223
  224 + @Override
  225 + public int resolvePartitionIndex(UUID entityId, int partitions) {
  226 + int hash = hashFunction.newHasher()
  227 + .putLong(entityId.getMostSignificantBits())
  228 + .putLong(entityId.getLeastSignificantBits()).hash().asInt();
  229 + return Math.abs(hash % partitions);
  230 + }
  231 +
222 232 private Map<ServiceQueueKey, List<ServiceInfo>> getServiceKeyListMap(List<ServiceInfo> services) {
223 233 final Map<ServiceQueueKey, List<ServiceInfo>> currentMap = new HashMap<>();
224 234 services.forEach(serviceInfo -> {
... ...
... ... @@ -20,9 +20,11 @@ import org.thingsboard.server.common.data.id.TenantId;
20 20 import org.thingsboard.server.common.msg.queue.ServiceType;
21 21 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
22 22 import org.thingsboard.server.gen.transport.TransportProtos;
  23 +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
23 24
24 25 import java.util.List;
25 26 import java.util.Set;
  27 +import java.util.UUID;
26 28
27 29 /**
28 30 * Once application is ready or cluster topology changes, this Service will produce {@link PartitionChangeEvent}
... ... @@ -55,4 +57,6 @@ public interface PartitionService {
55 57 * @return
56 58 */
57 59 TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId);
  60 +
  61 + int resolvePartitionIndex(UUID entityId, int partitions);
58 62 }
... ...
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.discovery;
17 17
18 18 import lombok.extern.slf4j.Slf4j;
19 19 import org.springframework.context.ApplicationListener;
  20 +import org.thingsboard.server.queue.discovery.event.TbApplicationEvent;
20 21
21 22 import java.util.concurrent.locks.Lock;
22 23 import java.util.concurrent.locks.ReentrantLock;
... ...
... ... @@ -33,12 +33,14 @@ import org.apache.zookeeper.KeeperException;
33 33 import org.springframework.beans.factory.annotation.Value;
34 34 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
35 35 import org.springframework.boot.context.event.ApplicationReadyEvent;
  36 +import org.springframework.context.ApplicationEventPublisher;
36 37 import org.springframework.context.event.EventListener;
37 38 import org.springframework.core.annotation.Order;
38 39 import org.springframework.stereotype.Service;
39 40 import org.springframework.util.Assert;
40 41 import org.thingsboard.common.util.ThingsBoardThreadFactory;
41 42 import org.thingsboard.server.gen.transport.TransportProtos;
  43 +import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent;
42 44
43 45 import javax.annotation.PostConstruct;
44 46 import javax.annotation.PreDestroy;
... ... @@ -77,7 +79,8 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
77 79
78 80 private volatile boolean stopped = true;
79 81
80   - public ZkDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) {
  82 + public ZkDiscoveryService(TbServiceInfoProvider serviceInfoProvider,
  83 + PartitionService partitionService) {
81 84 this.serviceInfoProvider = serviceInfoProvider;
82 85 this.partitionService = partitionService;
83 86 }
... ... @@ -126,7 +129,8 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
126 129 return;
127 130 }
128 131 publishCurrentServer();
129   - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers());
  132 + TransportProtos.ServiceInfo currentService = serviceInfoProvider.getServiceInfo();
  133 + partitionService.recalculatePartitions(currentService, getOtherServers());
130 134 }
131 135
132 136 public synchronized void publishCurrentServer() {
... ... @@ -281,11 +285,11 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
281 285 case CHILD_ADDED:
282 286 case CHILD_UPDATED:
283 287 case CHILD_REMOVED:
284   - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers());
  288 + TransportProtos.ServiceInfo currentService = serviceInfoProvider.getServiceInfo();
  289 + partitionService.recalculatePartitions(currentService, getOtherServers());
285 290 break;
286 291 default:
287 292 break;
288 293 }
289 294 }
290   -
291 295 }
... ...
common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ClusterTopologyChangeEvent.java renamed from common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java
... ... @@ -13,10 +13,9 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.queue.discovery;
  16 +package org.thingsboard.server.queue.discovery.event;
17 17
18 18 import lombok.Getter;
19   -import org.springframework.context.ApplicationEvent;
20 19 import org.thingsboard.server.common.msg.queue.ServiceQueueKey;
21 20
22 21 import java.util.Set;
... ...
common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java renamed from common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java
... ... @@ -13,10 +13,9 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.queue.discovery;
  16 +package org.thingsboard.server.queue.discovery.event;
17 17
18 18 import lombok.Getter;
19   -import org.springframework.context.ApplicationEvent;
20 19 import org.thingsboard.server.common.msg.queue.ServiceQueueKey;
21 20 import org.thingsboard.server.common.msg.queue.ServiceType;
22 21 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.queue.discovery.event;
  17 +
  18 +import lombok.Getter;
  19 +import lombok.ToString;
  20 +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo;
  21 +
  22 +import java.util.List;
  23 +
  24 +@Getter
  25 +@ToString
  26 +public class ServiceListChangedEvent extends TbApplicationEvent {
  27 + private final List<ServiceInfo> otherServices;
  28 + private final ServiceInfo currentService;
  29 +
  30 + public ServiceListChangedEvent(List<ServiceInfo> otherServices, ServiceInfo currentService) {
  31 + super(otherServices);
  32 + this.otherServices = otherServices;
  33 + this.currentService = currentService;
  34 + }
  35 +}
... ...
common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/TbApplicationEvent.java renamed from common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEvent.java
... ... @@ -13,7 +13,7 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.queue.discovery;
  16 +package org.thingsboard.server.queue.discovery.event;
17 17
18 18 import lombok.Getter;
19 19 import org.springframework.context.ApplicationEvent;
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.queue.util;
  17 +
  18 +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  19 +
  20 +import java.lang.annotation.ElementType;
  21 +import java.lang.annotation.Retention;
  22 +import java.lang.annotation.RetentionPolicy;
  23 +import java.lang.annotation.Target;
  24 +
  25 +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.snmp.enabled}'=='true')")
  26 +@Retention(RetentionPolicy.RUNTIME)
  27 +@Target({ElementType.TYPE, ElementType.METHOD})
  28 +public @interface TbSnmpTransportComponent {
  29 +}
... ...
... ... @@ -34,6 +34,7 @@ message ServiceInfo {
34 34 int64 tenantIdMSB = 3;
35 35 int64 tenantIdLSB = 4;
36 36 repeated QueueInfo ruleEngineQueues = 5;
  37 + repeated string transports = 6;
37 38 }
38 39
39 40 /**
... ... @@ -250,6 +251,34 @@ message GetEntityProfileResponseMsg {
250 251 bytes apiState = 3;
251 252 }
252 253
  254 +message GetDeviceRequestMsg {
  255 + int64 deviceIdMSB = 1;
  256 + int64 deviceIdLSB = 2;
  257 +}
  258 +
  259 +message GetDeviceResponseMsg {
  260 + int64 deviceProfileIdMSB = 1;
  261 + int64 deviceProfileIdLSB = 2;
  262 + bytes deviceTransportConfiguration = 3;
  263 +}
  264 +
  265 +message GetDeviceCredentialsRequestMsg {
  266 + int64 deviceIdMSB = 1;
  267 + int64 deviceIdLSB = 2;
  268 +}
  269 +
  270 +message GetDeviceCredentialsResponseMsg {
  271 + bytes deviceCredentialsData = 1;
  272 +}
  273 +
  274 +message GetSnmpDevicesRequestMsg {
  275 +
  276 +}
  277 +
  278 +message GetSnmpDevicesResponseMsg {
  279 + repeated string ids = 1;
  280 +}
  281 +
253 282 message EntityUpdateMsg {
254 283 string entityType = 1;
255 284 bytes data = 2;
... ... @@ -559,6 +588,9 @@ message TransportApiRequestMsg {
559 588 ProvisionDeviceRequestMsg provisionDeviceRequestMsg = 7;
560 589 ValidateDeviceLwM2MCredentialsRequestMsg validateDeviceLwM2MCredentialsRequestMsg = 8;
561 590 GetResourcesRequestMsg resourcesRequestMsg = 9;
  591 + GetSnmpDevicesRequestMsg snmpDevicesRequestMsg = 10;
  592 + GetDeviceRequestMsg deviceRequestMsg = 11;
  593 + GetDeviceCredentialsRequestMsg deviceCredentialsRequestMsg = 12;
562 594 }
563 595
564 596 /* Response from ThingsBoard Core Service to Transport Service */
... ... @@ -567,8 +599,11 @@ message TransportApiResponseMsg {
567 599 GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2;
568 600 GetEntityProfileResponseMsg entityProfileResponseMsg = 3;
569 601 ProvisionDeviceResponseMsg provisionDeviceResponseMsg = 4;
  602 + GetSnmpDevicesResponseMsg snmpDevicesResponseMsg = 5;
570 603 LwM2MResponseMsg lwM2MResponseMsg = 6;
571 604 GetResourcesResponseMsg resourcesResponseMsg = 7;
  605 + GetDeviceResponseMsg deviceResponseMsg = 8;
  606 + GetDeviceCredentialsResponseMsg deviceCredentialsResponseMsg = 9;
572 607 }
573 608
574 609 /* Messages that are handled by ThingsBoard Core Service */
... ...
... ... @@ -18,12 +18,12 @@ package org.thingsboard.server.transport.coap;
18 18 import lombok.extern.slf4j.Slf4j;
19 19 import org.eclipse.californium.core.CoapResource;
20 20 import org.eclipse.californium.core.CoapServer;
21   -
22 21 import org.eclipse.californium.core.network.CoapEndpoint;
23 22 import org.eclipse.californium.core.network.CoapEndpoint.Builder;
24 23 import org.springframework.beans.factory.annotation.Autowired;
25 24 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
26 25 import org.springframework.stereotype.Service;
  26 +import org.thingsboard.server.common.data.TbTransportService;
27 27
28 28 import javax.annotation.PostConstruct;
29 29 import javax.annotation.PreDestroy;
... ... @@ -34,7 +34,7 @@ import java.net.UnknownHostException;
34 34 @Service("CoapTransportService")
35 35 @ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.coap.enabled}'=='true')")
36 36 @Slf4j
37   -public class CoapTransportService {
  37 +public class CoapTransportService implements TbTransportService {
38 38
39 39 private static final String V1 = "v1";
40 40 private static final String API = "api";
... ... @@ -73,4 +73,9 @@ public class CoapTransportService {
73 73 this.server.destroy();
74 74 log.info("CoAP transport stopped!");
75 75 }
  76 +
  77 + @Override
  78 + public String getName() {
  79 + return "COAP";
  80 + }
76 81 }
... ...
... ... @@ -31,6 +31,7 @@ import org.springframework.web.bind.annotation.RequestParam;
31 31 import org.springframework.web.bind.annotation.RestController;
32 32 import org.springframework.web.context.request.async.DeferredResult;
33 33 import org.thingsboard.server.common.data.DeviceTransportType;
  34 +import org.thingsboard.server.common.data.TbTransportService;
34 35 import org.thingsboard.server.common.data.id.DeviceId;
35 36 import org.thingsboard.server.common.transport.SessionMsgListener;
36 37 import org.thingsboard.server.common.transport.TransportContext;
... ... @@ -41,7 +42,6 @@ import org.thingsboard.server.common.transport.auth.SessionInfoCreator;
41 42 import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse;
42 43 import org.thingsboard.server.gen.transport.TransportProtos;
43 44 import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg;
44   -import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto;
45 45 import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg;
46 46 import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg;
47 47 import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg;
... ... @@ -53,7 +53,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMs
53 53 import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg;
54 54 import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg;
55 55 import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg;
56   -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg;
57 56 import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg;
58 57
59 58 import javax.servlet.http.HttpServletRequest;
... ... @@ -69,7 +68,7 @@ import java.util.function.Consumer;
69 68 @ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.http.enabled}'=='true')")
70 69 @RequestMapping("/api/v1")
71 70 @Slf4j
72   -public class DeviceApiController {
  71 +public class DeviceApiController implements TbTransportService {
73 72
74 73 @Autowired
75 74 private HttpTransportContext transportContext;
... ... @@ -338,4 +337,9 @@ public class DeviceApiController {
338 337 .build(), TransportServiceCallback.EMPTY);
339 338 }
340 339
  340 + @Override
  341 + public String getName() {
  342 + return "HTTP";
  343 + }
  344 +
341 345 }
... ...
... ... @@ -20,12 +20,13 @@ import org.eclipse.leshan.core.response.ReadResponse;
20 20 import org.eclipse.leshan.server.registration.Registration;
21 21 import org.thingsboard.server.common.data.Device;
22 22 import org.thingsboard.server.common.data.DeviceProfile;
  23 +import org.thingsboard.server.common.data.TbTransportService;
23 24 import org.thingsboard.server.gen.transport.TransportProtos;
24 25
25 26 import java.util.Collection;
26 27 import java.util.Optional;
27 28
28   -public interface LwM2mTransportService {
  29 +public interface LwM2mTransportService extends TbTransportService {
29 30
30 31 void onRegistered(Registration registration, Collection<Observation> previousObsersations);
31 32
... ...
... ... @@ -40,6 +40,7 @@ import org.springframework.stereotype.Service;
40 40 import org.thingsboard.common.util.JacksonUtil;
41 41 import org.thingsboard.server.common.data.Device;
42 42 import org.thingsboard.server.common.data.DeviceProfile;
  43 +import org.thingsboard.server.common.data.DeviceTransportType;
43 44 import org.thingsboard.server.common.transport.TransportService;
44 45 import org.thingsboard.server.common.transport.adaptor.AdaptorException;
45 46 import org.thingsboard.server.common.transport.adaptor.JsonConverter;
... ... @@ -1118,4 +1119,9 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService {
1118 1119 namesIsWritable.addAll(new HashSet<>(keyNamesIsWritable.values()));
1119 1120 return new ArrayList<>(namesIsWritable);
1120 1121 }
  1122 +
  1123 + @Override
  1124 + public String getName() {
  1125 + return "LWM2M";
  1126 + }
1121 1127 }
... ...
... ... @@ -28,6 +28,7 @@ import org.springframework.beans.factory.annotation.Value;
28 28 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
29 29 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
30 30 import org.springframework.stereotype.Service;
  31 +import org.thingsboard.server.common.data.TbTransportService;
31 32
32 33 import javax.annotation.PostConstruct;
33 34 import javax.annotation.PreDestroy;
... ... @@ -38,7 +39,7 @@ import javax.annotation.PreDestroy;
38 39 @Service("MqttTransportService")
39 40 @ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.mqtt.enabled}'=='true')")
40 41 @Slf4j
41   -public class MqttTransportService {
  42 +public class MqttTransportService implements TbTransportService {
42 43
43 44 @Value("${transport.mqtt.bind_address}")
44 45 private String host;
... ... @@ -90,4 +91,9 @@ public class MqttTransportService {
90 91 }
91 92 log.info("MQTT transport stopped!");
92 93 }
  94 +
  95 + @Override
  96 + public String getName() {
  97 + return "MQTT";
  98 + }
93 99 }
... ...
... ... @@ -58,9 +58,5 @@
58 58 <groupId>org.snmp4j</groupId>
59 59 <artifactId>snmp4j</artifactId>
60 60 </dependency>
61   - <dependency>
62   - <groupId>org.thingsboard.common</groupId>
63   - <artifactId>dao-api</artifactId>
64   - </dependency>
65 61 </dependencies>
66 62 </project>
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.transport.snmp;
  17 +
  18 +import org.snmp4j.CommunityTarget;
  19 +import org.snmp4j.PDU;
  20 +import org.snmp4j.Snmp;
  21 +import org.snmp4j.Target;
  22 +import org.snmp4j.event.ResponseEvent;
  23 +import org.snmp4j.mp.SnmpConstants;
  24 +import org.snmp4j.smi.GenericAddress;
  25 +import org.snmp4j.smi.OID;
  26 +import org.snmp4j.smi.OctetString;
  27 +import org.snmp4j.smi.VariableBinding;
  28 +import org.snmp4j.transport.DefaultUdpTransportMapping;
  29 +import org.snmp4j.transport.UdpTransportMapping;
  30 +
  31 +import java.io.IOException;
  32 +
  33 +/**
  34 + * For testing purposes. Will be removed when the time comes
  35 + */
  36 +public class SnmpDeviceSimulator {
  37 + private final Target target;
  38 + private final OID oid = new OID(".1.3.6.1.2.1.1.1.0");
  39 + private Snmp snmp;
  40 +
  41 + public SnmpDeviceSimulator(int port) {
  42 + String address = "udp:127.0.0.1/" + port;
  43 +
  44 + CommunityTarget target = new CommunityTarget();
  45 + target.setCommunity(new OctetString("public"));
  46 + target.setAddress(GenericAddress.parse(address));
  47 + target.setRetries(2);
  48 + target.setTimeout(1500);
  49 + target.setVersion(SnmpConstants.version2c);
  50 +
  51 + this.target = target;
  52 + }
  53 +
  54 + public static void main(String[] args) throws IOException {
  55 + SnmpDeviceSimulator deviceSimulator = new SnmpDeviceSimulator(161);
  56 +
  57 + deviceSimulator.start();
  58 + String response = deviceSimulator.sendRequest(PDU.GET);
  59 +
  60 + System.out.println(response);
  61 + }
  62 +
  63 + public void start() throws IOException {
  64 + UdpTransportMapping transport = new DefaultUdpTransportMapping();
  65 + transport.addTransportListener((sourceTransport, incomingAddress, wholeMessage, tmStateReference) -> {
  66 + System.out.println();
  67 + });
  68 + snmp = new Snmp(transport);
  69 +
  70 + transport.listen();
  71 + }
  72 +
  73 + public String sendRequest(int pduType) throws IOException {
  74 + PDU pdu = new PDU();
  75 + pdu.add(new VariableBinding(oid));
  76 + pdu.setType(pduType);
  77 +
  78 + ResponseEvent responseEvent = snmp.send(pdu, target);
  79 + return responseEvent.getResponse().get(0).getVariable().toString();
  80 + }
  81 +}
... ...
... ... @@ -15,17 +15,19 @@
15 15 */
16 16 package org.thingsboard.server.transport.snmp;
17 17
18   -import lombok.Getter;
  18 +import lombok.RequiredArgsConstructor;
19 19 import lombok.extern.slf4j.Slf4j;
20 20 import org.snmp4j.PDU;
21   -import org.snmp4j.Snmp;
22 21 import org.snmp4j.smi.OID;
23 22 import org.snmp4j.smi.VariableBinding;
24   -import org.springframework.beans.factory.annotation.Autowired;
25   -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
26   -import org.springframework.stereotype.Service;
  23 +import org.springframework.boot.context.event.ApplicationReadyEvent;
  24 +import org.springframework.context.event.EventListener;
  25 +import org.springframework.core.annotation.Order;
  26 +import org.springframework.stereotype.Component;
27 27 import org.thingsboard.server.common.data.Device;
28 28 import org.thingsboard.server.common.data.DeviceProfile;
  29 +import org.thingsboard.server.common.data.DeviceTransportType;
  30 +import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration;
29 31 import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration;
30 32 import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileKvMapping;
31 33 import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration;
... ... @@ -33,84 +35,180 @@ import org.thingsboard.server.common.data.id.DeviceId;
33 35 import org.thingsboard.server.common.data.id.DeviceProfileId;
34 36 import org.thingsboard.server.common.data.security.DeviceCredentials;
35 37 import org.thingsboard.server.common.data.security.DeviceCredentialsType;
  38 +import org.thingsboard.server.common.transport.DeviceUpdatedEvent;
36 39 import org.thingsboard.server.common.transport.TransportContext;
37   -import org.thingsboard.server.dao.device.DeviceCredentialsService;
38   -import org.thingsboard.server.transport.snmp.session.DeviceSessionCtx;
  40 +import org.thingsboard.server.common.transport.TransportDeviceProfileCache;
  41 +import org.thingsboard.server.common.transport.TransportService;
  42 +import org.thingsboard.server.common.transport.TransportServiceCallback;
  43 +import org.thingsboard.server.common.transport.auth.SessionInfoCreator;
  44 +import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse;
  45 +import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext;
  46 +import org.thingsboard.server.gen.transport.TransportProtos;
  47 +import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto;
  48 +import org.thingsboard.server.queue.util.TbSnmpTransportComponent;
  49 +import org.thingsboard.server.transport.snmp.service.ProtoTransportEntityService;
  50 +import org.thingsboard.server.transport.snmp.service.SnmpTransportBalancingService;
  51 +import org.thingsboard.server.transport.snmp.service.SnmpTransportService;
  52 +import org.thingsboard.server.transport.snmp.session.DeviceSessionContext;
39 53
40 54 import java.util.ArrayList;
  55 +import java.util.Collection;
41 56 import java.util.HashMap;
  57 +import java.util.LinkedList;
42 58 import java.util.List;
43 59 import java.util.Map;
44 60 import java.util.Optional;
  61 +import java.util.UUID;
45 62 import java.util.concurrent.ConcurrentHashMap;
  63 +import java.util.concurrent.ConcurrentLinkedDeque;
46 64 import java.util.concurrent.ExecutorService;
47 65 import java.util.stream.Collectors;
48 66
49   -@Service("SnmpTransportContext")
50   -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.snmp.enabled}'=='true')")
  67 +@TbSnmpTransportComponent
  68 +@Component
51 69 @Slf4j
  70 +@RequiredArgsConstructor
52 71 public class SnmpTransportContext extends TransportContext {
53   - @Autowired
54   - DeviceCredentialsService deviceCredentialsService;
55   -
56   - @Autowired
57   - SnmpTransportService snmpTransportService;
58   -
59   - @Getter
60   - private final Map<DeviceProfileId, SnmpProfileTransportConfiguration> profileTransportConfig = new ConcurrentHashMap<>();
61   - @Getter
62   - private final Map<DeviceProfileId, List<PDU>> pdusPerProfile = new ConcurrentHashMap<>();
63   - @Getter
64   - private final Map<DeviceId, DeviceSessionCtx> deviceSessions = new ConcurrentHashMap<>();
65   -
66   - public Optional<SnmpDeviceProfileKvMapping> findAttributesMapping(DeviceProfileId deviceProfileId, OID responseOid) {
67   - if (profileTransportConfig.containsKey(deviceProfileId)) {
68   - return findMapping(responseOid, profileTransportConfig.get(deviceProfileId).getAttributes());
69   - }
70   - return Optional.empty();
  72 + private final SnmpTransportService snmpTransportService;
  73 + private final TransportDeviceProfileCache deviceProfileCache;
  74 + private final TransportService transportService;
  75 + private final ProtoTransportEntityService protoEntityService;
  76 + private final SnmpTransportBalancingService balancingService;
  77 +
  78 + private final Map<DeviceId, DeviceSessionContext> sessions = new ConcurrentHashMap<>();
  79 + private final Map<DeviceProfileId, SnmpProfileTransportConfiguration> profilesTransportConfigs = new ConcurrentHashMap<>();
  80 + private final Map<DeviceProfileId, List<PDU>> profilesPdus = new ConcurrentHashMap<>();
  81 + private Collection<DeviceId> allSnmpDevicesIds = new ConcurrentLinkedDeque<>();
  82 +
  83 + @EventListener(ApplicationReadyEvent.class)
  84 + @Order(2)
  85 + public void initDevicesSessions() {
  86 + log.info("Initializing SNMP devices sessions");
  87 + allSnmpDevicesIds = protoEntityService.getAllSnmpDevicesIds().stream()
  88 + .map(DeviceId::new)
  89 + .collect(Collectors.toList());
  90 + log.trace("Found all SNMP devices ids: {}", allSnmpDevicesIds);
  91 +
  92 + List<DeviceId> managedDevicesIds = allSnmpDevicesIds.stream()
  93 + .filter(deviceId -> balancingService.isManagedByCurrentTransport(deviceId.getId()))
  94 + .collect(Collectors.toList());
  95 + log.info("SNMP devices managed by current SNMP transport: {}", managedDevicesIds);
  96 +
  97 + managedDevicesIds.stream()
  98 + .map(protoEntityService::getDeviceById)
  99 + .collect(Collectors.toList())
  100 + .forEach(this::establishDeviceSession);
71 101 }
72 102
73   - public Optional<SnmpDeviceProfileKvMapping> findTelemetryMapping(DeviceProfileId deviceProfileId, OID responseOid) {
74   - if (profileTransportConfig.containsKey(deviceProfileId)) {
75   - return findMapping(responseOid, profileTransportConfig.get(deviceProfileId).getTelemetry());
  103 + private void establishDeviceSession(Device device) {
  104 + if (device == null) return;
  105 + log.info("Establishing SNMP device session for device {}", device.getId());
  106 +
  107 + DeviceProfileId deviceProfileId = device.getDeviceProfileId();
  108 + DeviceProfile deviceProfile = deviceProfileCache.get(deviceProfileId);
  109 +
  110 + DeviceCredentials credentials = protoEntityService.getDeviceCredentialsByDeviceId(device.getId());
  111 + if (credentials.getCredentialsType() != DeviceCredentialsType.ACCESS_TOKEN) {
  112 + log.warn("[{}] Expected credentials type is {} but found {}", device.getId(), DeviceCredentialsType.ACCESS_TOKEN, credentials.getCredentialsType());
  113 + return;
76 114 }
77   - return Optional.empty();
78   - }
79 115
80   - private Optional<SnmpDeviceProfileKvMapping> findMapping(OID responseOid, List<SnmpDeviceProfileKvMapping> mappings) {
81   - return mappings.stream()
82   - .filter(kvMapping -> new OID(kvMapping.getOid()).equals(responseOid))
83   - //TODO: OID shouldn't be duplicated in the config, add backend and UI verification
84   - .findFirst();
85   - }
  116 + SnmpDeviceTransportConfiguration deviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration();
  117 + if (!deviceTransportConfiguration.isValid()) {
  118 + log.warn("SNMP device transport configuration is not valid");
  119 + return;
  120 + }
86 121
87   - public void initPduListPerProfile() {
88   - profileTransportConfig.forEach(this::updatePduListPerProfile);
89   - }
  122 + SnmpProfileTransportConfiguration profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration();
  123 + profilesPdus.computeIfAbsent(deviceProfileId, id -> createPdus(profileTransportConfiguration));
90 124
91   - public void updatePduListPerProfile(DeviceProfileId id, SnmpProfileTransportConfiguration config) {
92   - pdusPerProfile.put(id, createPduList(config));
  125 + DeviceSessionContext deviceSessionContext = new DeviceSessionContext(
  126 + device, deviceProfile,
  127 + credentials.getCredentialsId(), deviceTransportConfiguration,
  128 + this, snmpTransportService
  129 + );
  130 + registerSessionMsgListener(deviceSessionContext);
  131 + sessions.put(device.getId(), deviceSessionContext);
  132 + log.info("Established SNMP device session for device {}", device.getId());
93 133 }
94 134
95   - public void updateDeviceSessionCtx(Device device, DeviceProfile deviceProfile, Snmp snmp) {
96   - DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(device.getTenantId(), device.getId());
97   - if (DeviceCredentialsType.ACCESS_TOKEN.equals(credentials.getCredentialsType())) {
98   - SnmpDeviceTransportConfiguration snmpDeviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration();
99   - if (snmpDeviceTransportConfiguration.isValid()) {
100   - DeviceSessionCtx deviceSessionCtx = new DeviceSessionCtx(this, credentials.getCredentialsId(), snmpDeviceTransportConfiguration, snmp, device.getId(), deviceProfile);
101   - deviceSessionCtx.createSessionInfo(ctx -> getTransportService().registerAsyncSession(deviceSessionCtx.getSessionInfo(), deviceSessionCtx));
102   - this.deviceSessions.put(device.getId(), deviceSessionCtx);
103   - }
104   - } else {
  135 + private void updateDeviceSession(DeviceSessionContext sessionContext, Device device, DeviceProfile deviceProfile) {
  136 + log.info("Updating SNMP device session for device {}", device.getId());
  137 + DeviceProfileId deviceProfileId = deviceProfile.getId();
  138 +
  139 + DeviceCredentials credentials = protoEntityService.getDeviceCredentialsByDeviceId(device.getId());
  140 + if (credentials.getCredentialsType() != DeviceCredentialsType.ACCESS_TOKEN) {
105 141 log.warn("[{}] Expected credentials type is {} but found {}", device.getId(), DeviceCredentialsType.ACCESS_TOKEN, credentials.getCredentialsType());
  142 + destroyDeviceSession(sessionContext);
  143 + return;
  144 + }
  145 +
  146 + SnmpProfileTransportConfiguration profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration();
  147 + SnmpDeviceTransportConfiguration deviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration();
  148 + sessionContext.setProfileTransportConfiguration(profileTransportConfiguration);
  149 + sessionContext.setDeviceTransportConfiguration(deviceTransportConfiguration);
  150 + if (!deviceTransportConfiguration.isValid()) {
  151 + log.warn("SNMP device transport configuration is not valid");
  152 + destroyDeviceSession(sessionContext);
  153 + return;
  154 + }
  155 +
  156 + if (!profileTransportConfiguration.equals(profilesTransportConfigs.get(deviceProfileId))) {
  157 + profilesPdus.put(deviceProfileId, createPdus(profileTransportConfiguration));
  158 + profilesTransportConfigs.put(deviceProfileId, profileTransportConfiguration);
  159 + sessionContext.initTarget(profileTransportConfiguration, deviceTransportConfiguration);
  160 + } else if (!deviceTransportConfiguration.equals(sessionContext.getDeviceTransportConfiguration())) {
  161 + sessionContext.initTarget(profileTransportConfiguration, deviceTransportConfiguration);
  162 + } else {
  163 + log.trace("Configuration of the device {} was not updated", device);
106 164 }
107 165 }
108 166
109   - public ExecutorService getSnmpCallbackExecutor() {
110   - return snmpTransportService.getSnmpCallbackExecutor();
  167 + private void destroyDeviceSession(DeviceSessionContext sessionContext) {
  168 + if (sessionContext == null) return;
  169 + log.info("Destroying SNMP device session for device {}", sessionContext.getDevice().getId());
  170 + sessionContext.close();
  171 + transportService.deregisterSession(sessionContext.getSessionInfo());
  172 + sessions.remove(sessionContext.getDeviceId());
  173 + log.trace("Deregistered and removed session");
  174 +
  175 + DeviceProfileId deviceProfileId = sessionContext.getDeviceProfile().getId();
  176 + if (sessions.values().stream()
  177 + .map(DeviceAwareSessionContext::getDeviceProfile)
  178 + .noneMatch(deviceProfile -> deviceProfile.getId().equals(deviceProfileId))) {
  179 + log.trace("Removed values for device profile {} from configs and pdus caches", deviceProfileId);
  180 + profilesTransportConfigs.remove(deviceProfileId);
  181 + profilesPdus.remove(deviceProfileId);
  182 + }
111 183 }
112 184
113   - private List<PDU> createPduList(SnmpProfileTransportConfiguration deviceProfileConfig) {
  185 + private void registerSessionMsgListener(DeviceSessionContext deviceSessionContext) {
  186 + transportService.process(DeviceTransportType.SNMP,
  187 + TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(deviceSessionContext.getToken()).build(),
  188 + new TransportServiceCallback<ValidateDeviceCredentialsResponse>() {
  189 + @Override
  190 + public void onSuccess(ValidateDeviceCredentialsResponse msg) {
  191 + if (msg.hasDeviceInfo()) {
  192 + SessionInfoProto sessionInfo = SessionInfoCreator.create(
  193 + msg, SnmpTransportContext.this, UUID.randomUUID()
  194 + );
  195 +
  196 + transportService.registerAsyncSession(sessionInfo, deviceSessionContext);
  197 + deviceSessionContext.setSessionInfo(sessionInfo);
  198 + deviceSessionContext.setDeviceInfo(msg.getDeviceInfo());
  199 + } else {
  200 + log.warn("[{}] Failed to process device auth", deviceSessionContext.getDeviceId());
  201 + }
  202 + }
  203 +
  204 + @Override
  205 + public void onError(Throwable e) {
  206 + log.warn("[{}] Failed to process device auth: {}", deviceSessionContext.getDeviceId(), e);
  207 + }
  208 + });
  209 + }
  210 +
  211 + private List<PDU> createPdus(SnmpProfileTransportConfiguration deviceProfileConfig) {
114 212 Map<String, List<VariableBinding>> varBindingPerMethod = new HashMap<>();
115 213
116 214 deviceProfileConfig.getKvMappings().forEach(mapping -> varBindingPerMethod
... ... @@ -127,7 +225,106 @@ public class SnmpTransportContext extends TransportContext {
127 225 .collect(Collectors.toList());
128 226 }
129 227
130   - //TODO: Extract SNMP methods to enum
  228 + @EventListener(DeviceUpdatedEvent.class)
  229 + public void onDeviceUpdatedOrCreated(DeviceUpdatedEvent deviceUpdatedEvent) {
  230 + Device device = deviceUpdatedEvent.getDevice();
  231 + log.trace("Got creating or updating device event for device {}", device);
  232 + DeviceTransportType transportType = Optional.ofNullable(device.getDeviceData().getTransportConfiguration())
  233 + .map(DeviceTransportConfiguration::getType)
  234 + .orElse(null);
  235 + if (!allSnmpDevicesIds.contains(device.getId())) {
  236 + if (transportType != DeviceTransportType.SNMP) {
  237 + return;
  238 + }
  239 + allSnmpDevicesIds.add(device.getId());
  240 + if (balancingService.isManagedByCurrentTransport(device.getId().getId())) {
  241 + establishDeviceSession(device);
  242 + }
  243 + } else {
  244 + if (balancingService.isManagedByCurrentTransport(device.getId().getId())) {
  245 + DeviceSessionContext sessionContext = sessions.get(device.getId());
  246 + if (transportType == DeviceTransportType.SNMP) {
  247 + if (sessionContext != null) {
  248 + updateDeviceSession(sessionContext, device, deviceProfileCache.get(device.getDeviceProfileId()));
  249 + } else {
  250 + establishDeviceSession(device);
  251 + }
  252 + } else {
  253 + log.trace("Transport type was changed to {}", transportType);
  254 + destroyDeviceSession(sessionContext);
  255 + }
  256 + }
  257 + }
  258 + }
  259 +
  260 + public void onDeviceDeleted(DeviceSessionContext sessionContext) {
  261 + destroyDeviceSession(sessionContext);
  262 + }
  263 +
  264 + public void onDeviceProfileUpdated(DeviceProfile deviceProfile, DeviceSessionContext sessionContext) {
  265 + updateDeviceSession(sessionContext, sessionContext.getDevice(), deviceProfile);
  266 + }
  267 +
  268 + public void onSnmpTransportListChanged() {
  269 + log.trace("SNMP transport list changed. Updating sessions");
  270 + List<DeviceId> deleted = new LinkedList<>();
  271 + for (DeviceId deviceId : allSnmpDevicesIds) {
  272 + if (balancingService.isManagedByCurrentTransport(deviceId.getId())) {
  273 + if (!sessions.containsKey(deviceId)) {
  274 + Device device = protoEntityService.getDeviceById(deviceId);
  275 + if (device != null) {
  276 + log.info("SNMP device {} is now managed by current transport node", deviceId);
  277 + establishDeviceSession(device);
  278 + } else {
  279 + deleted.add(deviceId);
  280 + }
  281 + }
  282 + } else {
  283 + Optional.ofNullable(sessions.get(deviceId))
  284 + .ifPresent(sessionContext -> {
  285 + log.info("SNMP session for device {} is not managed by current transport node anymore", deviceId);
  286 + destroyDeviceSession(sessionContext);
  287 + });
  288 + }
  289 + }
  290 + log.trace("Removing deleted SNMP devices: {}", deleted);
  291 + allSnmpDevicesIds.removeAll(deleted);
  292 + }
  293 +
  294 +
  295 + public Collection<DeviceSessionContext> getSessions() {
  296 + return sessions.values();
  297 + }
  298 +
  299 + public Map<DeviceProfileId, List<PDU>> getProfilesPdus() {
  300 + return profilesPdus;
  301 + }
  302 +
  303 + public Optional<SnmpDeviceProfileKvMapping> getAttributesMapping(DeviceProfileId deviceProfileId, OID responseOid) {
  304 + if (profilesTransportConfigs.containsKey(deviceProfileId)) {
  305 + return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getAttributes());
  306 + }
  307 + return Optional.empty();
  308 + }
  309 +
  310 + public Optional<SnmpDeviceProfileKvMapping> getTelemetryMapping(DeviceProfileId deviceProfileId, OID responseOid) {
  311 + if (profilesTransportConfigs.containsKey(deviceProfileId)) {
  312 + return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getTelemetry());
  313 + }
  314 + return Optional.empty();
  315 + }
  316 +
  317 + private Optional<SnmpDeviceProfileKvMapping> getMapping(OID responseOid, List<SnmpDeviceProfileKvMapping> mappings) {
  318 + return mappings.stream()
  319 + .filter(kvMapping -> new OID(kvMapping.getOid()).equals(responseOid))
  320 + //TODO: OID shouldn't be duplicated in the config, add backend and UI verification
  321 + .findFirst();
  322 + }
  323 +
  324 + public ExecutorService getSnmpCallbackExecutor() {
  325 + return snmpTransportService.getSnmpCallbackExecutor();
  326 + }
  327 +
131 328 private int getSnmpMethod(String configMethod) {
132 329 switch (configMethod) {
133 330 case "get":
... ...
1   -/**
2   - * Copyright © 2016-2021 The Thingsboard Authors
3   - *
4   - * Licensed under the Apache License, Version 2.0 (the "License");
5   - * you may not use this file except in compliance with the License.
6   - * You may obtain a copy of the License at
7   - *
8   - * http://www.apache.org/licenses/LICENSE-2.0
9   - *
10   - * Unless required by applicable law or agreed to in writing, software
11   - * distributed under the License is distributed on an "AS IS" BASIS,
12   - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   - * See the License for the specific language governing permissions and
14   - * limitations under the License.
15   - */
16   -package org.thingsboard.server.transport.snmp;
17   -
18   -import lombok.Getter;
19   -import lombok.extern.slf4j.Slf4j;
20   -import org.snmp4j.Snmp;
21   -import org.snmp4j.transport.DefaultUdpTransportMapping;
22   -import org.springframework.beans.factory.annotation.Autowired;
23   -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
24   -import org.springframework.boot.context.event.ApplicationReadyEvent;
25   -import org.springframework.context.event.EventListener;
26   -import org.springframework.core.annotation.Order;
27   -import org.springframework.stereotype.Service;
28   -import org.thingsboard.common.util.ThingsBoardThreadFactory;
29   -import org.thingsboard.server.common.data.DeviceInfo;
30   -import org.thingsboard.server.common.data.DeviceProfile;
31   -import org.thingsboard.server.common.data.DeviceTransportType;
32   -import org.thingsboard.server.common.data.Tenant;
33   -import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration;
34   -import org.thingsboard.server.common.data.id.TenantId;
35   -import org.thingsboard.server.common.data.page.PageDataIterable;
36   -import org.thingsboard.server.dao.device.DeviceProfileService;
37   -import org.thingsboard.server.dao.device.DeviceService;
38   -import org.thingsboard.server.dao.tenant.TenantService;
39   -import org.thingsboard.server.transport.snmp.session.DeviceSessionCtx;
40   -
41   -import javax.annotation.PostConstruct;
42   -import javax.annotation.PreDestroy;
43   -import java.io.IOException;
44   -import java.util.concurrent.ExecutorService;
45   -import java.util.concurrent.Executors;
46   -import java.util.concurrent.ScheduledExecutorService;
47   -import java.util.concurrent.TimeUnit;
48   -
49   -@Service("SnmpTransportService")
50   -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.snmp.enabled}'=='true')")
51   -@Slf4j
52   -public class SnmpTransportService {
53   -
54   - private static final int ENTITY_PACK_LIMIT = 1024;
55   -
56   - @Autowired
57   - private SnmpTransportContext snmpTransportContext;
58   -
59   - @Autowired
60   - DeviceProfileService deviceProfileService;
61   -
62   - @Autowired
63   - TenantService tenantService;
64   -
65   - @Autowired
66   - DeviceService deviceService;
67   -
68   - @Getter
69   - private ExecutorService snmpCallbackExecutor;
70   - private Snmp snmp;
71   - private ScheduledExecutorService pollingExecutor;
72   -
73   - @PostConstruct
74   - public void init() {
75   - log.info("Starting SNMP transport...");
76   - pollingExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("snmp-polling"));
77   - //TODO: Set parallelism value in the config
78   - snmpCallbackExecutor = Executors.newWorkStealingPool(20);
79   - initializeSnmp();
80   - log.info("SNMP transport started!");
81   - }
82   -
83   - @PreDestroy
84   - public void shutdown() {
85   - log.info("Stopping SNMP transport!");
86   - if (pollingExecutor != null) {
87   - pollingExecutor.shutdownNow();
88   - }
89   - if (snmpCallbackExecutor != null) {
90   - snmpCallbackExecutor.shutdownNow();
91   - }
92   - if (snmp != null) {
93   - try {
94   - snmp.close();
95   - } catch (IOException e) {
96   - log.error(e.getMessage(), e);
97   - }
98   - }
99   - log.info("SNMP transport stopped!");
100   - }
101   -
102   - @EventListener(ApplicationReadyEvent.class)
103   - @Order(value = 2)
104   - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
105   - log.info("Received application ready event. Starting SNMP polling.");
106   - initSessionCtxList();
107   - startPolling();
108   - }
109   -
110   - private void initializeSnmp() {
111   - try {
112   - this.snmp = new Snmp(new DefaultUdpTransportMapping());
113   - this.snmp.listen();
114   - } catch (IOException e) {
115   - //TODO: what should be done if transport wasn't initialized?
116   - log.error(e.getMessage(), e);
117   - }
118   - }
119   -
120   - private void initSessionCtxList() {
121   - //TODO: This approach works for monolith, in cluster the same data will be fetched by each node.
122   - for (Tenant tenant : new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT)) {
123   - TenantId tenantId = tenant.getTenantId();
124   - for (DeviceProfile deviceProfile : new PageDataIterable<>(pageLink -> deviceProfileService.findDeviceProfiles(tenantId, pageLink), ENTITY_PACK_LIMIT)) {
125   - if (DeviceTransportType.SNMP.equals(deviceProfile.getTransportType())) {
126   - snmpTransportContext.getProfileTransportConfig().put(deviceProfile.getId(),
127   - (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration());
128   - initDeviceSessions(deviceProfile);
129   - }
130   - }
131   - }
132   - snmpTransportContext.initPduListPerProfile();
133   - }
134   -
135   - private void initDeviceSessions(DeviceProfile deviceProfile) {
136   - for (DeviceInfo deviceInfo : new PageDataIterable<>(pageLink -> deviceService.findDeviceInfosByTenantIdAndDeviceProfileId(deviceProfile.getTenantId(), deviceProfile.getId(), pageLink), ENTITY_PACK_LIMIT)) {
137   - snmpTransportContext.updateDeviceSessionCtx(deviceInfo, deviceProfile, snmp);
138   - }
139   - }
140   -
141   - private void startPolling() {
142   - //TODO: Get poll period from configuration;
143   - int poolPeriodSeconds = 1;
144   - pollingExecutor.scheduleAtFixedRate(() -> snmpTransportContext.getDeviceSessions().values().forEach(DeviceSessionCtx::executeSnmpRequest),
145   - 0, poolPeriodSeconds, TimeUnit.SECONDS);
146   - }
147   -}
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.transport.snmp.event;
  17 +
  18 +import lombok.RequiredArgsConstructor;
  19 +import org.springframework.stereotype.Component;
  20 +import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
  21 +import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent;
  22 +import org.thingsboard.server.queue.util.TbSnmpTransportComponent;
  23 +import org.thingsboard.server.transport.snmp.service.SnmpTransportBalancingService;
  24 +
  25 +@TbSnmpTransportComponent
  26 +@Component
  27 +@RequiredArgsConstructor
  28 +public class ServiceListChangedEventListener extends TbApplicationEventListener<ServiceListChangedEvent> {
  29 + private final SnmpTransportBalancingService snmpTransportBalancingService;
  30 +
  31 + @Override
  32 + protected void onTbApplicationEvent(ServiceListChangedEvent event) {
  33 + snmpTransportBalancingService.onServiceListChanged(event);
  34 + }
  35 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.transport.snmp.event;
  17 +
  18 +import org.thingsboard.server.queue.discovery.event.TbApplicationEvent;
  19 +
  20 +public class SnmpTransportListChangedEvent extends TbApplicationEvent {
  21 + public SnmpTransportListChangedEvent() {
  22 + super(new Object());
  23 + }
  24 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.transport.snmp.event;
  17 +
  18 +import lombok.RequiredArgsConstructor;
  19 +import org.springframework.stereotype.Component;
  20 +import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
  21 +import org.thingsboard.server.queue.util.TbSnmpTransportComponent;
  22 +import org.thingsboard.server.transport.snmp.SnmpTransportContext;
  23 +
  24 +@TbSnmpTransportComponent
  25 +@Component
  26 +@RequiredArgsConstructor
  27 +public class SnmpTransportListChangedEventListener extends TbApplicationEventListener<SnmpTransportListChangedEvent> {
  28 + private final SnmpTransportContext snmpTransportContext;
  29 +
  30 + @Override
  31 + protected void onTbApplicationEvent(SnmpTransportListChangedEvent event) {
  32 + snmpTransportContext.onSnmpTransportListChanged();
  33 + }
  34 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.transport.snmp.service;
  17 +
  18 +import lombok.RequiredArgsConstructor;
  19 +import org.springframework.stereotype.Service;
  20 +import org.thingsboard.server.common.data.Device;
  21 +import org.thingsboard.server.common.data.device.data.DeviceData;
  22 +import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration;
  23 +import org.thingsboard.server.common.data.id.DeviceId;
  24 +import org.thingsboard.server.common.data.id.DeviceProfileId;
  25 +import org.thingsboard.server.common.data.security.DeviceCredentials;
  26 +import org.thingsboard.server.common.transport.TransportService;
  27 +import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
  28 +import org.thingsboard.server.gen.transport.TransportProtos;
  29 +import org.thingsboard.server.queue.util.TbSnmpTransportComponent;
  30 +
  31 +import java.util.List;
  32 +import java.util.UUID;
  33 +import java.util.stream.Collectors;
  34 +
  35 +@TbSnmpTransportComponent
  36 +@Service
  37 +@RequiredArgsConstructor
  38 +public class ProtoTransportEntityService {
  39 + private final TransportService transportService;
  40 + private final DataDecodingEncodingService dataDecodingEncodingService;
  41 +
  42 + public Device getDeviceById(DeviceId id) {
  43 + TransportProtos.GetDeviceResponseMsg deviceProto = transportService.getDevice(TransportProtos.GetDeviceRequestMsg.newBuilder()
  44 + .setDeviceIdMSB(id.getId().getMostSignificantBits())
  45 + .setDeviceIdLSB(id.getId().getLeastSignificantBits())
  46 + .build());
  47 +
  48 + if (deviceProto == null) {
  49 + return null;
  50 + }
  51 +
  52 + DeviceProfileId deviceProfileId = new DeviceProfileId(new UUID(
  53 + deviceProto.getDeviceProfileIdMSB(), deviceProto.getDeviceProfileIdLSB())
  54 + );
  55 +
  56 + Device device = new Device();
  57 + device.setId(id);
  58 + device.setDeviceProfileId(deviceProfileId);
  59 +
  60 + DeviceTransportConfiguration deviceTransportConfiguration = (DeviceTransportConfiguration) dataDecodingEncodingService.decode(
  61 + deviceProto.getDeviceTransportConfiguration().toByteArray()
  62 + ).orElseThrow(() -> new IllegalStateException("Can't find device transport configuration"));
  63 +
  64 + DeviceData deviceData = new DeviceData();
  65 + deviceData.setTransportConfiguration(deviceTransportConfiguration);
  66 + device.setDeviceData(deviceData);
  67 +
  68 + return device;
  69 + }
  70 +
  71 + public DeviceCredentials getDeviceCredentialsByDeviceId(DeviceId deviceId) {
  72 + TransportProtos.GetDeviceCredentialsResponseMsg deviceCredentialsResponse = transportService.getDeviceCredentials(
  73 + TransportProtos.GetDeviceCredentialsRequestMsg.newBuilder()
  74 + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
  75 + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
  76 + .build()
  77 + );
  78 +
  79 + return (DeviceCredentials) dataDecodingEncodingService.decode(deviceCredentialsResponse.getDeviceCredentialsData().toByteArray())
  80 + .orElseThrow(() -> new IllegalArgumentException("Device credentials not found"));
  81 + }
  82 +
  83 + public List<UUID> getAllSnmpDevicesIds() {
  84 + TransportProtos.GetSnmpDevicesResponseMsg devicesIdsResponse = transportService.getSnmpDevicesIds(
  85 + TransportProtos.GetSnmpDevicesRequestMsg.getDefaultInstance()
  86 + );
  87 +
  88 + return devicesIdsResponse.getIdsList().stream()
  89 + .map(UUID::fromString)
  90 + .collect(Collectors.toList());
  91 + }
  92 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.transport.snmp.service;
  17 +
  18 +import lombok.RequiredArgsConstructor;
  19 +import lombok.extern.slf4j.Slf4j;
  20 +import org.springframework.context.ApplicationEventPublisher;
  21 +import org.springframework.stereotype.Service;
  22 +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo;
  23 +import org.thingsboard.server.queue.discovery.PartitionService;
  24 +import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent;
  25 +import org.thingsboard.server.queue.util.TbSnmpTransportComponent;
  26 +import org.thingsboard.server.transport.snmp.event.SnmpTransportListChangedEvent;
  27 +
  28 +import java.util.Comparator;
  29 +import java.util.List;
  30 +import java.util.UUID;
  31 +import java.util.stream.Collectors;
  32 +import java.util.stream.Stream;
  33 +
  34 +@TbSnmpTransportComponent
  35 +@Service
  36 +@RequiredArgsConstructor
  37 +@Slf4j
  38 +public class SnmpTransportBalancingService {
  39 + private final PartitionService partitionService;
  40 + private final ApplicationEventPublisher eventPublisher;
  41 + private final SnmpTransportService snmpTransportService;
  42 +
  43 + private int snmpTransportsCount = 1;
  44 + private Integer currentTransportPartitionIndex = 0;
  45 +
  46 + public void onServiceListChanged(ServiceListChangedEvent event) {
  47 + log.trace("Got service list changed event: {}", event);
  48 + recalculatePartitions(event.getOtherServices(), event.getCurrentService());
  49 + }
  50 +
  51 + public boolean isManagedByCurrentTransport(UUID entityId) {
  52 + boolean isManaged = resolvePartitionIndexForEntity(entityId) == currentTransportPartitionIndex;
  53 + if (!isManaged) {
  54 + log.trace("Entity {} is not managed by current SNMP transport node", entityId);
  55 + }
  56 + return isManaged;
  57 + }
  58 +
  59 + private int resolvePartitionIndexForEntity(UUID entityId) {
  60 + return partitionService.resolvePartitionIndex(entityId, snmpTransportsCount);
  61 + }
  62 +
  63 + private void recalculatePartitions(List<ServiceInfo> otherServices, ServiceInfo currentService) {
  64 + log.info("Recalculating partitions for SNMP transports");
  65 + List<ServiceInfo> snmpTransports = Stream.concat(otherServices.stream(), Stream.of(currentService))
  66 + .filter(service -> service.getTransportsList().contains(snmpTransportService.getName()))
  67 + .sorted(Comparator.comparing(ServiceInfo::getServiceId))
  68 + .collect(Collectors.toList());
  69 + log.trace("Found SNMP transports: {}", snmpTransports);
  70 +
  71 + int previousCurrentTransportPartitionIndex = currentTransportPartitionIndex;
  72 + int previousSnmpTransportsCount = snmpTransportsCount;
  73 +
  74 + if (!snmpTransports.isEmpty()) {
  75 + for (int i = 0; i < snmpTransports.size(); i++) {
  76 + if (snmpTransports.get(i).equals(currentService)) {
  77 + currentTransportPartitionIndex = i;
  78 + break;
  79 + }
  80 + }
  81 + snmpTransportsCount = snmpTransports.size();
  82 + }
  83 +
  84 + if (snmpTransportsCount != previousSnmpTransportsCount || currentTransportPartitionIndex != previousCurrentTransportPartitionIndex) {
  85 + log.info("SNMP transports partitions have changed: transports count = {}, current transport partition index = {}", snmpTransportsCount, currentTransportPartitionIndex);
  86 + eventPublisher.publishEvent(new SnmpTransportListChangedEvent());
  87 + } else {
  88 + log.info("SNMP transports partitions have not changed");
  89 + }
  90 + }
  91 +
  92 +}
... ...
common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java renamed from common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/SnmpSessionListener.java
... ... @@ -13,7 +13,7 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.transport.snmp.session;
  16 +package org.thingsboard.server.transport.snmp.service;
17 17
18 18 import com.google.gson.JsonElement;
19 19 import com.google.gson.JsonObject;
... ... @@ -25,9 +25,16 @@ import lombok.extern.slf4j.Slf4j;
25 25 import org.snmp4j.PDU;
26 26 import org.snmp4j.Snmp;
27 27 import org.snmp4j.event.ResponseEvent;
28   -import org.snmp4j.event.ResponseListener;
29 28 import org.snmp4j.smi.VariableBinding;
  29 +import org.snmp4j.transport.DefaultUdpTransportMapping;
  30 +import org.springframework.boot.context.event.ApplicationReadyEvent;
  31 +import org.springframework.context.annotation.Lazy;
  32 +import org.springframework.context.event.EventListener;
  33 +import org.springframework.core.annotation.Order;
  34 +import org.springframework.stereotype.Service;
  35 +import org.thingsboard.common.util.ThingsBoardThreadFactory;
30 36 import org.thingsboard.server.common.data.DeviceTransportType;
  37 +import org.thingsboard.server.common.data.TbTransportService;
31 38 import org.thingsboard.server.common.data.id.DeviceProfileId;
32 39 import org.thingsboard.server.common.data.kv.DataType;
33 40 import org.thingsboard.server.common.transport.TransportContext;
... ... @@ -39,27 +46,116 @@ import org.thingsboard.server.common.transport.auth.SessionInfoCreator;
39 46 import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse;
40 47 import org.thingsboard.server.gen.transport.TransportProtos;
41 48 import org.thingsboard.server.transport.snmp.SnmpTransportContext;
  49 +import org.thingsboard.server.queue.util.TbSnmpTransportComponent;
  50 +import org.thingsboard.server.transport.snmp.session.DeviceSessionContext;
42 51
  52 +import javax.annotation.PreDestroy;
  53 +import java.io.IOException;
43 54 import java.util.UUID;
  55 +import java.util.concurrent.ExecutorService;
  56 +import java.util.concurrent.Executors;
  57 +import java.util.concurrent.ScheduledExecutorService;
  58 +import java.util.concurrent.TimeUnit;
44 59 import java.util.function.Consumer;
45 60
  61 +@TbSnmpTransportComponent
  62 +@Service
46 63 @Slf4j
47   -@AllArgsConstructor
48   -public class SnmpSessionListener implements ResponseListener {
49   -
50   - @Getter
  64 +public class SnmpTransportService implements TbTransportService {
51 65 private final SnmpTransportContext snmpTransportContext;
52 66
53 67 @Getter
54   - private final String token;
  68 + private ExecutorService snmpCallbackExecutor;
  69 + @Getter
  70 + private Snmp snmp;
  71 + private ScheduledExecutorService pollingExecutor;
55 72
56   - @Override
57   - public void onResponse(ResponseEvent event) {
58   - ((Snmp) event.getSource()).cancel(event.getRequest(), this);
59   - snmpTransportContext.getSnmpCallbackExecutor().submit(() -> processSnmpResponse(event));
  73 + public SnmpTransportService(@Lazy SnmpTransportContext snmpTransportContext) {
  74 + this.snmpTransportContext = snmpTransportContext;
60 75 }
61 76
62   - private void processSnmpResponse(ResponseEvent event) {
  77 + // @PostConstruct
  78 + private void init() {
  79 + log.info("Starting SNMP transport...");
  80 + pollingExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("snmp-polling"));
  81 + //TODO: Set parallelism value in the config
  82 + snmpCallbackExecutor = Executors.newWorkStealingPool(20);
  83 +
  84 + initializeSnmp();
  85 +
  86 + log.info("SNMP transport started!");
  87 + }
  88 +
  89 + @PreDestroy
  90 + public void shutdown() {
  91 + log.info("Stopping SNMP transport!");
  92 + if (pollingExecutor != null) {
  93 + pollingExecutor.shutdownNow();
  94 + }
  95 + if (snmpCallbackExecutor != null) {
  96 + snmpCallbackExecutor.shutdownNow();
  97 + }
  98 + if (snmp != null) {
  99 + try {
  100 + snmp.close();
  101 + } catch (IOException e) {
  102 + log.error(e.getMessage(), e);
  103 + }
  104 + }
  105 + log.info("SNMP transport stopped!");
  106 + }
  107 +
  108 + @EventListener(ApplicationReadyEvent.class)
  109 + @Order(value = 10)
  110 + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
  111 + log.info("Received application ready event. Starting SNMP polling.");
  112 +// startPolling();
  113 + }
  114 +
  115 + private void initializeSnmp() {
  116 + try {
  117 + this.snmp = new Snmp(new DefaultUdpTransportMapping());
  118 + this.snmp.listen();
  119 + } catch (IOException e) {
  120 + //TODO: what should be done if transport wasn't initialized?
  121 + log.error(e.getMessage(), e);
  122 + }
  123 + }
  124 +
  125 + private void startPolling() {
  126 + //TODO: Get poll period from configuration;
  127 + int pollPeriodSeconds = 1;
  128 +
  129 + pollingExecutor.scheduleWithFixedDelay(() -> {
  130 + snmpTransportContext.getSessions().forEach(this::executeSnmpRequest);
  131 + }, 0, pollPeriodSeconds, TimeUnit.SECONDS);
  132 + }
  133 +
  134 + private void executeSnmpRequest(DeviceSessionContext sessionContext) {
  135 + long timeNow = System.currentTimeMillis();
  136 + long nextRequestExecutionTime = sessionContext.getPreviousRequestExecutedAt() + sessionContext.getProfileTransportConfiguration().getPollPeriodMs();
  137 +
  138 + if (nextRequestExecutionTime < timeNow) {
  139 + sessionContext.setPreviousRequestExecutedAt(timeNow);
  140 +
  141 + DeviceProfileId deviceProfileId = sessionContext.getDeviceProfile().getId();
  142 + snmpTransportContext.getProfilesPdus().get(deviceProfileId).forEach(pdu -> {
  143 + try {
  144 + log.debug("[{}] Sending SNMP message...", pdu.getRequestID());
  145 + snmp.send(pdu, sessionContext.getTarget(), deviceProfileId, sessionContext);
  146 + } catch (IOException e) {
  147 + log.error(e.getMessage(), e);
  148 + }
  149 + });
  150 + }
  151 + }
  152 +
  153 + public void onNewDeviceResponse(ResponseEvent responseEvent, DeviceSessionContext sessionContext) {
  154 + ((Snmp) responseEvent.getSource()).cancel(responseEvent.getRequest(), sessionContext);
  155 + snmpTransportContext.getSnmpCallbackExecutor().submit(() -> processSnmpResponse(responseEvent, sessionContext));
  156 + }
  157 +
  158 + private void processSnmpResponse(ResponseEvent event, DeviceSessionContext sessionContext) {
63 159 PDU response = event.getResponse();
64 160 if (event.getError() != null) {
65 161 log.warn("Response error: {}", event.getError().getMessage(), event.getError());
... ... @@ -72,8 +168,8 @@ public class SnmpSessionListener implements ResponseListener {
72 168 TransportService transportService = snmpTransportContext.getTransportService();
73 169 for (int i = 0; i < response.size(); i++) {
74 170 VariableBinding vb = response.get(i);
75   - snmpTransportContext.findAttributesMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT,
76   - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(token).build(),
  171 + snmpTransportContext.getAttributesMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT,
  172 + TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(sessionContext.getToken()).build(),
77 173 new DeviceAuthCallback(snmpTransportContext, sessionInfo -> {
78 174 try {
79 175 transportService.process(sessionInfo,
... ... @@ -84,8 +180,8 @@ public class SnmpSessionListener implements ResponseListener {
84 180 log.warn("Failed to process SNMP response: {}", e.getMessage(), e);
85 181 }
86 182 })));
87   - snmpTransportContext.findTelemetryMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT,
88   - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(token).build(),
  183 + snmpTransportContext.getTelemetryMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT,
  184 + TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(sessionContext.getToken()).build(),
89 185 new DeviceAuthCallback(snmpTransportContext, sessionInfo -> {
90 186 try {
91 187 transportService.process(sessionInfo,
... ... @@ -103,6 +199,14 @@ public class SnmpSessionListener implements ResponseListener {
103 199 }
104 200 }
105 201
  202 + private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) {
  203 + snmpTransportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder()
  204 + .setAttributeSubscription(false)
  205 + .setRpcSubscription(false)
  206 + .setLastActivityTime(System.currentTimeMillis())
  207 + .build(), TransportServiceCallback.EMPTY);
  208 + }
  209 +
106 210 private TransportProtos.PostAttributeMsg convertToPostAttributes(String keyName, DataType dataType, String payload) throws AdaptorException {
107 211 try {
108 212 return JsonConverter.convertToAttributesProto(getKvJson(keyName, dataType, payload));
... ... @@ -143,14 +247,6 @@ public class SnmpSessionListener implements ResponseListener {
143 247 return new JsonParser().parse(result.toString());
144 248 }
145 249
146   - private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) {
147   - snmpTransportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder()
148   - .setAttributeSubscription(false)
149   - .setRpcSubscription(false)
150   - .setLastActivityTime(System.currentTimeMillis())
151   - .build(), TransportServiceCallback.EMPTY);
152   - }
153   -
154 250 @AllArgsConstructor
155 251 private static class DeviceAuthCallback implements TransportServiceCallback<ValidateDeviceCredentialsResponse> {
156 252 private final TransportContext transportContext;
... ... @@ -170,4 +266,9 @@ public class SnmpSessionListener implements ResponseListener {
170 266 log.warn("Failed to process device auth", e);
171 267 }
172 268 }
  269 +
  270 + @Override
  271 + public String getName() {
  272 + return "SNMP";
  273 + }
173 274 }
... ...
common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java renamed from common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java
... ... @@ -19,21 +19,18 @@ import lombok.Getter;
19 19 import lombok.Setter;
20 20 import lombok.extern.slf4j.Slf4j;
21 21 import org.snmp4j.CommunityTarget;
22   -import org.snmp4j.Snmp;
23 22 import org.snmp4j.Target;
  23 +import org.snmp4j.event.ResponseEvent;
  24 +import org.snmp4j.event.ResponseListener;
24 25 import org.snmp4j.mp.SnmpConstants;
25 26 import org.snmp4j.smi.GenericAddress;
26 27 import org.snmp4j.smi.OctetString;
27 28 import org.thingsboard.server.common.data.Device;
28 29 import org.thingsboard.server.common.data.DeviceProfile;
29   -import org.thingsboard.server.common.data.DeviceTransportType;
30 30 import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration;
31 31 import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration;
32 32 import org.thingsboard.server.common.data.id.DeviceId;
33 33 import org.thingsboard.server.common.transport.SessionMsgListener;
34   -import org.thingsboard.server.common.transport.TransportServiceCallback;
35   -import org.thingsboard.server.common.transport.auth.SessionInfoCreator;
36   -import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse;
37 34 import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext;
38 35 import org.thingsboard.server.gen.transport.TransportProtos;
39 36 import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg;
... ... @@ -42,154 +39,92 @@ import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotifica
42 39 import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg;
43 40 import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg;
44 41 import org.thingsboard.server.transport.snmp.SnmpTransportContext;
  42 +import org.thingsboard.server.transport.snmp.service.SnmpTransportService;
45 43
46   -import java.io.IOException;
47   -import java.util.Optional;
48 44 import java.util.UUID;
49 45 import java.util.concurrent.atomic.AtomicInteger;
50   -import java.util.function.Consumer;
51 46
52 47 @Slf4j
53   -public class DeviceSessionCtx extends DeviceAwareSessionContext implements SessionMsgListener {
54   - private final AtomicInteger msgIdSeq = new AtomicInteger(0);
55   -
  48 +public class DeviceSessionContext extends DeviceAwareSessionContext implements SessionMsgListener, ResponseListener {
56 49 @Getter
57   - @Setter
58   - private SnmpDeviceTransportConfiguration deviceTransportConfig;
  50 + private Target target;
  51 + private final String token;
59 52 @Getter
60 53 @Setter
61   - private SnmpSessionListener snmpSessionListener;
  54 + private SnmpProfileTransportConfiguration profileTransportConfiguration;
62 55 @Getter
63 56 @Setter
64   - private Target target;
  57 + private SnmpDeviceTransportConfiguration deviceTransportConfiguration;
65 58 @Getter
66   - @Setter
67   - private volatile TransportProtos.SessionInfoProto sessionInfo;
  59 + private final Device device;
  60 +
  61 + private final SnmpTransportContext snmpTransportContext;
  62 + private final SnmpTransportService snmpTransportService;
68 63
69   - private Snmp snmp;
70   - private SnmpProfileTransportConfiguration snmpProfileTransportConfiguration;
  64 + @Getter
  65 + @Setter
71 66 private long previousRequestExecutedAt = 0;
  67 + private final AtomicInteger msgIdSeq = new AtomicInteger(0);
  68 + private boolean isActive = true;
72 69
73   - public DeviceSessionCtx(SnmpTransportContext transportContext, String token, SnmpDeviceTransportConfiguration deviceTransportConfig,
74   - Snmp snmp, DeviceId deviceId, DeviceProfile deviceProfile) {
  70 + public DeviceSessionContext(Device device, DeviceProfile deviceProfile,
  71 + String token, SnmpDeviceTransportConfiguration deviceTransportConfiguration,
  72 + SnmpTransportContext snmpTransportContext, SnmpTransportService snmpTransportService) {
75 73 super(UUID.randomUUID());
76   - this.snmpSessionListener = new SnmpSessionListener(transportContext, token);
77   - super.setDeviceId(deviceId);
  74 + super.setDeviceId(device.getId());
78 75 super.setDeviceProfile(deviceProfile);
79   - //TODO: What should be done if snmp null?
80   - if (snmp != null) {
81   - this.snmp = snmp;
82   - }
83   - this.snmpProfileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration();
84   - initTarget(this.snmpProfileTransportConfiguration, deviceTransportConfig);
85   - }
  76 + this.device = device;
86 77
87   - @Override
88   - public int nextMsgId() {
89   - return msgIdSeq.incrementAndGet();
90   - }
  78 + this.token = token;
  79 + this.snmpTransportContext = snmpTransportContext;
  80 + this.snmpTransportService = snmpTransportService;
91 81
92   - @Override
93   - public void onGetAttributesResponse(GetAttributeResponseMsg getAttributesResponse) {
94   - }
95   -
96   - @Override
97   - public void onAttributeUpdate(AttributeUpdateNotificationMsg attributeUpdateNotification) {
98   - }
99   -
100   - @Override
101   - public void onRemoteSessionCloseCommand(SessionCloseNotificationProto sessionCloseNotification) {
102   - }
103   -
104   - @Override
105   - public void onToDeviceRpcRequest(ToDeviceRpcRequestMsg toDeviceRequest) {
106   - }
  82 + this.profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration();
  83 + this.deviceTransportConfiguration = deviceTransportConfiguration;
107 84
108   - @Override
109   - public void onToServerRpcResponse(ToServerRpcResponseMsg toServerResponse) {
  85 + initTarget(this.profileTransportConfiguration, this.deviceTransportConfiguration);
110 86 }
111 87
112 88 @Override
113 89 public void onDeviceProfileUpdate(TransportProtos.SessionInfoProto newSessionInfo, DeviceProfile deviceProfile) {
114   - super.onDeviceProfileUpdate(sessionInfo, deviceProfile);
115   - if (DeviceTransportType.SNMP.equals(deviceProfile.getTransportType())) {
116   - snmpProfileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration();
117   - snmpSessionListener.getSnmpTransportContext().getProfileTransportConfig().put(
118   - deviceProfile.getId(),
119   - snmpProfileTransportConfiguration);
120   - snmpSessionListener.getSnmpTransportContext().updatePduListPerProfile(deviceProfile.getId(), snmpProfileTransportConfiguration);
121   - } else {
122   - //TODO: should the context be removed from the map?
  90 + super.onDeviceProfileUpdate(newSessionInfo, deviceProfile);
  91 + if (isActive) {
  92 + snmpTransportContext.onDeviceProfileUpdated(deviceProfile, this);
123 93 }
124 94 }
125 95
126 96 @Override
127   - public void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, Device device, Optional<DeviceProfile> deviceProfileOpt) {
128   - super.onDeviceUpdate(sessionInfo, device, deviceProfileOpt);
129   - if (super.getDeviceProfile() != null && DeviceTransportType.SNMP.equals(super.getDeviceProfile().getTransportType())) {
130   - snmpSessionListener.getSnmpTransportContext().updateDeviceSessionCtx(device, deviceProfile, null);
131   - SnmpProfileTransportConfiguration profileTransportConfig = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration();
132   - SnmpDeviceTransportConfiguration deviceTransportConfig = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration();
133   - initTarget(profileTransportConfig, deviceTransportConfig);
134   - } else {
135   - //TODO: should the context be removed from the map?
136   - }
137   - }
138   -
139   - public void createSessionInfo(Consumer<TransportProtos.SessionInfoProto> registerSession) {
140   - getSnmpSessionListener().getSnmpTransportContext().getTransportService().process(DeviceTransportType.SNMP,
141   - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(getSnmpSessionListener().getToken()).build(),
142   - new TransportServiceCallback<ValidateDeviceCredentialsResponse>() {
143   - @Override
144   - public void onSuccess(ValidateDeviceCredentialsResponse msg) {
145   - if (msg.hasDeviceInfo()) {
146   - sessionInfo = SessionInfoCreator.create(msg, getSnmpSessionListener().getSnmpTransportContext(), UUID.randomUUID());
147   - registerSession.accept(sessionInfo);
148   - setDeviceInfo(msg.getDeviceInfo());
149   - } else {
150   - log.warn("[{}] Failed to process device auth", getDeviceId());
151   - }
152   - }
153   -
154   - @Override
155   - public void onError(Throwable e) {
156   - log.warn("[{}] Failed to process device auth", getDeviceId(), e);
157   - }
158   - });
  97 + public void onDeviceDeleted(DeviceId deviceId) {
  98 + snmpTransportContext.onDeviceDeleted(this);
159 99 }
160 100
161   - public void executeSnmpRequest() {
162   - long timeNow = System.currentTimeMillis();
163   - long nextRequestExecutionTime = previousRequestExecutedAt + snmpProfileTransportConfiguration.getPoolPeriodMs();
164   - if (nextRequestExecutionTime < timeNow) {
165   - previousRequestExecutedAt = timeNow;
166   -
167   - snmpSessionListener.getSnmpTransportContext().getPdusPerProfile().get(deviceProfile.getId()).forEach(pdu -> {
168   - try {
169   - log.debug("[{}] Sending SNMP message...", pdu.getRequestID());
170   - snmp.send(pdu,
171   - target,
172   - deviceProfile.getId(),
173   - snmpSessionListener);
174   - } catch (IOException e) {
175   - log.error(e.getMessage(), e);
176   - }
177   - });
  101 + @Override
  102 + public void onResponse(ResponseEvent event) {
  103 + if (isActive) {
  104 + snmpTransportService.onNewDeviceResponse(event, this);
178 105 }
179 106 }
180 107
181   - private void initTarget(SnmpProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) {
182   - this.deviceTransportConfig = deviceTransportConfig;
  108 + public void initTarget(SnmpProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) {
  109 + log.trace("Initializing target for SNMP session of device {}", device);
183 110 CommunityTarget communityTarget = new CommunityTarget();
184   - communityTarget.setAddress(GenericAddress.parse(GenericAddress.TYPE_UDP + ":" + this.deviceTransportConfig.getAddress() + "/" + this.deviceTransportConfig.getPort()));
185   - communityTarget.setVersion(getSnmpVersion(this.deviceTransportConfig.getProtocolVersion()));
186   - communityTarget.setCommunity(new OctetString(this.deviceTransportConfig.getCommunity()));
  111 + communityTarget.setAddress(GenericAddress.parse(GenericAddress.TYPE_UDP + ":" + deviceTransportConfig.getAddress() + "/" + deviceTransportConfig.getPort()));
  112 + communityTarget.setVersion(getSnmpVersion(deviceTransportConfig.getProtocolVersion()));
  113 + communityTarget.setCommunity(new OctetString(deviceTransportConfig.getCommunity()));
187 114 communityTarget.setTimeout(profileTransportConfig.getTimeoutMs());
188 115 communityTarget.setRetries(profileTransportConfig.getRetries());
189 116 this.target = communityTarget;
190 117 log.info("SNMP target initialized: {}", this.target);
191 118 }
192 119
  120 + public void close() {
  121 + isActive = false;
  122 + }
  123 +
  124 + public String getToken() {
  125 + return token;
  126 + }
  127 +
193 128 //TODO: replace with enum, wtih preliminary discussion of type version in config (string or integer)
194 129 private int getSnmpVersion(String configSnmpVersion) {
195 130 switch (configSnmpVersion) {
... ... @@ -203,4 +138,29 @@ public class DeviceSessionCtx extends DeviceAwareSessionContext implements Sessi
203 138 return -1;
204 139 }
205 140 }
  141 +
  142 + @Override
  143 + public int nextMsgId() {
  144 + return msgIdSeq.incrementAndGet();
  145 + }
  146 +
  147 + @Override
  148 + public void onGetAttributesResponse(GetAttributeResponseMsg getAttributesResponse) {
  149 + }
  150 +
  151 + @Override
  152 + public void onAttributeUpdate(AttributeUpdateNotificationMsg attributeUpdateNotification) {
  153 + }
  154 +
  155 + @Override
  156 + public void onRemoteSessionCloseCommand(SessionCloseNotificationProto sessionCloseNotification) {
  157 + }
  158 +
  159 + @Override
  160 + public void onToDeviceRpcRequest(ToDeviceRpcRequestMsg toDeviceRequest) {
  161 + }
  162 +
  163 + @Override
  164 + public void onToServerRpcResponse(ToServerRpcResponseMsg toServerResponse) {
  165 + }
206 166 }
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.transport;
  17 +
  18 +import lombok.Getter;
  19 +import org.thingsboard.server.common.data.Device;
  20 +
  21 +@Getter
  22 +public class DeviceUpdatedEvent {
  23 + private final Device device;
  24 +
  25 + public DeviceUpdatedEvent(Device device) {
  26 + this.device = device;
  27 + }
  28 +}
... ...
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.common.transport;
17 17
18 18 import org.thingsboard.server.common.data.Device;
19 19 import org.thingsboard.server.common.data.DeviceProfile;
  20 +import org.thingsboard.server.common.data.id.DeviceId;
20 21 import org.thingsboard.server.gen.transport.TransportProtos;
21 22 import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg;
22 23 import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg;
... ... @@ -49,4 +50,7 @@ public interface SessionMsgListener {
49 50
50 51 default void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, Device device, Optional<DeviceProfile> deviceProfileOpt) {
51 52 }
  53 +
  54 + default void onDeviceDeleted(DeviceId deviceId) {
  55 + }
52 56 }
... ...
... ... @@ -21,6 +21,7 @@ import lombok.Getter;
21 21 import lombok.extern.slf4j.Slf4j;
22 22 import org.springframework.beans.factory.annotation.Autowired;
23 23 import org.springframework.stereotype.Service;
  24 +import org.thingsboard.server.common.data.DeviceTransportType;
24 25 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
25 26 import org.thingsboard.server.queue.scheduler.SchedulerComponent;
26 27 import org.thingsboard.server.queue.util.TbTransportComponent;
... ...
... ... @@ -22,11 +22,17 @@ import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsRes
22 22 import org.thingsboard.server.common.transport.service.SessionMetaData;
23 23 import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg;
24 24 import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg;
  25 +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceCredentialsRequestMsg;
  26 +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceCredentialsResponseMsg;
  27 +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceRequestMsg;
  28 +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceResponseMsg;
25 29 import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg;
26 30 import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg;
27 31 import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg;
28 32 import org.thingsboard.server.gen.transport.TransportProtos.GetResourcesRequestMsg;
29 33 import org.thingsboard.server.gen.transport.TransportProtos.GetResourcesResponseMsg;
  34 +import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesRequestMsg;
  35 +import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesResponseMsg;
30 36 import org.thingsboard.server.gen.transport.TransportProtos.LwM2MRequestMsg;
31 37 import org.thingsboard.server.gen.transport.TransportProtos.LwM2MResponseMsg;
32 38 import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg;
... ... @@ -55,6 +61,12 @@ public interface TransportService {
55 61
56 62 GetResourcesResponseMsg getResources(GetResourcesRequestMsg msg);
57 63
  64 + GetSnmpDevicesResponseMsg getSnmpDevicesIds(GetSnmpDevicesRequestMsg requestMsg);
  65 +
  66 + GetDeviceResponseMsg getDevice(GetDeviceRequestMsg requestMsg);
  67 +
  68 + GetDeviceCredentialsResponseMsg getDeviceCredentials(GetDeviceCredentialsRequestMsg requestMsg);
  69 +
58 70 void process(DeviceTransportType transportType, ValidateDeviceTokenRequestMsg msg,
59 71 TransportServiceCallback<ValidateDeviceCredentialsResponse> callback);
60 72
... ...
... ... @@ -112,8 +112,8 @@ public class DefaultTransportDeviceProfileCache implements TransportDeviceProfil
112 112 profile = profileOpt.get();
113 113 this.put(profile);
114 114 } else {
115   - log.warn("[{}] Can't device profile: {}", id, entityProfileMsg.getData());
116   - throw new RuntimeException("Can't device profile!");
  115 + log.warn("[{}] Can't find device profile: {}", id, entityProfileMsg.getData());
  116 + throw new RuntimeException("Can't find device profile!");
117 117 }
118 118 } finally {
119 119 deviceProfileFetchLock.unlock();
... ...
... ... @@ -23,6 +23,7 @@ import com.google.gson.JsonObject;
23 23 import com.google.protobuf.ByteString;
24 24 import lombok.extern.slf4j.Slf4j;
25 25 import org.springframework.beans.factory.annotation.Value;
  26 +import org.springframework.context.ApplicationEventPublisher;
26 27 import org.springframework.stereotype.Service;
27 28 import org.thingsboard.common.util.ThingsBoardThreadFactory;
28 29 import org.thingsboard.server.common.data.ApiUsageRecordKey;
... ... @@ -47,6 +48,7 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException;
47 48 import org.thingsboard.server.common.stats.MessagesStats;
48 49 import org.thingsboard.server.common.stats.StatsFactory;
49 50 import org.thingsboard.server.common.stats.StatsType;
  51 +import org.thingsboard.server.common.transport.DeviceUpdatedEvent;
50 52 import org.thingsboard.server.common.transport.SessionMsgListener;
51 53 import org.thingsboard.server.common.transport.TransportDeviceProfileCache;
52 54 import org.thingsboard.server.common.transport.TransportService;
... ... @@ -61,7 +63,6 @@ import org.thingsboard.server.common.transport.util.JsonUtils;
61 63 import org.thingsboard.server.gen.transport.TransportProtos;
62 64 import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg;
63 65 import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg;
64   -import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto;
65 66 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
66 67 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
67 68 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
... ... @@ -131,6 +132,7 @@ public class DefaultTransportService implements TransportService {
131 132 private final TransportRateLimitService rateLimitService;
132 133 private final DataDecodingEncodingService dataDecodingEncodingService;
133 134 private final SchedulerComponent scheduler;
  135 + private final ApplicationEventPublisher eventPublisher;
134 136
135 137 protected TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> transportApiRequestTemplate;
136 138 protected TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> ruleEngineMsgProducer;
... ... @@ -157,7 +159,8 @@ public class DefaultTransportService implements TransportService {
157 159 TransportDeviceProfileCache deviceProfileCache,
158 160 TransportTenantProfileCache tenantProfileCache,
159 161 TbApiUsageClient apiUsageClient, TransportRateLimitService rateLimitService,
160   - DataDecodingEncodingService dataDecodingEncodingService, SchedulerComponent scheduler) {
  162 + DataDecodingEncodingService dataDecodingEncodingService, SchedulerComponent scheduler,
  163 + ApplicationEventPublisher eventPublisher) {
161 164 this.serviceInfoProvider = serviceInfoProvider;
162 165 this.queueProvider = queueProvider;
163 166 this.producerProvider = producerProvider;
... ... @@ -169,6 +172,7 @@ public class DefaultTransportService implements TransportService {
169 172 this.rateLimitService = rateLimitService;
170 173 this.dataDecodingEncodingService = dataDecodingEncodingService;
171 174 this.scheduler = scheduler;
  175 + this.eventPublisher = eventPublisher;
172 176 }
173 177
174 178 @PostConstruct
... ... @@ -267,6 +271,58 @@ public class DefaultTransportService implements TransportService {
267 271 }
268 272
269 273 @Override
  274 + public TransportProtos.GetSnmpDevicesResponseMsg getSnmpDevicesIds(TransportProtos.GetSnmpDevicesRequestMsg requestMsg) {
  275 + TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg> protoMsg = new TbProtoQueueMsg<>(
  276 + UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder()
  277 + .setSnmpDevicesRequestMsg(requestMsg)
  278 + .build()
  279 + );
  280 +
  281 + try {
  282 + TbProtoQueueMsg<TransportApiResponseMsg> response = transportApiRequestTemplate.send(protoMsg).get();
  283 + return response.getValue().getSnmpDevicesResponseMsg();
  284 + } catch (InterruptedException | ExecutionException e) {
  285 + throw new RuntimeException(e);
  286 + }
  287 + }
  288 +
  289 + @Override
  290 + public TransportProtos.GetDeviceResponseMsg getDevice(TransportProtos.GetDeviceRequestMsg requestMsg) {
  291 + TbProtoQueueMsg<TransportApiRequestMsg> protoMsg = new TbProtoQueueMsg<>(
  292 + UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder()
  293 + .setDeviceRequestMsg(requestMsg)
  294 + .build()
  295 + );
  296 +
  297 + try {
  298 + TransportApiResponseMsg response = transportApiRequestTemplate.send(protoMsg).get().getValue();
  299 + if (response.hasDeviceResponseMsg()) {
  300 + return response.getDeviceResponseMsg();
  301 + } else {
  302 + return null;
  303 + }
  304 + } catch (InterruptedException | ExecutionException e) {
  305 + throw new RuntimeException(e);
  306 + }
  307 + }
  308 +
  309 + @Override
  310 + public TransportProtos.GetDeviceCredentialsResponseMsg getDeviceCredentials(TransportProtos.GetDeviceCredentialsRequestMsg requestMsg) {
  311 + TbProtoQueueMsg<TransportApiRequestMsg> protoMsg = new TbProtoQueueMsg<>(
  312 + UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder()
  313 + .setDeviceCredentialsRequestMsg(requestMsg)
  314 + .build()
  315 + );
  316 +
  317 + try {
  318 + TbProtoQueueMsg<TransportApiResponseMsg> response = transportApiRequestTemplate.send(protoMsg).get();
  319 + return response.getValue().getDeviceCredentialsResponseMsg();
  320 + } catch (InterruptedException | ExecutionException e) {
  321 + throw new RuntimeException(e);
  322 + }
  323 + }
  324 +
  325 + @Override
270 326 public void process(DeviceTransportType transportType, TransportProtos.ValidateDeviceTokenRequestMsg msg,
271 327 TransportServiceCallback<ValidateDeviceCredentialsResponse> callback) {
272 328 log.trace("Processing msg: {}", msg);
... ... @@ -685,7 +741,10 @@ public class DefaultTransportService implements TransportService {
685 741 }
686 742 } else if (EntityType.DEVICE.equals(entityType)) {
687 743 Optional<Device> deviceOpt = dataDecodingEncodingService.decode(msg.getData().toByteArray());
688   - deviceOpt.ifPresent(this::onDeviceUpdate);
  744 + deviceOpt.ifPresent(device -> {
  745 + onDeviceUpdate(device);
  746 + eventPublisher.publishEvent(new DeviceUpdatedEvent(device));
  747 + });
689 748 }
690 749 } else if (toSessionMsg.hasEntityDeleteMsg()) {
691 750 TransportProtos.EntityDeleteMsg msg = toSessionMsg.getEntityDeleteMsg();
... ... @@ -699,6 +758,7 @@ public class DefaultTransportService implements TransportService {
699 758 rateLimitService.remove(new TenantId(entityUuid));
700 759 } else if (EntityType.DEVICE.equals(entityType)) {
701 760 rateLimitService.remove(new DeviceId(entityUuid));
  761 + onDeviceDeleted(new DeviceId(entityUuid));
702 762 }
703 763 } else if (toSessionMsg.hasResourceUpdateMsg()) {
704 764 //TODO: update resource cache
... ... @@ -764,6 +824,17 @@ public class DefaultTransportService implements TransportService {
764 824 });
765 825 }
766 826
  827 + private void onDeviceDeleted(DeviceId deviceId) {
  828 + sessions.forEach((id, md) -> {
  829 + DeviceId sessionDeviceId = new DeviceId(new UUID(md.getSessionInfo().getDeviceIdMSB(), md.getSessionInfo().getDeviceIdLSB()));
  830 + if (sessionDeviceId.equals(deviceId)) {
  831 + transportCallbackExecutor.submit(() -> {
  832 + md.getListener().onDeviceDeleted(deviceId);
  833 + });
  834 + }
  835 + });
  836 + }
  837 +
767 838 protected UUID toSessionId(TransportProtos.SessionInfoProto sessionInfo) {
768 839 return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB());
769 840 }
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.device;
18 18 import com.google.common.util.concurrent.ListenableFuture;
19 19 import org.thingsboard.server.common.data.Device;
20 20 import org.thingsboard.server.common.data.DeviceInfo;
  21 +import org.thingsboard.server.common.data.DeviceTransportType;
21 22 import org.thingsboard.server.common.data.EntitySubtype;
22 23 import org.thingsboard.server.common.data.id.TenantId;
23 24 import org.thingsboard.server.common.data.page.PageData;
... ... @@ -215,4 +216,7 @@ public interface DeviceDao extends Dao<Device>, TenantEntityDao {
215 216 * @return the list of device objects
216 217 */
217 218 PageData<Device> findDevicesByTenantIdAndProfileId(UUID tenantId, UUID profileId, PageLink pageLink);
  219 +
  220 + List<UUID> findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType);
  221 +
218 222 }
... ...
... ... @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.Customer;
36 36 import org.thingsboard.server.common.data.Device;
37 37 import org.thingsboard.server.common.data.DeviceInfo;
38 38 import org.thingsboard.server.common.data.DeviceProfile;
  39 +import org.thingsboard.server.common.data.DeviceTransportType;
39 40 import org.thingsboard.server.common.data.EntitySubtype;
40 41 import org.thingsboard.server.common.data.EntityType;
41 42 import org.thingsboard.server.common.data.EntityView;
... ... @@ -80,6 +81,7 @@ import java.util.Collections;
80 81 import java.util.Comparator;
81 82 import java.util.List;
82 83 import java.util.Optional;
  84 +import java.util.UUID;
83 85 import java.util.concurrent.ExecutionException;
84 86 import java.util.stream.Collectors;
85 87
... ... @@ -554,6 +556,11 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe
554 556 return savedDevice;
555 557 }
556 558
  559 + @Override
  560 + public List<UUID> findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType) {
  561 + return deviceDao.findDevicesIdsByDeviceProfileTransportType(transportType);
  562 + }
  563 +
557 564 private DataValidator<Device> deviceValidator =
558 565 new DataValidator<Device>() {
559 566
... ...
... ... @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable;
20 20 import org.springframework.data.jpa.repository.Query;
21 21 import org.springframework.data.repository.PagingAndSortingRepository;
22 22 import org.springframework.data.repository.query.Param;
  23 +import org.thingsboard.server.common.data.DeviceTransportType;
23 24 import org.thingsboard.server.dao.model.sql.DeviceEntity;
24 25 import org.thingsboard.server.dao.model.sql.DeviceInfoEntity;
25 26
... ... @@ -170,4 +171,9 @@ public interface DeviceRepository extends PagingAndSortingRepository<DeviceEntit
170 171 Long countByDeviceProfileId(UUID deviceProfileId);
171 172
172 173 Long countByTenantId(UUID tenantId);
  174 +
  175 + @Query("SELECT d.id FROM DeviceEntity d " +
  176 + "INNER JOIN DeviceProfileEntity p ON d.deviceProfileId = p.id " +
  177 + "WHERE p.transportType = :transportType")
  178 + List<UUID> findIdsByDeviceProfileTransportType(@Param("transportType") DeviceTransportType transportType);
173 179 }
... ...
... ... @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component;
22 22 import org.springframework.util.StringUtils;
23 23 import org.thingsboard.server.common.data.Device;
24 24 import org.thingsboard.server.common.data.DeviceInfo;
  25 +import org.thingsboard.server.common.data.DeviceTransportType;
25 26 import org.thingsboard.server.common.data.EntitySubtype;
26 27 import org.thingsboard.server.common.data.EntityType;
27 28 import org.thingsboard.server.common.data.id.TenantId;
... ... @@ -115,6 +116,11 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao<DeviceEntity, Device>
115 116 }
116 117
117 118 @Override
  119 + public List<UUID> findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType) {
  120 + return deviceRepository.findIdsByDeviceProfileTransportType(transportType);
  121 + }
  122 +
  123 + @Override
118 124 public PageData<DeviceInfo> findDeviceInfosByTenantIdAndCustomerId(UUID tenantId, UUID customerId, PageLink pageLink) {
119 125 return DaoUtil.toPageData(
120 126 deviceRepository.findDeviceInfosByTenantIdAndCustomerId(
... ...
  1 +<?xml version="1.0" encoding="UTF-8" ?>
  2 +<!--
  3 +
  4 + Copyright © 2016-2021 The Thingsboard Authors
  5 +
  6 + Licensed under the Apache License, Version 2.0 (the "License");
  7 + you may not use this file except in compliance with the License.
  8 + You may obtain a copy of the License at
  9 +
  10 + http://www.apache.org/licenses/LICENSE-2.0
  11 +
  12 + Unless required by applicable law or agreed to in writing, software
  13 + distributed under the License is distributed on an "AS IS" BASIS,
  14 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 + See the License for the specific language governing permissions and
  16 + limitations under the License.
  17 +
  18 +-->
  19 +<!DOCTYPE configuration>
  20 +<configuration scan="true" scanPeriod="10 seconds">
  21 +
  22 + <appender name="fileLogAppender"
  23 + class="ch.qos.logback.core.rolling.RollingFileAppender">
  24 + <file>/var/log/tb-snmp-transport/${TB_SERVICE_ID}/tb-snmp-transport.log</file>
  25 + <rollingPolicy
  26 + class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
  27 + <fileNamePattern>/var/log/tb-snmp-transport/${TB_SERVICE_ID}/tb-snmp-transport.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
  28 + <maxFileSize>100MB</maxFileSize>
  29 + <maxHistory>30</maxHistory>
  30 + <totalSizeCap>3GB</totalSizeCap>
  31 + </rollingPolicy>
  32 + <encoder>
  33 + <pattern>%d{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
  34 + </encoder>
  35 + </appender>
  36 +
  37 + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
  38 + <encoder>
  39 + <pattern>%d{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
  40 + </encoder>
  41 + </appender>
  42 +
  43 + <logger name="org.thingsboard.server" level="INFO" />
  44 +
  45 + <root level="INFO">
  46 + <appender-ref ref="fileLogAppender"/>
  47 + <appender-ref ref="STDOUT"/>
  48 + </root>
  49 +
  50 +</configuration>
\ No newline at end of file
... ...
  1 +#
  2 +# Copyright © 2016-2021 The Thingsboard Authors
  3 +#
  4 +# Licensed under the Apache License, Version 2.0 (the "License");
  5 +# you may not use this file except in compliance with the License.
  6 +# You may obtain a copy of the License at
  7 +#
  8 +# http://www.apache.org/licenses/LICENSE-2.0
  9 +#
  10 +# Unless required by applicable law or agreed to in writing, software
  11 +# distributed under the License is distributed on an "AS IS" BASIS,
  12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +# See the License for the specific language governing permissions and
  14 +# limitations under the License.
  15 +#
  16 +
  17 +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/tb-snmp-transport/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M"
  18 +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-snmp-transport/${TB_SERVICE_ID}-heapdump.bin"
  19 +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark"
  20 +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10"
  21 +export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError"
  22 +export LOG_FILENAME=tb-snmp-transport.out
  23 +export LOADER_PATH=/usr/share/tb-snmp-transport/conf
... ...
... ... @@ -14,7 +14,7 @@
14 14 # limitations under the License.
15 15 #
16 16
17   -FROM thingsboard/openjdk8
  17 +FROM thingsboard/openjdk11
18 18
19 19 COPY start-tb-snmp-transport.sh ${pkg.name}.deb /tmp/
20 20
... ...
  1 +<?xml version="1.0" encoding="UTF-8" ?>
  2 +<!--
  3 +
  4 + Copyright © 2016-2021 The Thingsboard Authors
  5 +
  6 + Licensed under the Apache License, Version 2.0 (the "License");
  7 + you may not use this file except in compliance with the License.
  8 + You may obtain a copy of the License at
  9 +
  10 + http://www.apache.org/licenses/LICENSE-2.0
  11 +
  12 + Unless required by applicable law or agreed to in writing, software
  13 + distributed under the License is distributed on an "AS IS" BASIS,
  14 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 + See the License for the specific language governing permissions and
  16 + limitations under the License.
  17 +
  18 +-->
  19 +<!DOCTYPE configuration>
  20 +<configuration>
  21 +
  22 + <appender name="fileLogAppender"
  23 + class="ch.qos.logback.core.rolling.RollingFileAppender">
  24 + <file>${pkg.logFolder}/${pkg.name}.log</file>
  25 + <rollingPolicy
  26 + class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
  27 + <fileNamePattern>${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
  28 + <maxFileSize>100MB</maxFileSize>
  29 + <maxHistory>30</maxHistory>
  30 + <totalSizeCap>3GB</totalSizeCap>
  31 + </rollingPolicy>
  32 + <encoder>
  33 + <pattern>%d{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
  34 + </encoder>
  35 + </appender>
  36 +
  37 + <logger name="org.thingsboard.server" level="INFO" />
  38 +
  39 + <logger name="com.microsoft.azure.servicebus.primitives.CoreMessageReceiver" level="OFF" />
  40 +
  41 + <root level="INFO">
  42 + <appender-ref ref="fileLogAppender"/>
  43 + </root>
  44 +
  45 +</configuration>
... ...
  1 +#
  2 +# Copyright © 2016-2021 The Thingsboard Authors
  3 +#
  4 +# Licensed under the Apache License, Version 2.0 (the "License");
  5 +# you may not use this file except in compliance with the License.
  6 +# You may obtain a copy of the License at
  7 +#
  8 +# http://www.apache.org/licenses/LICENSE-2.0
  9 +#
  10 +# Unless required by applicable law or agreed to in writing, software
  11 +# distributed under the License is distributed on an "AS IS" BASIS,
  12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +# See the License for the specific language governing permissions and
  14 +# limitations under the License.
  15 +#
  16 +
  17 +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M"
  18 +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError"
  19 +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark"
  20 +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10"
  21 +export LOG_FILENAME=${pkg.name}.out
  22 +export LOADER_PATH=${pkg.installFolder}/conf
... ...
  1 +<?xml version="1.0" encoding="UTF-8" ?>
  2 +<!--
  3 +
  4 + Copyright © 2016-2021 The Thingsboard Authors
  5 +
  6 + Licensed under the Apache License, Version 2.0 (the "License");
  7 + you may not use this file except in compliance with the License.
  8 + You may obtain a copy of the License at
  9 +
  10 + http://www.apache.org/licenses/LICENSE-2.0
  11 +
  12 + Unless required by applicable law or agreed to in writing, software
  13 + distributed under the License is distributed on an "AS IS" BASIS,
  14 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 + See the License for the specific language governing permissions and
  16 + limitations under the License.
  17 +
  18 +-->
  19 +<!DOCTYPE configuration>
  20 +<configuration scan="true" scanPeriod="10 seconds">
  21 +
  22 + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
  23 + <encoder>
  24 + <pattern>%d{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
  25 + </encoder>
  26 + </appender>
  27 +
  28 + <logger name="org.thingsboard.server" level="TRACE" />
  29 +
  30 + <logger name="com.microsoft.azure.servicebus.primitives.CoreMessageReceiver" level="OFF" />
  31 +
  32 + <root level="INFO">
  33 + <appender-ref ref="STDOUT"/>
  34 + </root>
  35 +
  36 +</configuration>
\ No newline at end of file
... ...
  1 +#
  2 +# Copyright © 2016-2021 The Thingsboard Authors
  3 +#
  4 +# Licensed under the Apache License, Version 2.0 (the "License");
  5 +# you may not use this file except in compliance with the License.
  6 +# You may obtain a copy of the License at
  7 +#
  8 +# http://www.apache.org/licenses/LICENSE-2.0
  9 +#
  10 +# Unless required by applicable law or agreed to in writing, software
  11 +# distributed under the License is distributed on an "AS IS" BASIS,
  12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +# See the License for the specific language governing permissions and
  14 +# limitations under the License.
  15 +#
  16 +
  17 +server:
  18 + # Server bind address
  19 + address: "${HTTP_BIND_ADDRESS:0.0.0.0}"
  20 + # Server bind port
  21 + port: "${HTTP_BIND_PORT:8085}"
  22 + # Server SSL configuration
  23 +
  24 +# Zookeeper connection parameters. Used for service discovery.
  25 +zk:
  26 + # Enable/disable zookeeper discovery service.
  27 + enabled: "${ZOOKEEPER_ENABLED:false}"
  28 + # Zookeeper connect string
  29 + url: "${ZOOKEEPER_URL:localhost:2181}"
  30 + # Zookeeper retry interval in milliseconds
  31 + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}"
  32 + # Zookeeper connection timeout in milliseconds
  33 + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}"
  34 + # Zookeeper session timeout in milliseconds
  35 + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}"
  36 + # Name of the directory in zookeeper 'filesystem'
  37 + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}"
  38 +
  39 +cluster:
  40 + stats:
  41 + enabled: "${TB_CLUSTER_STATS_ENABLED:false}"
  42 + print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}"
  43 +
  44 +cache:
  45 + # caffeine or redis
  46 + type: "${CACHE_TYPE:caffeine}"
  47 + attributes:
  48 + # make sure that if cache.type is 'redis' and cache.attributes.enabled is 'true' that you change 'maxmemory-policy' Redis config property to 'allkeys-lru', 'allkeys-lfu' or 'allkeys-random'
  49 + enabled: "${CACHE_ATTRIBUTES_ENABLED:true}"
  50 +
  51 +caffeine:
  52 + specs:
  53 + relations:
  54 + timeToLiveInMinutes: 1440
  55 + maxSize: 0
  56 + deviceCredentials:
  57 + timeToLiveInMinutes: 1440
  58 + maxSize: 0
  59 + devices:
  60 + timeToLiveInMinutes: 1440
  61 + maxSize: 0
  62 + sessions:
  63 + timeToLiveInMinutes: 1440
  64 + maxSize: 0
  65 + assets:
  66 + timeToLiveInMinutes: 1440
  67 + maxSize: 0
  68 + entityViews:
  69 + timeToLiveInMinutes: 1440
  70 + maxSize: 0
  71 + claimDevices:
  72 + timeToLiveInMinutes: 1
  73 + maxSize: 0
  74 + securitySettings:
  75 + timeToLiveInMinutes: 1440
  76 + maxSize: 0
  77 + tenantProfiles:
  78 + timeToLiveInMinutes: 1440
  79 + maxSize: 0
  80 + deviceProfiles:
  81 + timeToLiveInMinutes: 1440
  82 + maxSize: 0
  83 + attributes:
  84 + timeToLiveInMinutes: 1440
  85 + maxSize: 100000
  86 +
  87 +redis:
  88 + # standalone or cluster
  89 + connection:
  90 + type: "${REDIS_CONNECTION_TYPE:standalone}"
  91 + standalone:
  92 + host: "${REDIS_HOST:localhost}"
  93 + port: "${REDIS_PORT:6379}"
  94 + useDefaultClientConfig: "${REDIS_USE_DEFAULT_CLIENT_CONFIG:true}"
  95 + # this value may be used only if you used not default ClientConfig
  96 + clientName: "${REDIS_CLIENT_NAME:standalone}"
  97 + # this value may be used only if you used not default ClientConfig
  98 + connectTimeout: "${REDIS_CLIENT_CONNECT_TIMEOUT:30000}"
  99 + # this value may be used only if you used not default ClientConfig
  100 + readTimeout: "${REDIS_CLIENT_READ_TIMEOUT:60000}"
  101 + # this value may be used only if you used not default ClientConfig
  102 + usePoolConfig: "${REDIS_CLIENT_USE_POOL_CONFIG:false}"
  103 + cluster:
  104 + # Comma-separated list of "host:port" pairs to bootstrap from.
  105 + nodes: "${REDIS_NODES:}"
  106 + # Maximum number of redirects to follow when executing commands across the cluster.
  107 + max-redirects: "${REDIS_MAX_REDIRECTS:12}"
  108 + useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}"
  109 + # db index
  110 + db: "${REDIS_DB:0}"
  111 + # db password
  112 + password: "${REDIS_PASSWORD:}"
  113 + # pool config
  114 + pool_config:
  115 + maxTotal: "${REDIS_POOL_CONFIG_MAX_TOTAL:128}"
  116 + maxIdle: "${REDIS_POOL_CONFIG_MAX_IDLE:128}"
  117 + minIdle: "${REDIS_POOL_CONFIG_MIN_IDLE:16}"
  118 + testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:true}"
  119 + testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:true}"
  120 + testWhileIdle: "${REDIS_POOL_CONFIG_TEST_WHILE_IDLE:true}"
  121 + minEvictableMs: "${REDIS_POOL_CONFIG_MIN_EVICTABLE_MS:60000}"
  122 + evictionRunsMs: "${REDIS_POOL_CONFIG_EVICTION_RUNS_MS:30000}"
  123 + maxWaitMills: "${REDIS_POOL_CONFIG_MAX_WAIT_MS:60000}"
  124 + numberTestsPerEvictionRun: "${REDIS_POOL_CONFIG_NUMBER_TESTS_PER_EVICTION_RUN:3}"
  125 + blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}"
  126 +
  127 +# Check new version updates parameters
  128 +updates:
  129 + # Enable/disable updates checking.
  130 + enabled: "${UPDATES_ENABLED:true}"
  131 +
  132 +# spring freemarker configuration
  133 +spring.freemarker.checkTemplateLocation: "false"
  134 +
  135 +audit-log:
  136 + # Enable/disable audit log functionality.
  137 + enabled: "${AUDIT_LOG_ENABLED:true}"
  138 + # Specify partitioning size for audit log by tenant id storage. Example MINUTES, HOURS, DAYS, MONTHS
  139 + by_tenant_partitioning: "${AUDIT_LOG_BY_TENANT_PARTITIONING:MONTHS}"
  140 + # Number of days as history period if startTime and endTime are not specified
  141 + default_query_period: "${AUDIT_LOG_DEFAULT_QUERY_PERIOD:30}"
  142 + # Logging levels per each entity type.
  143 + # Allowed values: OFF (disable), W (log write operations), RW (log read and write operations)
  144 + logging-level:
  145 + mask:
  146 + "device": "${AUDIT_LOG_MASK_DEVICE:W}"
  147 + "asset": "${AUDIT_LOG_MASK_ASSET:W}"
  148 + "dashboard": "${AUDIT_LOG_MASK_DASHBOARD:W}"
  149 + "customer": "${AUDIT_LOG_MASK_CUSTOMER:W}"
  150 + "user": "${AUDIT_LOG_MASK_USER:W}"
  151 + "rule_chain": "${AUDIT_LOG_MASK_RULE_CHAIN:W}"
  152 + "alarm": "${AUDIT_LOG_MASK_ALARM:W}"
  153 + "entity_view": "${AUDIT_LOG_MASK_ENTITY_VIEW:W}"
  154 + "device_profile": "${AUDIT_LOG_MASK_DEVICE_PROFILE:W}"
  155 + sink:
  156 + # Type of external sink. possible options: none, elasticsearch
  157 + type: "${AUDIT_LOG_SINK_TYPE:none}"
  158 + # Name of the index where audit logs stored
  159 + # Index name could contain next placeholders (not mandatory):
  160 + # @{TENANT} - substituted by tenant ID
  161 + # @{DATE} - substituted by current date in format provided in audit_log.sink.date_format
  162 + index_pattern: "${AUDIT_LOG_SINK_INDEX_PATTERN:@{TENANT}_AUDIT_LOG_@{DATE}}"
  163 + # Date format. Details of the pattern could be found here:
  164 + # https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html
  165 + date_format: "${AUDIT_LOG_SINK_DATE_FORMAT:YYYY.MM.DD}"
  166 + scheme_name: "${AUDIT_LOG_SINK_SCHEME_NAME:http}" # http or https
  167 + host: "${AUDIT_LOG_SINK_HOST:localhost}"
  168 + port: "${AUDIT_LOG_SINK_PORT:9200}"
  169 + user_name: "${AUDIT_LOG_SINK_USER_NAME:}"
  170 + password: "${AUDIT_LOG_SINK_PASSWORD:}"
  171 +
  172 +state:
  173 + # Should be greater then transport.sessions.report_timeout
  174 + defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}"
  175 + defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}"
  176 + persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}"
  177 +
  178 +transport:
  179 + sessions:
  180 + inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}"
  181 + report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}"
  182 + json:
  183 + # Cast String data types to Numeric if possible when processing Telemetry/Attributes JSON
  184 + type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}"
  185 + # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check)
  186 + max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}"
  187 + # Enable/disable http/mqtt/coap transport protocols (has higher priority than certain protocol's 'enabled' property)
  188 + api_enabled: "${TB_TRANSPORT_API_ENABLED:true}"
  189 + # Local LwM2M transport parameters
  190 + snmp:
  191 + enabled: "${SNMP_ENABLED:true}"
  192 +
  193 +queue:
  194 + type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ)
  195 + in_memory:
  196 + stats:
  197 + # For debug lvl
  198 + print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}"
  199 + kafka:
  200 + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}"
  201 + acks: "${TB_KAFKA_ACKS:all}"
  202 + retries: "${TB_KAFKA_RETRIES:1}"
  203 + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}"
  204 + linger.ms: "${TB_KAFKA_LINGER_MS:1}"
  205 + buffer.memory: "${TB_BUFFER_MEMORY:33554432}"
  206 + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}"
  207 + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}"
  208 + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}"
  209 + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}"
  210 + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}"
  211 + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}"
  212 + confluent:
  213 + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}"
  214 + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}"
  215 + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}"
  216 + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}"
  217 + other:
  218 + topic-properties:
  219 + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
  220 + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
  221 + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
  222 + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
  223 + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}"
  224 + consumer-stats:
  225 + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}"
  226 + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}"
  227 + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}"
  228 + aws_sqs:
  229 + use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}"
  230 + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}"
  231 + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}"
  232 + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}"
  233 + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}"
  234 + queue-properties:
  235 + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}"
  236 + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}"
  237 + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}"
  238 + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}"
  239 + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}"
  240 + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds
  241 + pubsub:
  242 + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}"
  243 + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}"
  244 + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes
  245 + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}"
  246 + queue-properties:
  247 + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
  248 + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
  249 + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
  250 + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
  251 + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
  252 + service_bus:
  253 + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}"
  254 + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}"
  255 + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}"
  256 + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}"
  257 + queue-properties:
  258 + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
  259 + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
  260 + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
  261 + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
  262 + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
  263 + rabbitmq:
  264 + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}"
  265 + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}"
  266 + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}"
  267 + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}"
  268 + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}"
  269 + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}"
  270 + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}"
  271 + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}"
  272 + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}"
  273 + queue-properties:
  274 + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
  275 + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
  276 + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
  277 + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
  278 + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
  279 + partitions:
  280 + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256
  281 + transport_api:
  282 + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}"
  283 + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}"
  284 + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}"
  285 + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}"
  286 + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}"
  287 + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}"
  288 + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}"
  289 + core:
  290 + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}"
  291 + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}"
  292 + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}"
  293 + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}"
  294 + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}"
  295 + stats:
  296 + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}"
  297 + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}"
  298 + js:
  299 + # JS Eval request topic
  300 + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}"
  301 + # JS Eval responses topic prefix that is combined with node id
  302 + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}"
  303 + # JS Eval max pending requests
  304 + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}"
  305 + # JS Eval max request timeout
  306 + max_eval_requests_timeout: "${REMOTE_JS_MAX_EVAL_REQUEST_TIMEOUT:60000}"
  307 + # JS max request timeout
  308 + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}"
  309 + # JS response poll interval
  310 + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}"
  311 + rule-engine:
  312 + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}"
  313 + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}"
  314 + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:2000}"
  315 + stats:
  316 + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}"
  317 + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}"
  318 + queues:
  319 + - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}"
  320 + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}"
  321 + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}"
  322 + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}"
  323 + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:2000}"
  324 + submit-strategy:
  325 + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL
  326 + # For BATCH only
  327 + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch
  328 + processing-strategy:
  329 + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:SKIP_ALL_FAILURES}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
  330 + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
  331 + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited
  332 + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages;
  333 + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries;
  334 + max-pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:3}"# Max allowed time in seconds for pause between retries.
  335 + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}"
  336 + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}"
  337 + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}"
  338 + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}"
  339 + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:2000}"
  340 + submit-strategy:
  341 + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL
  342 + # For BATCH only
  343 + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch
  344 + processing-strategy:
  345 + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
  346 + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
  347 + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited
  348 + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages;
  349 + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries;
  350 + max-pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}"# Max allowed time in seconds for pause between retries.
  351 + - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}"
  352 + topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}"
  353 + poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}"
  354 + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}"
  355 + pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:2000}"
  356 + submit-strategy:
  357 + type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL
  358 + # For BATCH only
  359 + batch-size: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch
  360 + processing-strategy:
  361 + type: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
  362 + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
  363 + retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited
  364 + failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages;
  365 + pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries;
  366 + max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}"# Max allowed time in seconds for pause between retries.
  367 + transport:
  368 + # For high priority notifications that require minimum latency and processing time
  369 + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}"
  370 + poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}"
  371 +
  372 +event:
  373 + debug:
  374 + max-symbols: "${TB_MAX_DEBUG_EVENT_SYMBOLS:4096}"
  375 +
  376 +service:
  377 + type: "${TB_SERVICE_TYPE:tb-transport}"
  378 + # Unique id for this service (autogenerated if empty)
  379 + id: "${TB_SERVICE_ID:}"
  380 + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id.
  381 +
  382 +metrics:
  383 + # Enable/disable actuator metrics.
  384 + enabled: "${METRICS_ENABLED:false}"
  385 + timer:
  386 + # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,).
  387 + percentiles: "${METRICS_TIMER_PERCENTILES:0.5}"
  388 +
  389 +management:
  390 + endpoints:
  391 + web:
  392 + exposure:
  393 + # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
  394 + include: '${METRICS_ENDPOINTS_EXPOSE:info}'
... ...