Commit 6d20ca441ec43a1f678f38f72526509cad47701c

Authored by Sergey Matvienko
Committed by Andrii Shvaika
1 parent 5834bbd7

Merge TS fix using cherry-pick

  1 +package org.thingsboard.server.transport.lwm2m.server;
  2 +
  3 +import org.junit.jupiter.api.BeforeEach;
  4 +import org.junit.jupiter.api.Test;
  5 +import org.thingsboard.server.gen.transport.TransportProtos;
  6 +
  7 +import java.util.List;
  8 +import java.util.concurrent.ConcurrentHashMap;
  9 +import java.util.concurrent.ConcurrentMap;
  10 +import java.util.concurrent.TimeUnit;
  11 +import java.util.concurrent.atomic.AtomicLong;
  12 +
  13 +import static java.util.Collections.emptyList;
  14 +import static org.assertj.core.api.Assertions.assertThat;
  15 +import static org.mockito.ArgumentMatchers.any;
  16 +import static org.mockito.ArgumentMatchers.anyLong;
  17 +import static org.mockito.ArgumentMatchers.anyString;
  18 +import static org.mockito.BDDMockito.willReturn;
  19 +import static org.mockito.Mockito.mock;
  20 +import static org.mockito.Mockito.never;
  21 +import static org.mockito.Mockito.spy;
  22 +import static org.mockito.Mockito.times;
  23 +import static org.mockito.Mockito.verify;
  24 +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_TELEMETRY;
  25 +
  26 +class LwM2mTransportServerHelperTest {
  27 +
  28 + public static final String KEY_SW_STATE = "sw_state";
  29 + public static final String DOWNLOADING = "DOWNLOADING";
  30 +
  31 + long now;
  32 + List<TransportProtos.KeyValueProto> kvList;
  33 + ConcurrentMap<String, AtomicLong> keyTsLatestMap;
  34 + LwM2mTransportServerHelper helper;
  35 + LwM2mTransportContext context;
  36 +
  37 +
  38 + @BeforeEach
  39 + void setUp() {
  40 + now = System.currentTimeMillis();
  41 + context = mock(LwM2mTransportContext.class);
  42 + helper = spy(new LwM2mTransportServerHelper(context));
  43 + willReturn(now).given(helper).getCurrentTimeMillis();
  44 + kvList = List.of(
  45 + TransportProtos.KeyValueProto.newBuilder().setKey(KEY_SW_STATE).setStringV(DOWNLOADING).build(),
  46 + TransportProtos.KeyValueProto.newBuilder().setKey(LOG_LWM2M_TELEMETRY).setStringV("Transport log example").build()
  47 + );
  48 + keyTsLatestMap = new ConcurrentHashMap<>();
  49 + }
  50 +
  51 + @Test
  52 + void givenKeyAndLatestTsMapAndCurrentTs_whenGetTs_thenVerifyNoGetTsByKeyCall() {
  53 + assertThat(helper.getTs(null, null)).isEqualTo(now);
  54 + assertThat(helper.getTs(null, keyTsLatestMap)).isEqualTo(now);
  55 + assertThat(helper.getTs(emptyList(), null)).isEqualTo(now);
  56 + assertThat(helper.getTs(emptyList(), keyTsLatestMap)).isEqualTo(now);
  57 + assertThat(helper.getTs(kvList, null)).isEqualTo(now);
  58 +
  59 + verify(helper, never()).getTsByKey(anyString(), any(ConcurrentMap.class), anyLong());
  60 + verify(helper, times(5)).getCurrentTimeMillis();
  61 + }
  62 +
  63 + @Test
  64 + void givenKeyAndLatestTsMapAndCurrentTs_whenGetTs_thenVerifyGetTsByKeyCallByFirstKey() {
  65 + assertThat(helper.getTs(kvList, keyTsLatestMap)).isEqualTo(now);
  66 +
  67 + verify(helper, times(1)).getTsByKey(kvList.get(0).getKey(), keyTsLatestMap, now);
  68 + verify(helper, times(1)).getTsByKey(anyString(), any(ConcurrentMap.class), anyLong());
  69 + }
  70 +
  71 + @Test
  72 + void givenKeyAndEmptyLatestTsMap_whenGetTsByKey_thenAddToMapAndReturnNow() {
  73 + assertThat(keyTsLatestMap).as("ts latest map before").isEmpty();
  74 +
  75 + assertThat(helper.getTsByKey(KEY_SW_STATE, keyTsLatestMap, now)).as("getTsByKey").isEqualTo(now);
  76 +
  77 + assertThat(keyTsLatestMap).as("ts latest map after").hasSize(1);
  78 + assertThat(keyTsLatestMap.get(KEY_SW_STATE)).as("key present").isNotNull();
  79 + assertThat(keyTsLatestMap.get(KEY_SW_STATE).get()).as("ts in map by key").isEqualTo(now);
  80 + }
  81 +
  82 + @Test
  83 + void givenKeyAndLatestTsMapWithExistedKey_whenGetTsByKey_thenCallSwapOrIncrementMethod() {
  84 + keyTsLatestMap.put(KEY_SW_STATE, new AtomicLong());
  85 + keyTsLatestMap.put("other", new AtomicLong());
  86 + assertThat(keyTsLatestMap).as("ts latest map").hasSize(2);
  87 + willReturn(now).given(helper).compareAndSwapOrIncrementTsAtomically(any(AtomicLong.class), anyLong());
  88 +
  89 + assertThat(helper.getTsByKey(KEY_SW_STATE, keyTsLatestMap, now)).as("getTsByKey").isEqualTo(now);
  90 +
  91 + verify(helper, times(1)).compareAndSwapOrIncrementTsAtomically(keyTsLatestMap.get(KEY_SW_STATE), now);
  92 + verify(helper, times(1)).compareAndSwapOrIncrementTsAtomically(any(AtomicLong.class), anyLong());
  93 + }
  94 +
  95 + @Test
  96 + void givenMapWithTsValueLessThanNow_whenCompareAndSwapOrIncrementTsAtomically_thenReturnNow() {
  97 + keyTsLatestMap.put(KEY_SW_STATE, new AtomicLong(now - 1));
  98 + assertThat(helper.compareAndSwapOrIncrementTsAtomically(keyTsLatestMap.get(KEY_SW_STATE), now)).isEqualTo(now);
  99 + }
  100 +
  101 + @Test
  102 + void givenMapWithTsValueEqualsNow_whenCompareAndSwapOrIncrementTsAtomically_thenReturnNowIncremented() {
  103 + keyTsLatestMap.put(KEY_SW_STATE, new AtomicLong(now));
  104 + assertThat(helper.compareAndSwapOrIncrementTsAtomically(keyTsLatestMap.get(KEY_SW_STATE), now)).isEqualTo(now + 1);
  105 + }
  106 +
  107 + @Test
  108 + void givenMapWithTsValueGreaterThanNow_whenCompareAndSwapOrIncrementTsAtomically_thenReturnGreaterThanNowIncremented() {
  109 + final long nextHourTs = now + TimeUnit.HOURS.toMillis(1);
  110 + keyTsLatestMap.put(KEY_SW_STATE, new AtomicLong(nextHourTs));
  111 + assertThat(helper.compareAndSwapOrIncrementTsAtomically(keyTsLatestMap.get(KEY_SW_STATE), now)).isEqualTo(nextHourTs + 1);
  112 + }
  113 +
  114 +}
\ No newline at end of file
... ...
... ... @@ -14,21 +14,6 @@
14 14 * limitations under the License.
15 15 */
16 16 package org.thingsboard.server.transport.lwm2m.server;
17   -/**
18   - * Copyright © 2016-2020 The Thingsboard Authors
19   - * <p>
20   - * Licensed under the Apache License, Version 2.0 (the "License");
21   - * you may not use this file except in compliance with the License.
22   - * You may obtain a copy of the License at
23   - * <p>
24   - * http://www.apache.org/licenses/LICENSE-2.0
25   - * <p>
26   - * Unless required by applicable law or agreed to in writing, software
27   - * distributed under the License is distributed on an "AS IS" BASIS,
28   - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
29   - * See the License for the specific language governing permissions and
30   - * limitations under the License.
31   - */
32 17
33 18 import lombok.RequiredArgsConstructor;
34 19 import lombok.extern.slf4j.Slf4j;
... ... @@ -37,10 +22,7 @@ import org.eclipse.leshan.core.model.DefaultDDFFileValidator;
37 22 import org.eclipse.leshan.core.model.InvalidDDFFileException;
38 23 import org.eclipse.leshan.core.model.ObjectModel;
39 24 import org.eclipse.leshan.core.model.ResourceModel;
40   -import org.eclipse.leshan.core.node.LwM2mPath;
41   -import org.eclipse.leshan.core.node.LwM2mResource;
42 25 import org.eclipse.leshan.core.node.codec.CodecException;
43   -import org.eclipse.leshan.core.request.ContentFormat;
44 26 import org.springframework.stereotype.Component;
45 27 import org.thingsboard.server.common.transport.TransportServiceCallback;
46 28 import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse;
... ... @@ -49,19 +31,17 @@ import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg;
49 31 import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg;
50 32 import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto;
51 33 import org.thingsboard.server.queue.util.TbLwM2mTransportComponent;
52   -import org.thingsboard.server.transport.lwm2m.server.adaptors.LwM2MJsonAdaptor;
53   -import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient;
54   -import org.thingsboard.server.transport.lwm2m.server.client.ResourceValue;
55 34
  35 +import javax.annotation.Nonnull;
  36 +import javax.annotation.Nullable;
56 37 import java.io.ByteArrayInputStream;
57 38 import java.io.IOException;
58 39 import java.util.ArrayList;
59 40 import java.util.List;
60   -import java.util.concurrent.TimeUnit;
61   -import java.util.concurrent.atomic.AtomicInteger;
  41 +import java.util.concurrent.ConcurrentMap;
  42 +import java.util.concurrent.atomic.AtomicLong;
62 43
63 44 import static org.thingsboard.server.gen.transport.TransportProtos.KeyValueType.BOOLEAN_V;
64   -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.fromVersionedIdToObjectId;
65 45
66 46 @Slf4j
67 47 @Component
... ... @@ -70,11 +50,6 @@ import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.f
70 50 public class LwM2mTransportServerHelper {
71 51
72 52 private final LwM2mTransportContext context;
73   - private final AtomicInteger atomicTs = new AtomicInteger(0);
74   -
75   - public long getTS() {
76   - return TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) * 1000L + (atomicTs.getAndIncrement() % 1000);
77   - }
78 53
79 54 public void sendParametersOnThingsboardAttribute(List<TransportProtos.KeyValueProto> result, SessionInfoProto sessionInfo) {
80 55 PostAttributeMsg.Builder request = PostAttributeMsg.newBuilder();
... ... @@ -83,16 +58,67 @@ public class LwM2mTransportServerHelper {
83 58 context.getTransportService().process(sessionInfo, postAttributeMsg, TransportServiceCallback.EMPTY);
84 59 }
85 60
86   - public void sendParametersOnThingsboardTelemetry(List<TransportProtos.KeyValueProto> result, SessionInfoProto sessionInfo) {
87   - PostTelemetryMsg.Builder request = PostTelemetryMsg.newBuilder();
88   - TransportProtos.TsKvListProto.Builder builder = TransportProtos.TsKvListProto.newBuilder();
89   - builder.setTs(this.getTS());
90   - builder.addAllKv(result);
91   - request.addTsKvList(builder.build());
92   - PostTelemetryMsg postTelemetryMsg = request.build();
  61 + public void sendParametersOnThingsboardTelemetry(List<TransportProtos.KeyValueProto> kvList, SessionInfoProto sessionInfo) {
  62 + sendParametersOnThingsboardTelemetry(kvList, sessionInfo, null);
  63 + }
  64 +
  65 + public void sendParametersOnThingsboardTelemetry(List<TransportProtos.KeyValueProto> kvList, SessionInfoProto sessionInfo, @Nullable ConcurrentMap<String, AtomicLong> keyTsLatestMap) {
  66 + TransportProtos.TsKvListProto tsKvList = toTsKvList(kvList, keyTsLatestMap);
  67 +
  68 + PostTelemetryMsg postTelemetryMsg = PostTelemetryMsg.newBuilder()
  69 + .addTsKvList(tsKvList)
  70 + .build();
  71 +
93 72 context.getTransportService().process(sessionInfo, postTelemetryMsg, TransportServiceCallback.EMPTY);
94 73 }
95 74
  75 + TransportProtos.TsKvListProto toTsKvList(List<TransportProtos.KeyValueProto> kvList, ConcurrentMap<String, AtomicLong> keyTsLatestMap) {
  76 + return TransportProtos.TsKvListProto.newBuilder()
  77 + .setTs(getTs(kvList, keyTsLatestMap))
  78 + .addAllKv(kvList)
  79 + .build();
  80 + }
  81 +
  82 + long getTs(List<TransportProtos.KeyValueProto> kvList, ConcurrentMap<String, AtomicLong> keyTsLatestMap) {
  83 + if (keyTsLatestMap == null || kvList == null || kvList.isEmpty()) {
  84 + return getCurrentTimeMillis();
  85 + }
  86 +
  87 + return getTsByKey(kvList.get(0).getKey(), keyTsLatestMap, getCurrentTimeMillis());
  88 + }
  89 +
  90 + long getTsByKey(@Nonnull String key, @Nonnull ConcurrentMap<String, AtomicLong> keyTsLatestMap, final long tsNow) {
  91 + AtomicLong tsLatestAtomic = keyTsLatestMap.putIfAbsent(key, new AtomicLong(tsNow));
  92 + if (tsLatestAtomic == null) {
  93 + return tsNow; // it is a first known timestamp for this key. return as the latest
  94 + }
  95 +
  96 + return compareAndSwapOrIncrementTsAtomically(tsLatestAtomic, tsNow);
  97 + }
  98 +
  99 + /**
  100 + * This algorithm is sensitive to wall-clock time shift.
  101 + * Once time have shifted *backward*, the latest ts never came back.
  102 + * Ts latest will be incremented until current time overtake the latest ts.
  103 + * In normal environment without race conditions method will return current ts (wall-clock)
  104 + * */
  105 + long compareAndSwapOrIncrementTsAtomically(AtomicLong tsLatestAtomic, final long tsNow) {
  106 + long tsLatest;
  107 + while ((tsLatest = tsLatestAtomic.get()) < tsNow) {
  108 + if (tsLatestAtomic.compareAndSet(tsLatest, tsNow)) {
  109 + return tsNow; //swap successful
  110 + }
  111 + }
  112 + return tsLatestAtomic.incrementAndGet(); //return next ms
  113 + }
  114 +
  115 + /**
  116 + * For the test ability to mock system timer
  117 + * */
  118 + long getCurrentTimeMillis() {
  119 + return System.currentTimeMillis();
  120 + }
  121 +
96 122 /**
97 123 * @return - sessionInfo after access connect client
98 124 */
... ...
... ... @@ -50,8 +50,10 @@ import java.util.Optional;
50 50 import java.util.Set;
51 51 import java.util.UUID;
52 52 import java.util.concurrent.ConcurrentHashMap;
  53 +import java.util.concurrent.ConcurrentMap;
53 54 import java.util.concurrent.Future;
54 55 import java.util.concurrent.atomic.AtomicInteger;
  56 +import java.util.concurrent.atomic.AtomicLong;
55 57 import java.util.concurrent.locks.Lock;
56 58 import java.util.concurrent.locks.ReentrantLock;
57 59 import java.util.stream.Collectors;
... ... @@ -80,6 +82,8 @@ public class LwM2mClient implements Serializable {
80 82 private final Map<String, ResourceValue> resources;
81 83 @Getter
82 84 private final Map<String, TsKvProto> sharedAttributes;
  85 + @Getter
  86 + private final ConcurrentMap<String, AtomicLong> keyTsLatestMap;
83 87
84 88 @Getter
85 89 private TenantId tenantId;
... ... @@ -126,6 +130,7 @@ public class LwM2mClient implements Serializable {
126 130 this.endpoint = endpoint;
127 131 this.sharedAttributes = new ConcurrentHashMap<>();
128 132 this.resources = new ConcurrentHashMap<>();
  133 + this.keyTsLatestMap = new ConcurrentHashMap<>();
129 134 this.state = LwM2MClientState.CREATED;
130 135 this.lock = new ReentrantLock();
131 136 this.retryAttempts = new AtomicInteger(0);
... ...
... ... @@ -39,7 +39,7 @@ public class DefaultLwM2MTelemetryLogService implements LwM2MTelemetryLogService
39 39 if (logMsg.length() > 1024) {
40 40 logMsg = logMsg.substring(0, 1024);
41 41 }
42   - this.helper.sendParametersOnThingsboardTelemetry(this.helper.getKvStringtoThingsboard(LOG_LWM2M_TELEMETRY, logMsg), client.getSession());
  42 + this.helper.sendParametersOnThingsboardTelemetry(this.helper.getKvStringtoThingsboard(LOG_LWM2M_TELEMETRY, logMsg), client.getSession(), client.getKeyTsLatestMap());
43 43 }
44 44 }
45 45
... ...
... ... @@ -602,7 +602,7 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl
602 602 kvProto = TransportProtos.KeyValueProto.newBuilder().setKey(LOG_LWM2M_TELEMETRY);
603 603 kvProto.setType(TransportProtos.KeyValueType.STRING_V).setStringV(log);
604 604 result.add(kvProto.build());
605   - helper.sendParametersOnThingsboardTelemetry(result, client.getSession());
  605 + helper.sendParametersOnThingsboardTelemetry(result, client.getSession(), client.getKeyTsLatestMap());
606 606 }
607 607
608 608 private static Optional<OtaPackageUpdateStatus> toOtaPackageUpdateStatus(FirmwareUpdateResult fwUpdateResult) {
... ...
... ... @@ -19,12 +19,14 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
19 19 import lombok.Data;
20 20 import lombok.EqualsAndHashCode;
21 21 import lombok.NoArgsConstructor;
  22 +import lombok.ToString;
22 23 import org.thingsboard.server.common.data.ota.OtaPackageType;
23 24 import org.thingsboard.server.transport.lwm2m.server.ota.LwM2MClientOtaInfo;
24 25
25 26 @Data
26 27 @EqualsAndHashCode(callSuper = true)
27 28 @NoArgsConstructor
  29 +@ToString(callSuper = true)
28 30 public class LwM2MClientFwOtaInfo extends LwM2MClientOtaInfo<LwM2MFirmwareUpdateStrategy, FirmwareUpdateState, FirmwareUpdateResult> {
29 31
30 32 private Integer deliveryMethod;
... ...
... ... @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
19 19 import lombok.Data;
20 20 import lombok.EqualsAndHashCode;
21 21 import lombok.NoArgsConstructor;
  22 +import lombok.ToString;
22 23 import org.thingsboard.server.common.data.StringUtils;
23 24 import org.thingsboard.server.common.data.ota.OtaPackageType;
24 25 import org.thingsboard.server.transport.lwm2m.server.ota.LwM2MClientOtaInfo;
... ... @@ -29,6 +30,7 @@ import org.thingsboard.server.transport.lwm2m.server.ota.firmware.LwM2MFirmwareU
29 30 @Data
30 31 @EqualsAndHashCode(callSuper = true)
31 32 @NoArgsConstructor
  33 +@ToString(callSuper = true)
32 34 public class LwM2MClientSwOtaInfo extends LwM2MClientOtaInfo<LwM2MSoftwareUpdateStrategy, SoftwareUpdateState, SoftwareUpdateResult> {
33 35
34 36 public LwM2MClientSwOtaInfo(String endpoint, String baseUrl, LwM2MSoftwareUpdateStrategy strategy) {
... ...