Commit 89ffda9a78c79f01eafba8b8eafb1fa9f1bc79fd
Merge branch 'develop/2.0' of github.com:thingsboard/thingsboard into develop/2.0
Showing
25 changed files
with
427 additions
and
88 deletions
... | ... | @@ -23,6 +23,7 @@ import org.springframework.beans.factory.annotation.Value; |
23 | 23 | import org.springframework.stereotype.Service; |
24 | 24 | import org.thingsboard.server.common.data.id.TenantId; |
25 | 25 | import org.thingsboard.server.common.msg.TbMsg; |
26 | +import org.thingsboard.server.common.transport.quota.tenant.TenantQuotaService; | |
26 | 27 | import org.thingsboard.server.dao.queue.MsgQueue; |
27 | 28 | |
28 | 29 | import javax.annotation.PostConstruct; |
... | ... | @@ -48,6 +49,9 @@ public class DefaultMsgQueueService implements MsgQueueService { |
48 | 49 | @Autowired |
49 | 50 | private MsgQueue msgQueue; |
50 | 51 | |
52 | + @Autowired | |
53 | + private TenantQuotaService quotaService; | |
54 | + | |
51 | 55 | private ScheduledExecutorService cleanupExecutor; |
52 | 56 | |
53 | 57 | private Map<TenantId, AtomicLong> pendingCountPerTenant = new ConcurrentHashMap<>(); |
... | ... | @@ -70,6 +74,11 @@ public class DefaultMsgQueueService implements MsgQueueService { |
70 | 74 | |
71 | 75 | @Override |
72 | 76 | public ListenableFuture<Void> put(TenantId tenantId, TbMsg msg, UUID nodeId, long clusterPartition) { |
77 | + if(quotaService.isQuotaExceeded(tenantId.getId().toString())) { | |
78 | + log.warn("Tenant TbMsg Quota exceeded for [{}:{}] . Reject", tenantId.getId()); | |
79 | + return Futures.immediateFailedFuture(new RuntimeException("Tenant TbMsg Quota exceeded")); | |
80 | + } | |
81 | + | |
73 | 82 | AtomicLong pendingMsgCount = pendingCountPerTenant.computeIfAbsent(tenantId, key -> new AtomicLong()); |
74 | 83 | if (pendingMsgCount.incrementAndGet() < queueMaxSize) { |
75 | 84 | return msgQueue.put(tenantId, msg, nodeId, clusterPartition); | ... | ... |
... | ... | @@ -131,9 +131,28 @@ quota: |
131 | 131 | whitelist: "${QUOTA_HOST_WHITELIST:localhost,127.0.0.1}" |
132 | 132 | # Array of blacklist hosts |
133 | 133 | blacklist: "${QUOTA_HOST_BLACKLIST:}" |
134 | - log: | |
135 | - topSize: 10 | |
136 | - intervalMin: 2 | |
134 | + log: | |
135 | + topSize: 10 | |
136 | + intervalMin: 2 | |
137 | + rule: | |
138 | + tenant: | |
139 | + # Max allowed number of API requests in interval for single tenant | |
140 | + limit: "${QUOTA_TENANT_LIMIT:100000}" | |
141 | + # Interval duration | |
142 | + intervalMs: "${QUOTA_TENANT_INTERVAL_MS:60000}" | |
143 | + # Maximum silence duration for tenant after which Tenant removed from QuotaService. Must be bigger than intervalMs | |
144 | + ttlMs: "${QUOTA_TENANT_TTL_MS:60000}" | |
145 | + # Interval for scheduled task that cleans expired records. TTL is used for expiring | |
146 | + cleanPeriodMs: "${QUOTA_TENANT_CLEAN_PERIOD_MS:300000}" | |
147 | + # Enable Host API Limits | |
148 | + enabled: "${QUOTA_TENANT_ENABLED:false}" | |
149 | + # Array of whitelist tenants | |
150 | + whitelist: "${QUOTA_TENANT_WHITELIST:}" | |
151 | + # Array of blacklist tenants | |
152 | + blacklist: "${QUOTA_HOST_BLACKLIST:}" | |
153 | + log: | |
154 | + topSize: 10 | |
155 | + intervalMin: 2 | |
137 | 156 | |
138 | 157 | database: |
139 | 158 | type: "${DATABASE_TYPE:sql}" # cassandra OR sql | ... | ... |
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/AbstractQuotaService.java
renamed from
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/HostRequestsQuotaService.java
... | ... | @@ -15,33 +15,24 @@ |
15 | 15 | */ |
16 | 16 | package org.thingsboard.server.common.transport.quota; |
17 | 17 | |
18 | -import lombok.extern.slf4j.Slf4j; | |
19 | -import org.springframework.beans.factory.annotation.Value; | |
20 | -import org.springframework.stereotype.Service; | |
21 | -import org.thingsboard.server.common.transport.quota.inmemory.HostRequestIntervalRegistry; | |
22 | 18 | import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner; |
23 | 19 | import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger; |
20 | +import org.thingsboard.server.common.transport.quota.inmemory.KeyBasedIntervalRegistry; | |
24 | 21 | |
25 | 22 | import javax.annotation.PostConstruct; |
26 | 23 | import javax.annotation.PreDestroy; |
27 | 24 | |
28 | -/** | |
29 | - * @author Vitaliy Paromskiy | |
30 | - * @version 1.0 | |
31 | - */ | |
32 | -@Service | |
33 | -@Slf4j | |
34 | -public class HostRequestsQuotaService implements QuotaService { | |
25 | +public class AbstractQuotaService implements QuotaService { | |
35 | 26 | |
36 | - private final HostRequestIntervalRegistry requestRegistry; | |
37 | - private final HostRequestLimitPolicy requestsPolicy; | |
27 | + private final KeyBasedIntervalRegistry requestRegistry; | |
28 | + private final RequestLimitPolicy requestsPolicy; | |
38 | 29 | private final IntervalRegistryCleaner registryCleaner; |
39 | 30 | private final IntervalRegistryLogger registryLogger; |
40 | 31 | private final boolean enabled; |
41 | 32 | |
42 | - public HostRequestsQuotaService(HostRequestIntervalRegistry requestRegistry, HostRequestLimitPolicy requestsPolicy, | |
33 | + public AbstractQuotaService(KeyBasedIntervalRegistry requestRegistry, RequestLimitPolicy requestsPolicy, | |
43 | 34 | IntervalRegistryCleaner registryCleaner, IntervalRegistryLogger registryLogger, |
44 | - @Value("${quota.host.enabled}") boolean enabled) { | |
35 | + boolean enabled) { | |
45 | 36 | this.requestRegistry = requestRegistry; |
46 | 37 | this.requestsPolicy = requestsPolicy; |
47 | 38 | this.registryCleaner = registryCleaner; | ... | ... |
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/RequestLimitPolicy.java
0 → 100644
1 | +/** | |
2 | + * Copyright © 2016-2018 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.common.transport.quota; | |
17 | + | |
18 | + | |
19 | +public abstract class RequestLimitPolicy { | |
20 | + | |
21 | + private final long limit; | |
22 | + | |
23 | + public RequestLimitPolicy(long limit) { | |
24 | + this.limit = limit; | |
25 | + } | |
26 | + | |
27 | + public boolean isValid(long currentValue) { | |
28 | + return currentValue <= limit; | |
29 | + } | |
30 | +} | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2018 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.common.transport.quota.host; | |
17 | + | |
18 | +import org.springframework.beans.factory.annotation.Value; | |
19 | +import org.springframework.stereotype.Component; | |
20 | +import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner; | |
21 | + | |
22 | +@Component | |
23 | +public class HostIntervalRegistryCleaner extends IntervalRegistryCleaner { | |
24 | + | |
25 | + public HostIntervalRegistryCleaner(HostRequestIntervalRegistry intervalRegistry, | |
26 | + @Value("${quota.host.cleanPeriodMs}") long cleanPeriodMs) { | |
27 | + super(intervalRegistry, cleanPeriodMs); | |
28 | + } | |
29 | +} | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2018 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.common.transport.quota.host; | |
17 | + | |
18 | +import lombok.extern.slf4j.Slf4j; | |
19 | +import org.springframework.beans.factory.annotation.Value; | |
20 | +import org.springframework.stereotype.Component; | |
21 | +import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger; | |
22 | + | |
23 | +import java.util.Map; | |
24 | +import java.util.concurrent.TimeUnit; | |
25 | + | |
26 | +@Component | |
27 | +@Slf4j | |
28 | +public class HostIntervalRegistryLogger extends IntervalRegistryLogger { | |
29 | + | |
30 | + private final long logIntervalMin; | |
31 | + | |
32 | + public HostIntervalRegistryLogger(@Value("${quota.host.log.topSize}") int topSize, | |
33 | + @Value("${quota.host.log.intervalMin}") long logIntervalMin, | |
34 | + HostRequestIntervalRegistry intervalRegistry) { | |
35 | + super(topSize, logIntervalMin, intervalRegistry); | |
36 | + this.logIntervalMin = logIntervalMin; | |
37 | + } | |
38 | + | |
39 | + protected void log(Map<String, Long> top, int uniqHosts, long requestsCount) { | |
40 | + long rps = requestsCount / TimeUnit.MINUTES.toSeconds(logIntervalMin); | |
41 | + StringBuilder builder = new StringBuilder("Quota Statistic : "); | |
42 | + builder.append("uniqHosts : ").append(uniqHosts).append("; "); | |
43 | + builder.append("requestsCount : ").append(requestsCount).append("; "); | |
44 | + builder.append("RPS : ").append(rps).append(" "); | |
45 | + builder.append("top -> "); | |
46 | + for (Map.Entry<String, Long> host : top.entrySet()) { | |
47 | + builder.append(host.getKey()).append(" : ").append(host.getValue()).append("; "); | |
48 | + } | |
49 | + | |
50 | + log.info(builder.toString()); | |
51 | + } | |
52 | +} | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2018 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.common.transport.quota.host; | |
17 | + | |
18 | +import lombok.extern.slf4j.Slf4j; | |
19 | +import org.springframework.beans.factory.annotation.Value; | |
20 | +import org.springframework.stereotype.Component; | |
21 | +import org.thingsboard.server.common.transport.quota.inmemory.KeyBasedIntervalRegistry; | |
22 | + | |
23 | +/** | |
24 | + * @author Vitaliy Paromskiy | |
25 | + * @version 1.0 | |
26 | + */ | |
27 | +@Component | |
28 | +@Slf4j | |
29 | +public class HostRequestIntervalRegistry extends KeyBasedIntervalRegistry { | |
30 | + | |
31 | + public HostRequestIntervalRegistry(@Value("${quota.host.intervalMs}") long intervalDurationMs, | |
32 | + @Value("${quota.host.ttlMs}") long ttlMs, | |
33 | + @Value("${quota.host.whitelist}") String whiteList, | |
34 | + @Value("${quota.host.blacklist}") String blackList) { | |
35 | + super(intervalDurationMs, ttlMs, whiteList, blackList, "host"); | |
36 | + } | |
37 | +} | ... | ... |
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/host/HostRequestLimitPolicy.java
renamed from
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/HostRequestLimitPolicy.java
... | ... | @@ -13,26 +13,21 @@ |
13 | 13 | * See the License for the specific language governing permissions and |
14 | 14 | * limitations under the License. |
15 | 15 | */ |
16 | -package org.thingsboard.server.common.transport.quota; | |
16 | +package org.thingsboard.server.common.transport.quota.host; | |
17 | 17 | |
18 | 18 | import org.springframework.beans.factory.annotation.Value; |
19 | 19 | import org.springframework.stereotype.Component; |
20 | +import org.thingsboard.server.common.transport.quota.RequestLimitPolicy; | |
20 | 21 | |
21 | 22 | /** |
22 | 23 | * @author Vitaliy Paromskiy |
23 | 24 | * @version 1.0 |
24 | 25 | */ |
25 | 26 | @Component |
26 | -public class HostRequestLimitPolicy { | |
27 | - | |
28 | - private final long limit; | |
27 | +public class HostRequestLimitPolicy extends RequestLimitPolicy { | |
29 | 28 | |
30 | 29 | public HostRequestLimitPolicy(@Value("${quota.host.limit}") long limit) { |
31 | - this.limit = limit; | |
32 | - } | |
33 | - | |
34 | - public boolean isValid(long currentValue) { | |
35 | - return currentValue <= limit; | |
30 | + super(limit); | |
36 | 31 | } |
37 | 32 | |
38 | 33 | } | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2018 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.common.transport.quota.host; | |
17 | + | |
18 | +import lombok.extern.slf4j.Slf4j; | |
19 | +import org.springframework.beans.factory.annotation.Value; | |
20 | +import org.springframework.stereotype.Service; | |
21 | +import org.thingsboard.server.common.transport.quota.AbstractQuotaService; | |
22 | + | |
23 | +/** | |
24 | + * @author Vitaliy Paromskiy | |
25 | + * @version 1.0 | |
26 | + */ | |
27 | +@Service | |
28 | +@Slf4j | |
29 | +public class HostRequestsQuotaService extends AbstractQuotaService { | |
30 | + | |
31 | + public HostRequestsQuotaService(HostRequestIntervalRegistry requestRegistry, HostRequestLimitPolicy requestsPolicy, | |
32 | + HostIntervalRegistryCleaner registryCleaner, HostIntervalRegistryLogger registryLogger, | |
33 | + @Value("${quota.host.enabled}") boolean enabled) { | |
34 | + super(requestRegistry, requestsPolicy, registryCleaner, registryLogger, enabled); | |
35 | + } | |
36 | + | |
37 | +} | ... | ... |
... | ... | @@ -16,10 +16,7 @@ |
16 | 16 | package org.thingsboard.server.common.transport.quota.inmemory; |
17 | 17 | |
18 | 18 | import lombok.extern.slf4j.Slf4j; |
19 | -import org.springframework.beans.factory.annotation.Value; | |
20 | -import org.springframework.stereotype.Component; | |
21 | 19 | |
22 | -import javax.annotation.PreDestroy; | |
23 | 20 | import java.util.concurrent.Executors; |
24 | 21 | import java.util.concurrent.ScheduledExecutorService; |
25 | 22 | import java.util.concurrent.TimeUnit; |
... | ... | @@ -28,15 +25,14 @@ import java.util.concurrent.TimeUnit; |
28 | 25 | * @author Vitaliy Paromskiy |
29 | 26 | * @version 1.0 |
30 | 27 | */ |
31 | -@Component | |
32 | 28 | @Slf4j |
33 | -public class IntervalRegistryCleaner { | |
29 | +public abstract class IntervalRegistryCleaner { | |
34 | 30 | |
35 | - private final HostRequestIntervalRegistry intervalRegistry; | |
31 | + private final KeyBasedIntervalRegistry intervalRegistry; | |
36 | 32 | private final long cleanPeriodMs; |
37 | 33 | private ScheduledExecutorService executor; |
38 | 34 | |
39 | - public IntervalRegistryCleaner(HostRequestIntervalRegistry intervalRegistry, @Value("${quota.host.cleanPeriodMs}") long cleanPeriodMs) { | |
35 | + public IntervalRegistryCleaner(KeyBasedIntervalRegistry intervalRegistry, long cleanPeriodMs) { | |
40 | 36 | this.intervalRegistry = intervalRegistry; |
41 | 37 | this.cleanPeriodMs = cleanPeriodMs; |
42 | 38 | } | ... | ... |
... | ... | @@ -17,8 +17,6 @@ package org.thingsboard.server.common.transport.quota.inmemory; |
17 | 17 | |
18 | 18 | import com.google.common.collect.MinMaxPriorityQueue; |
19 | 19 | import lombok.extern.slf4j.Slf4j; |
20 | -import org.springframework.beans.factory.annotation.Value; | |
21 | -import org.springframework.stereotype.Component; | |
22 | 20 | |
23 | 21 | import java.util.Comparator; |
24 | 22 | import java.util.Map; |
... | ... | @@ -32,17 +30,15 @@ import java.util.stream.Collectors; |
32 | 30 | * @author Vitaliy Paromskiy |
33 | 31 | * @version 1.0 |
34 | 32 | */ |
35 | -@Component | |
36 | 33 | @Slf4j |
37 | -public class IntervalRegistryLogger { | |
34 | +public abstract class IntervalRegistryLogger { | |
38 | 35 | |
39 | 36 | private final int topSize; |
40 | - private final HostRequestIntervalRegistry intervalRegistry; | |
37 | + private final KeyBasedIntervalRegistry intervalRegistry; | |
41 | 38 | private final long logIntervalMin; |
42 | 39 | private ScheduledExecutorService executor; |
43 | 40 | |
44 | - public IntervalRegistryLogger(@Value("${quota.log.topSize}") int topSize, @Value("${quota.log.intervalMin}") long logIntervalMin, | |
45 | - HostRequestIntervalRegistry intervalRegistry) { | |
41 | + public IntervalRegistryLogger(int topSize, long logIntervalMin, KeyBasedIntervalRegistry intervalRegistry) { | |
46 | 42 | this.topSize = topSize; |
47 | 43 | this.logIntervalMin = logIntervalMin; |
48 | 44 | this.intervalRegistry = intervalRegistry; |
... | ... | @@ -79,17 +75,5 @@ public class IntervalRegistryLogger { |
79 | 75 | return topQueue.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); |
80 | 76 | } |
81 | 77 | |
82 | - private void log(Map<String, Long> top, int uniqHosts, long requestsCount) { | |
83 | - long rps = requestsCount / TimeUnit.MINUTES.toSeconds(logIntervalMin); | |
84 | - StringBuilder builder = new StringBuilder("Quota Statistic : "); | |
85 | - builder.append("uniqHosts : ").append(uniqHosts).append("; "); | |
86 | - builder.append("requestsCount : ").append(requestsCount).append("; "); | |
87 | - builder.append("RPS : ").append(rps).append(" "); | |
88 | - builder.append("top -> "); | |
89 | - for (Map.Entry<String, Long> host : top.entrySet()) { | |
90 | - builder.append(host.getKey()).append(" : ").append(host.getValue()).append("; "); | |
91 | - } | |
92 | - | |
93 | - log.info(builder.toString()); | |
94 | - } | |
78 | + protected abstract void log(Map<String, Long> top, int uniqHosts, long requestsCount); | |
95 | 79 | } | ... | ... |
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/KeyBasedIntervalRegistry.java
renamed from
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/HostRequestIntervalRegistry.java
... | ... | @@ -18,22 +18,14 @@ package org.thingsboard.server.common.transport.quota.inmemory; |
18 | 18 | import com.google.common.collect.Sets; |
19 | 19 | import lombok.extern.slf4j.Slf4j; |
20 | 20 | import org.apache.commons.lang3.StringUtils; |
21 | -import org.springframework.beans.factory.annotation.Value; | |
22 | -import org.springframework.stereotype.Component; | |
23 | 21 | |
24 | -import javax.annotation.PostConstruct; | |
25 | 22 | import java.util.Map; |
26 | 23 | import java.util.Set; |
27 | 24 | import java.util.concurrent.ConcurrentHashMap; |
28 | 25 | import java.util.stream.Collectors; |
29 | 26 | |
30 | -/** | |
31 | - * @author Vitaliy Paromskiy | |
32 | - * @version 1.0 | |
33 | - */ | |
34 | -@Component | |
35 | 27 | @Slf4j |
36 | -public class HostRequestIntervalRegistry { | |
28 | +public abstract class KeyBasedIntervalRegistry { | |
37 | 29 | |
38 | 30 | private final Map<String, IntervalCount> hostCounts = new ConcurrentHashMap<>(); |
39 | 31 | private final long intervalDurationMs; |
... | ... | @@ -41,23 +33,20 @@ public class HostRequestIntervalRegistry { |
41 | 33 | private final Set<String> whiteList; |
42 | 34 | private final Set<String> blackList; |
43 | 35 | |
44 | - public HostRequestIntervalRegistry(@Value("${quota.host.intervalMs}") long intervalDurationMs, | |
45 | - @Value("${quota.host.ttlMs}") long ttlMs, | |
46 | - @Value("${quota.host.whitelist}") String whiteList, | |
47 | - @Value("${quota.host.blacklist}") String blackList) { | |
36 | + public KeyBasedIntervalRegistry(long intervalDurationMs, long ttlMs, String whiteList, String blackList, String name) { | |
48 | 37 | this.intervalDurationMs = intervalDurationMs; |
49 | 38 | this.ttlMs = ttlMs; |
50 | 39 | this.whiteList = Sets.newHashSet(StringUtils.split(whiteList, ',')); |
51 | 40 | this.blackList = Sets.newHashSet(StringUtils.split(blackList, ',')); |
41 | + validate(name); | |
52 | 42 | } |
53 | 43 | |
54 | - @PostConstruct | |
55 | - public void init() { | |
44 | + private void validate(String name) { | |
56 | 45 | if (ttlMs < intervalDurationMs) { |
57 | - log.warn("TTL for IntervalRegistry [{}] smaller than interval duration [{}]", ttlMs, intervalDurationMs); | |
46 | + log.warn("TTL for {} IntervalRegistry [{}] smaller than interval duration [{}]", name, ttlMs, intervalDurationMs); | |
58 | 47 | } |
59 | - log.info("Start Host Quota Service with whitelist {}", whiteList); | |
60 | - log.info("Start Host Quota Service with blacklist {}", blackList); | |
48 | + log.info("Start {} KeyBasedIntervalRegistry with whitelist {}", name, whiteList); | |
49 | + log.info("Start {} KeyBasedIntervalRegistry with blacklist {}", name, blackList); | |
61 | 50 | } |
62 | 51 | |
63 | 52 | public long tick(String clientHostId) { | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2018 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.common.transport.quota.tenant; | |
17 | + | |
18 | +import org.springframework.beans.factory.annotation.Value; | |
19 | +import org.springframework.stereotype.Component; | |
20 | +import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner; | |
21 | + | |
22 | +@Component | |
23 | +public class TenantIntervalRegistryCleaner extends IntervalRegistryCleaner { | |
24 | + | |
25 | + public TenantIntervalRegistryCleaner(TenantMsgsIntervalRegistry intervalRegistry, | |
26 | + @Value("${quota.rule.tenant.cleanPeriodMs}") long cleanPeriodMs) { | |
27 | + super(intervalRegistry, cleanPeriodMs); | |
28 | + } | |
29 | +} | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2018 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.common.transport.quota.tenant; | |
17 | + | |
18 | +import lombok.extern.slf4j.Slf4j; | |
19 | +import org.springframework.beans.factory.annotation.Value; | |
20 | +import org.springframework.stereotype.Component; | |
21 | +import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger; | |
22 | + | |
23 | +import java.util.Map; | |
24 | +import java.util.concurrent.TimeUnit; | |
25 | + | |
26 | +@Slf4j | |
27 | +@Component | |
28 | +public class TenantIntervalRegistryLogger extends IntervalRegistryLogger { | |
29 | + | |
30 | + private final long logIntervalMin; | |
31 | + | |
32 | + public TenantIntervalRegistryLogger(@Value("${quota.rule.tenant.log.topSize}") int topSize, | |
33 | + @Value("${quota.rule.tenant.log.intervalMin}") long logIntervalMin, | |
34 | + TenantMsgsIntervalRegistry intervalRegistry) { | |
35 | + super(topSize, logIntervalMin, intervalRegistry); | |
36 | + this.logIntervalMin = logIntervalMin; | |
37 | + } | |
38 | + | |
39 | + protected void log(Map<String, Long> top, int uniqHosts, long requestsCount) { | |
40 | + long rps = requestsCount / TimeUnit.MINUTES.toSeconds(logIntervalMin); | |
41 | + StringBuilder builder = new StringBuilder("Tenant Quota Statistic : "); | |
42 | + builder.append("uniqTenants : ").append(uniqHosts).append("; "); | |
43 | + builder.append("requestsCount : ").append(requestsCount).append("; "); | |
44 | + builder.append("RPS : ").append(rps).append(" "); | |
45 | + builder.append("top -> "); | |
46 | + for (Map.Entry<String, Long> host : top.entrySet()) { | |
47 | + builder.append(host.getKey()).append(" : ").append(host.getValue()).append("; "); | |
48 | + } | |
49 | + | |
50 | + log.info(builder.toString()); | |
51 | + } | |
52 | +} | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2018 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.common.transport.quota.tenant; | |
17 | + | |
18 | +import org.springframework.beans.factory.annotation.Value; | |
19 | +import org.springframework.stereotype.Component; | |
20 | +import org.thingsboard.server.common.transport.quota.inmemory.KeyBasedIntervalRegistry; | |
21 | + | |
22 | +@Component | |
23 | +public class TenantMsgsIntervalRegistry extends KeyBasedIntervalRegistry { | |
24 | + | |
25 | + public TenantMsgsIntervalRegistry(@Value("${quota.rule.tenant.intervalMs}") long intervalDurationMs, | |
26 | + @Value("${quota.rule.tenant.ttlMs}") long ttlMs, | |
27 | + @Value("${quota.rule.tenant.whitelist}") String whiteList, | |
28 | + @Value("${quota.rule.tenant.blacklist}") String blackList) { | |
29 | + super(intervalDurationMs, ttlMs, whiteList, blackList, "Rule Tenant"); | |
30 | + } | |
31 | +} | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2018 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.common.transport.quota.tenant; | |
17 | + | |
18 | +import org.springframework.beans.factory.annotation.Value; | |
19 | +import org.springframework.stereotype.Component; | |
20 | +import org.thingsboard.server.common.transport.quota.AbstractQuotaService; | |
21 | + | |
22 | +@Component | |
23 | +public class TenantQuotaService extends AbstractQuotaService { | |
24 | + | |
25 | + public TenantQuotaService(TenantMsgsIntervalRegistry requestRegistry, TenantRequestLimitPolicy requestsPolicy, | |
26 | + TenantIntervalRegistryCleaner registryCleaner, TenantIntervalRegistryLogger registryLogger, | |
27 | + @Value("${quota.rule.tenant.enabled}") boolean enabled) { | |
28 | + super(requestRegistry, requestsPolicy, registryCleaner, registryLogger, enabled); | |
29 | + } | |
30 | +} | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2018 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.common.transport.quota.tenant; | |
17 | + | |
18 | +import org.springframework.beans.factory.annotation.Value; | |
19 | +import org.springframework.stereotype.Component; | |
20 | +import org.thingsboard.server.common.transport.quota.RequestLimitPolicy; | |
21 | + | |
22 | +@Component | |
23 | +public class TenantRequestLimitPolicy extends RequestLimitPolicy { | |
24 | + | |
25 | + public TenantRequestLimitPolicy(@Value("${quota.rule.tenant.limit}") long limit) { | |
26 | + super(limit); | |
27 | + } | |
28 | +} | ... | ... |
... | ... | @@ -16,6 +16,7 @@ |
16 | 16 | package org.thingsboard.server.common.transport.quota; |
17 | 17 | |
18 | 18 | import org.junit.Test; |
19 | +import org.thingsboard.server.common.transport.quota.host.HostRequestLimitPolicy; | |
19 | 20 | |
20 | 21 | import static org.junit.Assert.assertFalse; |
21 | 22 | import static org.junit.Assert.assertTrue; | ... | ... |
... | ... | @@ -17,9 +17,7 @@ package org.thingsboard.server.common.transport.quota; |
17 | 17 | |
18 | 18 | import org.junit.Before; |
19 | 19 | import org.junit.Test; |
20 | -import org.thingsboard.server.common.transport.quota.inmemory.HostRequestIntervalRegistry; | |
21 | -import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner; | |
22 | -import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger; | |
20 | +import org.thingsboard.server.common.transport.quota.host.*; | |
23 | 21 | |
24 | 22 | import static org.junit.Assert.assertFalse; |
25 | 23 | import static org.junit.Assert.assertTrue; |
... | ... | @@ -35,8 +33,8 @@ public class HostRequestsQuotaServiceTest { |
35 | 33 | |
36 | 34 | private HostRequestIntervalRegistry requestRegistry = mock(HostRequestIntervalRegistry.class); |
37 | 35 | private HostRequestLimitPolicy requestsPolicy = mock(HostRequestLimitPolicy.class); |
38 | - private IntervalRegistryCleaner registryCleaner = mock(IntervalRegistryCleaner.class); | |
39 | - private IntervalRegistryLogger registryLogger = mock(IntervalRegistryLogger.class); | |
36 | + private HostIntervalRegistryCleaner registryCleaner = mock(HostIntervalRegistryCleaner.class); | |
37 | + private HostIntervalRegistryLogger registryLogger = mock(HostIntervalRegistryLogger.class); | |
40 | 38 | |
41 | 39 | @Before |
42 | 40 | public void init() { | ... | ... |
... | ... | @@ -15,11 +15,9 @@ |
15 | 15 | */ |
16 | 16 | package org.thingsboard.server.common.transport.quota.inmemory; |
17 | 17 | |
18 | -import com.google.common.collect.Sets; | |
19 | 18 | import org.junit.Before; |
20 | 19 | import org.junit.Test; |
21 | - | |
22 | -import java.util.Collections; | |
20 | +import org.thingsboard.server.common.transport.quota.host.HostRequestIntervalRegistry; | |
23 | 21 | |
24 | 22 | import static org.junit.Assert.assertEquals; |
25 | 23 | ... | ... |
... | ... | @@ -18,6 +18,8 @@ package org.thingsboard.server.common.transport.quota.inmemory; |
18 | 18 | import com.google.common.collect.ImmutableMap; |
19 | 19 | import org.junit.Before; |
20 | 20 | import org.junit.Test; |
21 | +import org.thingsboard.server.common.transport.quota.host.HostIntervalRegistryLogger; | |
22 | +import org.thingsboard.server.common.transport.quota.host.HostRequestIntervalRegistry; | |
21 | 23 | |
22 | 24 | import java.util.Collections; |
23 | 25 | import java.util.Map; |
... | ... | @@ -37,7 +39,7 @@ public class IntervalRegistryLoggerTest { |
37 | 39 | |
38 | 40 | @Before |
39 | 41 | public void init() { |
40 | - logger = new IntervalRegistryLogger(3, 10, requestRegistry); | |
42 | + logger = new HostIntervalRegistryLogger(3, 10, requestRegistry); | |
41 | 43 | } |
42 | 44 | |
43 | 45 | @Test | ... | ... |
... | ... | @@ -27,6 +27,7 @@ import org.springframework.stereotype.Service; |
27 | 27 | import org.thingsboard.server.common.transport.SessionMsgProcessor; |
28 | 28 | import org.thingsboard.server.common.transport.auth.DeviceAuthService; |
29 | 29 | import org.thingsboard.server.common.transport.quota.QuotaService; |
30 | +import org.thingsboard.server.common.transport.quota.host.HostRequestsQuotaService; | |
30 | 31 | import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; |
31 | 32 | |
32 | 33 | import javax.annotation.PostConstruct; |
... | ... | @@ -55,7 +56,7 @@ public class CoapTransportService { |
55 | 56 | private DeviceAuthService authService; |
56 | 57 | |
57 | 58 | @Autowired(required = false) |
58 | - private QuotaService quotaService; | |
59 | + private HostRequestsQuotaService quotaService; | |
59 | 60 | |
60 | 61 | |
61 | 62 | @Value("${coap.bind_address}") | ... | ... |
... | ... | @@ -50,7 +50,7 @@ import org.thingsboard.server.common.msg.session.*; |
50 | 50 | import org.thingsboard.server.common.transport.SessionMsgProcessor; |
51 | 51 | import org.thingsboard.server.common.transport.auth.DeviceAuthResult; |
52 | 52 | import org.thingsboard.server.common.transport.auth.DeviceAuthService; |
53 | -import org.thingsboard.server.common.transport.quota.QuotaService; | |
53 | +import org.thingsboard.server.common.transport.quota.host.HostRequestsQuotaService; | |
54 | 54 | |
55 | 55 | import java.util.ArrayList; |
56 | 56 | import java.util.List; |
... | ... | @@ -134,8 +134,8 @@ public class CoapServerTest { |
134 | 134 | } |
135 | 135 | |
136 | 136 | @Bean |
137 | - public static QuotaService quotaService() { | |
138 | - return key -> false; | |
137 | + public static HostRequestsQuotaService quotaService() { | |
138 | + return new HostRequestsQuotaService(null, null, null, null, false); | |
139 | 139 | } |
140 | 140 | } |
141 | 141 | ... | ... |
... | ... | @@ -36,6 +36,7 @@ import org.thingsboard.server.common.transport.SessionMsgProcessor; |
36 | 36 | import org.thingsboard.server.common.transport.adaptor.JsonConverter; |
37 | 37 | import org.thingsboard.server.common.transport.auth.DeviceAuthService; |
38 | 38 | import org.thingsboard.server.common.transport.quota.QuotaService; |
39 | +import org.thingsboard.server.common.transport.quota.host.HostRequestsQuotaService; | |
39 | 40 | import org.thingsboard.server.transport.http.session.HttpSessionCtx; |
40 | 41 | |
41 | 42 | import javax.servlet.http.HttpServletRequest; |
... | ... | @@ -61,7 +62,7 @@ public class DeviceApiController { |
61 | 62 | private DeviceAuthService authService; |
62 | 63 | |
63 | 64 | @Autowired(required = false) |
64 | - private QuotaService quotaService; | |
65 | + private HostRequestsQuotaService quotaService; | |
65 | 66 | |
66 | 67 | @RequestMapping(value = "/{deviceToken}/attributes", method = RequestMethod.GET, produces = "application/json") |
67 | 68 | public DeferredResult<ResponseEntity> getDeviceAttributes(@PathVariable("deviceToken") String deviceToken, | ... | ... |
... | ... | @@ -29,7 +29,7 @@ import org.springframework.context.ApplicationContext; |
29 | 29 | import org.springframework.stereotype.Service; |
30 | 30 | import org.thingsboard.server.common.transport.SessionMsgProcessor; |
31 | 31 | import org.thingsboard.server.common.transport.auth.DeviceAuthService; |
32 | -import org.thingsboard.server.common.transport.quota.QuotaService; | |
32 | +import org.thingsboard.server.common.transport.quota.host.HostRequestsQuotaService; | |
33 | 33 | import org.thingsboard.server.dao.device.DeviceService; |
34 | 34 | import org.thingsboard.server.dao.relation.RelationService; |
35 | 35 | import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; |
... | ... | @@ -67,7 +67,7 @@ public class MqttTransportService { |
67 | 67 | private MqttSslHandlerProvider sslHandlerProvider; |
68 | 68 | |
69 | 69 | @Autowired(required = false) |
70 | - private QuotaService quotaService; | |
70 | + private HostRequestsQuotaService quotaService; | |
71 | 71 | |
72 | 72 | @Value("${mqtt.bind_address}") |
73 | 73 | private String host; | ... | ... |