Commit e102b55bc139deff72c554f8e7f6e7c9f0c029a4
1 parent
7203e7aa
Thread-safety usage of Kafka Consumer
Showing
2 changed files
with
18 additions
and
13 deletions
... | ... | @@ -5,7 +5,7 @@ |
5 | 5 | * you may not use this file except in compliance with the License. |
6 | 6 | * You may obtain a copy of the License at |
7 | 7 | * |
8 | - * http://www.apache.org/licenses/LICENSE-2.0 | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | 9 | * |
10 | 10 | * Unless required by applicable law or agreed to in writing, software |
11 | 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
... | ... | @@ -293,12 +293,10 @@ public class DefaultDeviceStateService implements DeviceStateService { |
293 | 293 | callback.onSuccess(); |
294 | 294 | } |
295 | 295 | } |
296 | - | |
297 | 296 | } catch (Exception e) { |
298 | 297 | log.trace("Failed to process queue msg: [{}]", proto, e); |
299 | 298 | callback.onFailure(e); |
300 | 299 | } |
301 | - | |
302 | 300 | } |
303 | 301 | |
304 | 302 | @Override |
... | ... | @@ -366,6 +364,10 @@ public class DefaultDeviceStateService implements DeviceStateService { |
366 | 364 | } |
367 | 365 | } |
368 | 366 | } |
367 | + log.info("Managing following partitions:"); | |
368 | + partitionedDevices.forEach((tpi, devices) -> { | |
369 | + log.info("[{}]: {} devices", tpi.getFullTopicName(), devices.size()); | |
370 | + }); | |
369 | 371 | } catch (Throwable t) { |
370 | 372 | log.warn("Failed to init device states from DB", t); |
371 | 373 | } | ... | ... |
... | ... | @@ -5,7 +5,7 @@ |
5 | 5 | * you may not use this file except in compliance with the License. |
6 | 6 | * You may obtain a copy of the License at |
7 | 7 | * |
8 | - * http://www.apache.org/licenses/LICENSE-2.0 | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | 9 | * |
10 | 10 | * Unless required by applicable law or agreed to in writing, software |
11 | 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
... | ... | @@ -45,6 +45,7 @@ public class TBKafkaConsumerTemplate<T extends TbQueueMsg> implements TbQueueCon |
45 | 45 | private final KafkaConsumer<String, byte[]> consumer; |
46 | 46 | private final TbKafkaDecoder<T> decoder; |
47 | 47 | private volatile boolean subscribed; |
48 | + private volatile Set<TopicPartitionInfo> partitions; | |
48 | 49 | |
49 | 50 | @Getter |
50 | 51 | private final String topic; |
... | ... | @@ -74,29 +75,31 @@ public class TBKafkaConsumerTemplate<T extends TbQueueMsg> implements TbQueueCon |
74 | 75 | |
75 | 76 | @Override |
76 | 77 | public void subscribe() { |
77 | - createTopicIfNotExists(topic); | |
78 | - consumer.subscribe(Collections.singletonList(topic)); | |
79 | - subscribed = true; | |
78 | + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null)); | |
79 | + subscribed = false; | |
80 | 80 | } |
81 | 81 | |
82 | 82 | @Override |
83 | 83 | public void subscribe(Set<TopicPartitionInfo> partitions) { |
84 | - List<String> topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); | |
85 | - topicNames.forEach(this::createTopicIfNotExists); | |
86 | - consumer.unsubscribe(); | |
87 | - consumer.subscribe(topicNames); | |
88 | - subscribed = true; | |
84 | + this.partitions = partitions; | |
85 | + subscribed = false; | |
89 | 86 | } |
90 | 87 | |
91 | 88 | @Override |
92 | 89 | public List<T> poll(long durationInMillis) { |
93 | - if (!subscribed) { | |
90 | + if (!subscribed && partitions == null) { | |
94 | 91 | try { |
95 | 92 | Thread.sleep(durationInMillis); |
96 | 93 | } catch (InterruptedException e) { |
97 | 94 | log.debug("Failed to await subscription", e); |
98 | 95 | } |
99 | 96 | } else { |
97 | + if (!subscribed) { | |
98 | + List<String> topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); | |
99 | + topicNames.forEach(this::createTopicIfNotExists); | |
100 | + consumer.subscribe(topicNames); | |
101 | + subscribed = true; | |
102 | + } | |
100 | 103 | ConsumerRecords<String, byte[]> records = consumer.poll(Duration.ofMillis(durationInMillis)); |
101 | 104 | if (records.count() > 0) { |
102 | 105 | List<T> result = new ArrayList<>(); | ... | ... |