Add external config and log volumes for transport microservices.

@@ -4,4 +4,5 @@ tb-node/log/**
 tb-node/db/**
 tb-node/postgres/**
 tb-node/cassandra/**
+tb-transports/*/log
 !.env

@@ -26,6 +26,21 @@ services:
   tb2:
     volumes:
       - tb-log-volume:/var/log/thingsboard
+  tb-coap-transport:
+    volumes:
+      - tb-coap-transport-log-volume:/var/log/tb-coap-transport
+  tb-http-transport1:
+    volumes:
+      - tb-http-transport-log-volume:/var/log/tb-http-transport
+  tb-http-transport2:
+    volumes:
+      - tb-http-transport-log-volume:/var/log/tb-http-transport
+  tb-mqtt-transport1:
+    volumes:
+      - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
+  tb-mqtt-transport2:
+    volumes:
+      - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
 
 volumes:
   postgres-db-volume:
@@ -34,3 +49,12 @@ volumes:
   tb-log-volume:
     external: true
     name: ${TB_LOG_VOLUME}
+  tb-coap-transport-log-volume:
+    external: true
+    name: ${TB_COAP_TRANSPORT_LOG_VOLUME}
+  tb-http-transport-log-volume:
+    external: true
+    name: ${TB_HTTP_TRANSPORT_LOG_VOLUME}
+  tb-mqtt-transport-log-volume:
+    external: true
+    name: ${TB_MQTT_TRANSPORT_LOG_VOLUME}

@@ -91,8 +91,13 @@ services:
     image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
     ports:
       - "1883"
+    environment:
+      TB_HOST: tb-mqtt-transport1
     env_file:
       - tb-mqtt-transport.env
+    volumes:
+      - ./tb-transports/mqtt/conf:/config
+      - ./tb-transports/mqtt/log:/var/log/tb-mqtt-transport
     depends_on:
       - kafka
   tb-mqtt-transport2:
@@ -100,8 +105,13 @@ services:
     image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
     ports:
       - "1883"
+    environment:
+      TB_HOST: tb-mqtt-transport2
     env_file:
       - tb-mqtt-transport.env
+    volumes:
+      - ./tb-transports/mqtt/conf:/config
+      - ./tb-transports/mqtt/log:/var/log/tb-mqtt-transport
     depends_on:
       - kafka
   tb-http-transport1:
@@ -109,8 +119,13 @@ services:
     image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
     ports:
       - "8081"
+    environment:
+      TB_HOST: tb-http-transport1
     env_file:
       - tb-http-transport.env
+    volumes:
+      - ./tb-transports/http/conf:/config
+      - ./tb-transports/http/log:/var/log/tb-http-transport
     depends_on:
       - kafka
   tb-http-transport2:
@@ -118,8 +133,13 @@ services:
     image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
     ports:
       - "8081"
+    environment:
+      TB_HOST: tb-http-transport2
     env_file:
       - tb-http-transport.env
+    volumes:
+      - ./tb-transports/http/conf:/config
+      - ./tb-transports/http/log:/var/log/tb-http-transport
     depends_on:
       - kafka
   tb-coap-transport:
@@ -127,8 +147,13 @@ services:
     image: "${DOCKER_REPO}/${COAP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
     ports:
       - "5683:5683/udp"
+    environment:
+      TB_HOST: tb-coap-transport
     env_file:
       - tb-coap-transport.env
+    volumes:
+      - ./tb-transports/coap/conf:/config
+      - ./tb-transports/coap/log:/var/log/tb-coap-transport
     depends_on:
       - kafka
   tb-web-ui1:

@@ -15,7 +15,7 @@
 #
 
 export JAVA_OPTS="$JAVA_OPTS -Dplatform=deb -Dinstall.data_dir=/usr/share/thingsboard/data"
-export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
+export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
 export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10"
 export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark"
 export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled"

@@ -21,10 +21,10 @@
 
     <appender name="fileLogAppender"
               class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>/var/log/${pkg.name}/${pkg.name}.log</file>
+        <file>/var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.log</file>
         <rollingPolicy
                 class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-            <fileNamePattern>/var/log/${pkg.name}/${pkg.name}.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <fileNamePattern>/var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
             <maxFileSize>100MB</maxFileSize>
             <maxHistory>30</maxHistory>
             <totalSizeCap>3GB</totalSizeCap>

@@ -14,10 +14,10 @@
 # limitations under the License.
 #
 
-export JAVA_OPTS="$JAVA_OPTS -Xloggc:@pkg.logFolder@/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
+export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
 export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10"
 export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark"
 export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled"
 export JAVA_OPTS="$JAVA_OPTS -XX:+CMSEdenChunksRecordAlways -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+ExitOnOutOfMemoryError"
-export LOG_FILENAME=${pkg.name}.out
-export LOADER_PATH=${pkg.installFolder}/conf
+export LOG_FILENAME=tb-coap-transport.out
+export LOADER_PATH=/usr/share/tb-coap-transport/conf

@@ -21,10 +21,10 @@
 
     <appender name="fileLogAppender"
               class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>/var/log/${pkg.name}/${pkg.name}.log</file>
+        <file>/var/log/tb-http-transport/${TB_HOST}/tb-http-transport.log</file>
         <rollingPolicy
                 class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-            <fileNamePattern>/var/log/${pkg.name}/${pkg.name}.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <fileNamePattern>/var/log/tb-http-transport/${TB_HOST}/tb-http-transport.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
             <maxFileSize>100MB</maxFileSize>
             <maxHistory>30</maxHistory>
             <totalSizeCap>3GB</totalSizeCap>

@@ -14,10 +14,10 @@
 # limitations under the License.
 #
 
-export JAVA_OPTS="$JAVA_OPTS -Xloggc:@pkg.logFolder@/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
+export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
 export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10"
 export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark"
 export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled"
 export JAVA_OPTS="$JAVA_OPTS -XX:+CMSEdenChunksRecordAlways -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+ExitOnOutOfMemoryError"
-export LOG_FILENAME=${pkg.name}.out
-export LOADER_PATH=${pkg.installFolder}/conf
+export LOG_FILENAME=tb-http-transport.out
+export LOADER_PATH=/usr/share/tb-http-transport/conf

@@ -21,10 +21,10 @@
 
     <appender name="fileLogAppender"
               class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>/var/log/${pkg.name}/${pkg.name}.log</file>
+        <file>/var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.log</file>
         <rollingPolicy
                 class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-            <fileNamePattern>/var/log/${pkg.name}/${pkg.name}.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <fileNamePattern>/var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
             <maxFileSize>100MB</maxFileSize>
             <maxHistory>30</maxHistory>
             <totalSizeCap>3GB</totalSizeCap>

@@ -14,10 +14,10 @@
 # limitations under the License.
 #
 
-export JAVA_OPTS="$JAVA_OPTS -Xloggc:@pkg.logFolder@/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
+export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
 export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10"
 export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark"
 export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled"
 export JAVA_OPTS="$JAVA_OPTS -XX:+CMSEdenChunksRecordAlways -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+ExitOnOutOfMemoryError"
-export LOG_FILENAME=${pkg.name}.out
-export LOADER_PATH=${pkg.installFolder}/conf
+export LOG_FILENAME=tb-mqtt-transport.out
+export LOADER_PATH=/usr/share/tb-mqtt-transport/conf

@@ -49,8 +49,7 @@ public class ContainerTestSuite {
                     .withPull(false)
                     .withLocalCompose(true)
                     .withTailChildContainers(true)
-                    .withEnv("POSTGRES_DATA_VOLUME", installTb.getPostgresDataVolume())
-                    .withEnv("TB_LOG_VOLUME", installTb.getTbLogVolume())
+                    .withEnv(installTb.getEnv())
                     .withEnv("LOAD_BALANCER_NAME", "")
                     .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(Duration.ofSeconds(120)));
         }

@@ -28,11 +28,18 @@ public class ThingsBoardDbInstaller extends ExternalResource {
 
     private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume";
     private final static String TB_LOG_VOLUME = "tb-log-test-volume";
+    private final static String TB_COAP_TRANSPORT_LOG_VOLUME = "tb-coap-transport-log-test-volume";
+    private final static String TB_HTTP_TRANSPORT_LOG_VOLUME = "tb-http-transport-log-test-volume";
+    private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume";
 
     private final DockerComposeExecutor dockerCompose;
 
     private final String postgresDataVolume;
     private final String tbLogVolume;
+    private final String tbCoapTransportLogVolume;
+    private final String tbHttpTransportLogVolume;
+    private final String tbMqttTransportLogVolume;
+    private final Map<String, String> env;
 
     public ThingsBoardDbInstaller() {
         List<File> composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"),
@@ -44,21 +51,23 @@ public class ThingsBoardDbInstaller extends ExternalResource {
 
         postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME;
         tbLogVolume = project + "_" + TB_LOG_VOLUME;
+        tbCoapTransportLogVolume = project + "_" + TB_COAP_TRANSPORT_LOG_VOLUME;
+        tbHttpTransportLogVolume = project + "_" + TB_HTTP_TRANSPORT_LOG_VOLUME;
+        tbMqttTransportLogVolume = project + "_" + TB_MQTT_TRANSPORT_LOG_VOLUME;
 
         dockerCompose = new DockerComposeExecutor(composeFiles, project);
 
-        Map<String, String> env = new HashMap<>();
+        env = new HashMap<>();
         env.put("POSTGRES_DATA_VOLUME", postgresDataVolume);
         env.put("TB_LOG_VOLUME", tbLogVolume);
+        env.put("TB_COAP_TRANSPORT_LOG_VOLUME", tbCoapTransportLogVolume);
+        env.put("TB_HTTP_TRANSPORT_LOG_VOLUME", tbHttpTransportLogVolume);
+        env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume);
         dockerCompose.withEnv(env);
     }
 
-    public String getPostgresDataVolume() {
-        return postgresDataVolume;
-    }
-
-    public String getTbLogVolume() {
-        return tbLogVolume;
+    public Map<String, String> getEnv() {
+        return env;
     }
 
     @Override
@@ -71,6 +80,15 @@ public class ThingsBoardDbInstaller extends ExternalResource {
             dockerCompose.withCommand("volume create " + tbLogVolume);
             dockerCompose.invokeDocker();
 
+            dockerCompose.withCommand("volume create " + tbCoapTransportLogVolume);
+            dockerCompose.invokeDocker();
+
+            dockerCompose.withCommand("volume create " + tbHttpTransportLogVolume);
+            dockerCompose.invokeDocker();
+
+            dockerCompose.withCommand("volume create " + tbMqttTransportLogVolume);
+            dockerCompose.invokeDocker();
+
             dockerCompose.withCommand("up -d redis postgres");
             dockerCompose.invokeCompose();
 
@@ -88,8 +106,12 @@ public class ThingsBoardDbInstaller extends ExternalResource {
     @Override
     protected void after() {
         copyLogs(tbLogVolume, "./target/tb-logs/");
+        copyLogs(tbCoapTransportLogVolume, "./target/tb-coap-transport-logs/");
+        copyLogs(tbHttpTransportLogVolume, "./target/tb-http-transport-logs/");
+        copyLogs(tbMqttTransportLogVolume, "./target/tb-mqtt-transport-logs/");
 
-        dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume);
+        dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume +
+                " " + tbCoapTransportLogVolume + " " + tbHttpTransportLogVolume + " " + tbMqttTransportLogVolume);
         dockerCompose.invokeDocker();
     }
 

@@ -16,7 +16,7 @@
 
 FROM openjdk:8-jdk
 
-COPY logback.xml ${pkg.name}.conf start-tb-coap-transport.sh ${pkg.name}.deb /tmp/
+COPY start-tb-coap-transport.sh ${pkg.name}.deb /tmp/
 
 RUN chmod a+x /tmp/*.sh \
     && mv /tmp/start-tb-coap-transport.sh /usr/bin
@@ -25,7 +25,4 @@ RUN dpkg -i /tmp/${pkg.name}.deb
 
 RUN update-rc.d ${pkg.name} disable
 
-RUN mv /tmp/logback.xml ${pkg.installFolder}/conf \
-    && mv /tmp/${pkg.name}.conf ${pkg.installFolder}/conf
-
 CMD ["start-tb-coap-transport.sh"]

@@ -15,15 +15,17 @@
 # limitations under the License.
 #
 
-CONF_FOLDER="${pkg.installFolder}/conf"
+CONF_FOLDER="/config"
 jarfile=${pkg.installFolder}/bin/${pkg.name}.jar
 configfile=${pkg.name}.conf
 
 source "${CONF_FOLDER}/${configfile}"
 
+export LOADER_PATH=/config,${LOADER_PATH}
+
 echo "Starting '${project.name}' ..."
 
 exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.coap.ThingsboardCoapTransportApplication \
                     -Dspring.jpa.hibernate.ddl-auto=none \
-                    -Dlogging.config=${CONF_FOLDER}/logback.xml \
+                    -Dlogging.config=/config/logback.xml \
                     org.springframework.boot.loader.PropertiesLauncher

@@ -16,7 +16,7 @@
 
 FROM openjdk:8-jdk
 
-COPY logback.xml ${pkg.name}.conf start-tb-http-transport.sh ${pkg.name}.deb /tmp/
+COPY start-tb-http-transport.sh ${pkg.name}.deb /tmp/
 
 RUN chmod a+x /tmp/*.sh \
     && mv /tmp/start-tb-http-transport.sh /usr/bin
@@ -25,7 +25,4 @@ RUN dpkg -i /tmp/${pkg.name}.deb
 
 RUN update-rc.d ${pkg.name} disable
 
-RUN mv /tmp/logback.xml ${pkg.installFolder}/conf \
-    && mv /tmp/${pkg.name}.conf ${pkg.installFolder}/conf
-
 CMD ["start-tb-http-transport.sh"]

@@ -15,15 +15,17 @@
 # limitations under the License.
 #
 
-CONF_FOLDER="${pkg.installFolder}/conf"
+CONF_FOLDER="/config"
 jarfile=${pkg.installFolder}/bin/${pkg.name}.jar
 configfile=${pkg.name}.conf
 
 source "${CONF_FOLDER}/${configfile}"
 
+export LOADER_PATH=/config,${LOADER_PATH}
+
 echo "Starting '${project.name}' ..."
 
 exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.http.ThingsboardHttpTransportApplication \
                     -Dspring.jpa.hibernate.ddl-auto=none \
-                    -Dlogging.config=${CONF_FOLDER}/logback.xml \
+                    -Dlogging.config=/config/logback.xml \
                     org.springframework.boot.loader.PropertiesLauncher

@@ -16,7 +16,7 @@
 
 FROM openjdk:8-jdk
 
-COPY logback.xml ${pkg.name}.conf start-tb-mqtt-transport.sh ${pkg.name}.deb /tmp/
+COPY start-tb-mqtt-transport.sh ${pkg.name}.deb /tmp/
 
 RUN chmod a+x /tmp/*.sh \
     && mv /tmp/start-tb-mqtt-transport.sh /usr/bin
@@ -25,7 +25,4 @@ RUN dpkg -i /tmp/${pkg.name}.deb
 
 RUN update-rc.d ${pkg.name} disable
 
-RUN mv /tmp/logback.xml ${pkg.installFolder}/conf \
-    && mv /tmp/${pkg.name}.conf ${pkg.installFolder}/conf
-
 CMD ["start-tb-mqtt-transport.sh"]

@@ -15,15 +15,17 @@
 # limitations under the License.
 #
 
-CONF_FOLDER="${pkg.installFolder}/conf"
+CONF_FOLDER="/config"
 jarfile=${pkg.installFolder}/bin/${pkg.name}.jar
 configfile=${pkg.name}.conf
 
 source "${CONF_FOLDER}/${configfile}"
 
+export LOADER_PATH=/config,${LOADER_PATH}
+
 echo "Starting '${project.name}' ..."
 
 exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.mqtt.ThingsboardMqttTransportApplication \
                     -Dspring.jpa.hibernate.ddl-auto=none \
-                    -Dlogging.config=${CONF_FOLDER}/logback.xml \
+                    -Dlogging.config=/config/logback.xml \
                     org.springframework.boot.loader.PropertiesLauncher