Merge with master

@@ -82,7 +82,8 @@ mqtt:
 # Uncomment the following lines to enable ssl for MQTT
 #  ssl:
 #    key_store: keystore/mqttserver.jks
-#    key_store_password: password
+#    key_store_password: server_ks_password
+#    key_password: server_key_password
 #    key_store_type: JKS
 
 # CoAP server parameters

@@ -15,6 +15,7 @@
  */
 package org.thingsboard.server.dao;
 
+import com.google.common.base.CharMatcher;
 import lombok.extern.slf4j.Slf4j;
 import org.bouncycastle.crypto.digests.SHA3Digest;
 import org.bouncycastle.pqc.math.linearalgebra.ByteUtils;
@@ -28,7 +29,10 @@ public class EncryptionUtil {
     }
 
     public static String trimNewLines(String input) {
-        return input.replaceAll("\n","").replaceAll("\r","");
+        return input.replaceAll("-----BEGIN CERTIFICATE-----", "")
+                .replaceAll("-----END CERTIFICATE-----", "")
+                .replaceAll("\n","")
+                .replaceAll("\r","");
     }
 
     public static String getSha3Hash(String data) {

@@ -48,6 +48,10 @@
             <groupId>org.eclipse.paho</groupId>
             <artifactId>org.eclipse.paho.client.mqttv3</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

+/**
+ * Copyright © 2016-2017 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.client.tools;
+
+/**
+ * @author Valerii Sosliuk
+ * This class is intended for manual MQTT SSL Testing
+ */
+
+import com.google.common.io.Resources;
+import org.eclipse.paho.client.mqttv3.*;
+
+import javax.net.ssl.*;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.security.*;
+import java.security.cert.CertificateException;
+
+public class MqttSslClient {
+
+
+    private static final String MQTT_URL = "ssl://localhost:1883";
+
+    private static final String clientId = "MQTT_SSL_JAVA_CLIENT";
+    private static final String accessToken = "C1_TEST_TOKEN";
+    private static final String keyStoreFile = "mqttclient.jks";
+    private static final String JKS="JKS";
+    private static final String TLS="TLS";
+    private static final String CLIENT_KEYSTORE_PASSWORD = "client_ks_password";
+    private static final String CLIENT_KEY_PASSWORD = "client_key_password";
+
+    public static void main(String[] args) {
+
+        try {
+
+            URL ksUrl = Resources.getResource(keyStoreFile);
+            File ksFile = new File(ksUrl.toURI());
+            URL tsUrl = Resources.getResource(keyStoreFile);
+            File tsFile = new File(tsUrl.toURI());
+
+            TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+
+            KeyStore trustStore = KeyStore.getInstance(JKS);
+            trustStore.load(new FileInputStream(tsFile), CLIENT_KEYSTORE_PASSWORD.toCharArray());
+            tmf.init(trustStore);
+            KeyStore ks = KeyStore.getInstance(JKS);
+
+            ks.load(new FileInputStream(ksFile), CLIENT_KEYSTORE_PASSWORD.toCharArray());
+            KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+            kmf.init(ks, CLIENT_KEY_PASSWORD.toCharArray());
+
+            KeyManager[] km = kmf.getKeyManagers();
+            TrustManager[] tm = tmf.getTrustManagers();
+            SSLContext sslContext = SSLContext.getInstance(TLS);
+            sslContext.init(km, tm, null);
+
+            MqttConnectOptions options = new MqttConnectOptions();
+            options.setSocketFactory(sslContext.getSocketFactory());
+            MqttAsyncClient client = new MqttAsyncClient(MQTT_URL, clientId);
+            client.connect(options);
+            Thread.sleep(3000);
+            MqttMessage message = new MqttMessage();
+            message.setPayload("{\"key1\":\"value1\", \"key2\":true, \"key3\": 3.0, \"key4\": 4}".getBytes());
+            client.publish("v1/devices/me/telemetry", message);
+            client.disconnect();
+            System.out.println("Disconnected");
+            System.exit(0);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
\ No newline at end of file

@@ -15,12 +15,22 @@
 #
 
 DOMAIN_SUFFIX="$(hostname)"
-PASSWORD="password"
+ORGANIZATIONAL_UNIT=Thingsboard
+ORGANIZATION=Thingsboard
+CITY=Piscataway
+STATE_OR_PROVINCE=NJ
+TWO_LETTER_COUNTRY_CODE=US
 
-CLIENT_TRUSTSTORE="client_truststore.pem"
-CLIENT_KEY_ALIAS="clientalias"
-CLIENT_FILE_PREFIX="mqttclient"
+SERVER_KEYSTORE_PASSWORD=server_ks_password
+SERVER_KEY_PASSWORD=server_key_password
 
 SERVER_KEY_ALIAS="serveralias"
 SERVER_FILE_PREFIX="mqttserver"
-SERVER_KEYSTORE_DIR="../../../../application/src/main/resources/keystore/"
\ No newline at end of file
+SERVER_KEYSTORE_DIR="../../../../application/src/main/resources/keystore/"
+
+CLIENT_KEYSTORE_PASSWORD=client_ks_password
+CLIENT_KEY_PASSWORD=client_key_password
+
+CLIENT_KEY_ALIAS="clientalias"
+CLIENT_FILE_PREFIX="mqttclient"
+

-# -*- coding: utf-8 -*-
 #
 # Copyright © 2016-2017 The Thingsboard Authors
 #
@@ -44,7 +43,7 @@ client.publish('v1/devices/me/attributes/request/1', "{\"clientKeys\":\"model\"}
 
 #client.tls_set(ca_certs="client_truststore.pem", certfile="mqttclient.nopass.pem", keyfile=None, cert_reqs=ssl.CERT_REQUIRED,
 #               tls_version=ssl.PROTOCOL_TLSv1, ciphers=None);
-client.tls_set(ca_certs="client_truststore.pem", certfile=None, keyfile=None, cert_reqs=ssl.CERT_REQUIRED,
+client.tls_set(ca_certs="mqttserver.pub.pem", certfile=None, keyfile=None, cert_reqs=ssl.CERT_REQUIRED,
                        tls_version=ssl.PROTOCOL_TLSv1, ciphers=None);
 
 client.username_pw_set("B1_TEST_TOKEN")

-#!/bin/sh
+#!/bin/bash
 #
 # Copyright © 2016-2017 The Thingsboard Authors
 #
@@ -17,7 +17,7 @@
 
 usage() {
     echo "This script generates client public/private rey pair, extracts them to a no-password RSA pem file,"
-    echo "and also imports server public key to client trust store"
+    echo "and imports server public key to client keystore"
     echo "usage: ./securemqttclient.keygen.sh [-p file]"
     echo "    -p | --props | --properties file  Properties file. default value is ./keygen.properties"
 	echo "    -h | --help  | ?                  Show this message"
@@ -44,17 +44,44 @@ done
 
 . $PROPERTIES_FILE
 
+if [ -f $CLIENT_FILE_PREFIX.jks ] || [ -f $CLIENT_FILE_PREFIX.pub.pem ] || [ -f $CLIENT_FILE_PREFIX.nopass.pem ] || [ -f $CLIENT_FILE_PREFIX.pem ] || [ -f $CLIENT_FILE_PREFIX.p12 ];
+then
+while :
+   do
+       read -p "Output files from previous server.keygen.sh script run found. Overwrite?[yes]" response
+       case $response in
+        [nN]|[nN][oO])
+            echo "Skipping"
+            echo "Done"
+            exit 0
+            ;;
+        [yY]|[yY][eE]|[yY][eE]|[sS]|[yY]|"")
+            echo "Cleaning up files"
+            rm -rf $CLIENT_FILE_PREFIX.jks
+            rm -rf $CLIENT_FILE_PREFIX.pub.pem
+            rm -rf $CLIENT_FILE_PREFIX.nopass.pem
+            rm -rf $CLIENT_FILE_PREFIX.pem
+            rm -rf $CLIENT_FILE_PREFIX.p12
+            break;
+            ;;
+        *)  echo "Please reply 'yes' or 'no'"
+            ;;
+        esac
+    done
+fi
+
 echo "Generating SSL Key Pair..."
 
 keytool -genkeypair -v \
   -alias $CLIENT_KEY_ALIAS \
   -dname "CN=$DOMAIN_SUFFIX, OU=Thingsboard, O=Thingsboard, L=Piscataway, ST=NJ, C=US" \
   -keystore $CLIENT_FILE_PREFIX.jks \
-  -keypass $PASSWORD \
-  -storepass $PASSWORD \
+  -keypass $CLIENT_KEY_PASSWORD \
+  -storepass $CLIENT_KEYSTORE_PASSWORD \
   -keyalg RSA \
   -keysize 2048 \
   -validity 9999
+
 echo "Converting keystore to pkcs12"
 keytool -importkeystore  \
   -srckeystore $CLIENT_FILE_PREFIX.jks \
@@ -62,28 +89,33 @@ keytool -importkeystore  \
   -srcalias $CLIENT_KEY_ALIAS \
   -srcstoretype jks \
   -deststoretype pkcs12 \
-  -keypass $PASSWORD \
-  -srcstorepass $PASSWORD \
-  -deststorepass $PASSWORD \
-  -srckeypass $PASSWORD \
-  -destkeypass $PASSWORD
+  -srcstorepass $CLIENT_KEYSTORE_PASSWORD \
+  -deststorepass $CLIENT_KEY_PASSWORD \
+  -srckeypass $CLIENT_KEY_PASSWORD \
+  -destkeypass $CLIENT_KEY_PASSWORD
 
 echo "Converting pkcs12 to pem"
 openssl pkcs12 -in $CLIENT_FILE_PREFIX.p12 \
   -out $CLIENT_FILE_PREFIX.pem \
-  -passin pass:$PASSWORD \
-  -passout pass:$PASSWORD \
+  -passin pass:$CLIENT_KEY_PASSWORD \
+  -passout pass:$CLIENT_KEY_PASSWORD \
 
-echo "Importing server public key..."
-keytool -export \
-  -alias $SERVER_KEY_ALIAS \
-  -keystore $SERVER_KEYSTORE_DIR/$SERVER_FILE_PREFIX.jks \
-  -file $CLIENT_TRUSTSTORE -rfc \
-  -storepass $PASSWORD
+echo "Importing server public key to $CLIENT_FILE_PREFIX.jks"
+keytool --importcert \
+   -file $SERVER_FILE_PREFIX.cer \
+   -keystore $CLIENT_FILE_PREFIX.jks \
+   -alias $SERVER_KEY_ALIAS \
+   -keypass $SERVER_KEY_PASSWORD \
+   -storepass $CLIENT_KEYSTORE_PASSWORD \
+   -noprompt
 
 echo "Exporting no-password pem certificate"
-openssl rsa -in $CLIENT_FILE_PREFIX.pem -out $CLIENT_FILE_PREFIX.nopass.pem -passin pass:$PASSWORD
+openssl rsa -in $CLIENT_FILE_PREFIX.pem -out $CLIENT_FILE_PREFIX.nopass.pem -passin pass:$CLIENT_KEY_PASSWORD
 tail -n +$(($(grep -m1 -n -e '-----BEGIN CERTIFICATE' $CLIENT_FILE_PREFIX.pem | cut -d: -f1) )) \
   $CLIENT_FILE_PREFIX.pem >> $CLIENT_FILE_PREFIX.nopass.pem
 
+echo "Exporting client public key"
+tail -n +$(($(grep -m1 -n -e '-----BEGIN CERTIFICATE' $CLIENT_FILE_PREFIX.pem | cut -d: -f1) )) \
+  $CLIENT_FILE_PREFIX.pem >> $CLIENT_FILE_PREFIX.pub.pem
+
 echo "Done."
\ No newline at end of file

-#!/bin/sh
+#!/bin/bash
 #
 # Copyright © 2016-2017 The Thingsboard Authors
 #
@@ -18,9 +18,9 @@
 usage() {
     echo "This script generates thingsboard server's ssl certificate"
     echo "and optionally copies it to the server's resource directory."
-    echo "usage: ./keygen.sh [-c flag] [-d directory]"
-    echo "    -c | --copy flag                  Set if copy keystore to server directory needed. Default value is true"
-    echo "    -d | --dir directory              Server keystore directory, where the generated keystore file will be copied."
+    echo "usage: ./server.keygen.sh [-c flag] [-d directory] [-p file]"
+    echo "    -c | --copy flag                  Specifies if the keystore should be copied to the server directory. Defaults to true"
+    echo "    -d | --dir directory              Server keystore directory, where the generated keystore file will be copied. If specified, overrides the value from the properties file"
     echo "                                      Default value is SERVER_KEYSTORE_DIR property from properties file"
     echo "    -p | --props | --properties file  Properties file. default value is ./keygen.properties"
 	echo "    -h | --help | ?                   Show this message"
@@ -32,23 +32,24 @@ PROPERTIES_FILE=keygen.properties
 
 while true; do
   case "$1" in
-    -c | --copy)                 COPY=$2 ;
-                                 shift
-                                 ;;
-    -d | --dir | --directory)    COPY_DIR=$2 ;
-                                 shift
-                                 ;;
-    -p | --props | --properties) PROPERTIES_FILE=$2 ;
-                                shift
-                                ;;
-    -h | --help | ?)            usage
-                                exit 0
-                                ;;
-    -- ) shift;
-         break
-         ;;
-    * ) break
-         ;;
+    -c | --copy)                  COPY=$2 ;
+                                  shift
+                                  ;;
+    -d | --dir | --directory )    COPY_DIR=$2 ;
+                                  shift
+                                  ;;
+    -p | --props | --properties ) PROPERTIES_FILE=$2 ;
+                                  shift
+                                  ;;
+    -- )                          shift;
+                                  break
+                                  ;;
+    "" )                          break
+                                  ;;
+
+    -h | --help | ? | *)          usage
+                                  exit 0
+                                  ;;
   esac
   shift
 done
@@ -57,18 +58,40 @@ if [[ "$COPY" != true ]] && [[ "$COPY" != false ]]; then
    usage
 fi
 
-echo "copy: $COPY; copy_dir: $COPY_DIR; PROPERTIES_FILE=$PROPERTIES_FILE";
-
 . $PROPERTIES_FILE
 
+if [ -f $SERVER_FILE_PREFIX.jks ] || [ -f $SERVER_FILE_PREFIX.cer ] || [ -f $SERVER_FILE_PREFIX.pub.pem ] || [ -f $SERVER_FILE_PREFIX.pub.der ];
+then
+while :
+   do
+       read -p "Output files from previous server.keygen.sh script run found. Overwrite?[yes]" response
+       case $response in
+        [nN]|[nN][oO])
+            echo "Skipping"
+            echo "Done"
+            exit 0
+            ;;
+        [yY]|[yY][eE]|[yY][eE]|[sS]|[yY]|"")
+            echo "Cleaning up files"
+            rm -rf $SERVER_FILE_PREFIX.jks
+            rm -rf $SERVER_FILE_PREFIX.pub.pem
+            rm -rf $SERVER_FILE_PREFIX.cer
+            break;
+            ;;
+        *)  echo "Please reply 'yes' or 'no'"
+            ;;
+        esac
+    done
+fi
+
 echo "Generating SSL Key Pair..."
 
 keytool -genkeypair -v \
   -alias $SERVER_KEY_ALIAS \
-  -dname "CN=$DOMAIN_SUFFIX, OU=Thingsboard, O=Thingsboard, L=Piscataway, ST=NJ, C=US" \
+  -dname "CN=$DOMAIN_SUFFIX, OU=$ORGANIZATIONAL_UNIT, O=$ORGANIZATION, L=$CITY, ST=$STATE_OR_PROVINCE, C=$TWO_LETTER_COUNTRY_CODE" \
   -keystore $SERVER_FILE_PREFIX.jks \
-  -keypass $PASSWORD \
-  -storepass $PASSWORD \
+  -keypass $SERVER_KEY_PASSWORD \
+  -storepass $SERVER_KEYSTORE_PASSWORD \
   -keyalg RSA \
   -keysize 2048 \
   -validity 9999
@@ -81,8 +104,15 @@ fi
 keytool -export \
   -alias $SERVER_KEY_ALIAS \
   -keystore $SERVER_FILE_PREFIX.jks \
-  -file $CLIENT_TRUSTSTORE -rfc \
-  -storepass $PASSWORD
+  -file $SERVER_FILE_PREFIX.pub.pem -rfc \
+  -storepass $SERVER_KEYSTORE_PASSWORD
+
+keytool -export \
+  -alias $SERVER_KEY_ALIAS \
+  -file $SERVER_FILE_PREFIX.cer \
+  -keystore $SERVER_FILE_PREFIX.jks \
+  -storepass $SERVER_KEYSTORE_PASSWORD \
+  -keypass $SERVER_KEY_PASSWORD
 
 status=$?
 if [[ $status != 0 ]]; then
@@ -92,27 +122,37 @@ fi
 
 if [[ $COPY = true ]]; then
     if [[ -z "$COPY_DIR" ]]; then
-        read -p  "Do you want to copy $SERVER_FILE_PREFIX.jks to server directory? " yn
-            case $yn in
-                [Yy]) echo "Please, specify destination dir: "
-                     read -p "(Default: $SERVER_KEYSTORE_DIR): " dir
-                     if [[ !  -z  $dir  ]]; then
-                        DESTINATION=$dir;
-                     else
-                        DESTINATION=$SERVER_KEYSTORE_DIR
-                     fi;
-                     break;;
-                * ) ;;
-            esac
+        read -p  "Do you want to copy $SERVER_FILE_PREFIX.jks to server directory?[yes]" yn
+            while :
+            do
+                case $yn in
+                    [nN]|[nN][oO])
+                        break
+                        ;;
+                    [yY]|[yY][eE]|[yY][eE]|[sS]|[yY]|"")
+                        read -p "(Default: $SERVER_KEYSTORE_DIR): " dir
+                         if [[ !  -z  $dir  ]]; then
+                            DESTINATION=$dir;
+                         else
+                            DESTINATION=$SERVER_KEYSTORE_DIR
+                         fi;
+                         break;;
+                    *)  echo "Please reply 'yes' or 'no'"
+                        ;;
+                 esac
+             done
     else
         DESTINATION=$COPY_DIR
     fi
-    mkdir -p $DESTINATION
-    cp $SERVER_FILE_PREFIX.jks $DESTINATION
-    if [ $? -ne 0 ]; then
-        echo "Failed to copy keystore file."
-    else
-        echo "File copied successfully."
+    echo "*** DEST: $DESTINATION"
+    if [[ -n $DESTINATION ]]; then
+        mkdir -p $DESTINATION
+        cp $SERVER_FILE_PREFIX.jks $DESTINATION
+        if [ $? -ne 0 ]; then
+            echo "Failed to copy keystore file."
+        else
+            echo "File copied successfully."
+        fi
     fi
 fi
 echo "Done."
\ No newline at end of file

@@ -41,7 +41,7 @@ client.on_connect = on_connect
 client.on_message = on_message
 client.publish('v1/devices/me/attributes/request/1', "{\"clientKeys\":\"model\"}", 1)
 
-client.username_pw_set("TEST_TOKEN")
+client.username_pw_set("B1_TEST_TOKEN")
 client.connect('127.0.0.1', 1883, 1)
 
 # Blocking call that processes network traffic, dispatches callbacks and

@@ -42,7 +42,7 @@ client.on_connect = on_connect
 client.on_message = on_message
 client.publish('v1/devices/me/attributes/request/1', "{\"clientKeys\":\"model\"}", 1)
 
-client.tls_set(ca_certs="client_truststore.pem", certfile="mqttclient.nopass.pem", keyfile=None, cert_reqs=ssl.CERT_REQUIRED,
+client.tls_set(ca_certs="mqttserver.pub.pem", certfile="mqttclient.nopass.pem", keyfile=None, cert_reqs=ssl.CERT_REQUIRED,
                        tls_version=ssl.PROTOCOL_TLSv1, ciphers=None);
 
 client.tls_insecure_set(False)

@@ -64,11 +64,9 @@
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
         </dependency>
-        <!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
-            <version>18.0</version>
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>

@@ -49,9 +49,11 @@ public class MqttSslHandlerProvider {
     private String keyStoreFile;
     @Value("${mqtt.ssl.key_store_password}")
     private String keyStorePassword;
+    @Value("${mqtt.ssl.key_password}")
+    private String keyPassword;
     @Value("${mqtt.ssl.key_store_type}")
     private String keyStoreType;
-
+    
     @Autowired
     private DeviceCredentialsService deviceCredentialsService;
 
@@ -72,7 +74,7 @@ public class MqttSslHandlerProvider {
 
             ks.load(new FileInputStream(ksFile), keyStorePassword.toCharArray());
             KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-            kmf.init(ks, keyStorePassword.toCharArray());
+            kmf.init(ks, keyPassword.toCharArray());
 
             KeyManager[] km = kmf.getKeyManagers();
             TrustManager x509wrapped = getX509TrustManager(tmFactory);

@@ -16,6 +16,7 @@
 package org.thingsboard.server.transport.mqtt.util;
 
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.util.Base64Utils;
 import org.thingsboard.server.dao.EncryptionUtil;
 import sun.misc.BASE64Encoder;
 
@@ -35,17 +36,13 @@ public class SslUtil {
 
     public static String getX509CertificateString(X509Certificate cert)
             throws CertificateEncodingException, IOException {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        BASE64Encoder encoder = new BASE64Encoder();
-        encoder.encodeBuffer(cert.getEncoded(), out);
-        return EncryptionUtil.trimNewLines(new String(out.toByteArray(), "UTF-8"));
+        Base64Utils.encodeToString(cert.getEncoded());
+        return EncryptionUtil.trimNewLines(Base64Utils.encodeToString(cert.getEncoded()));
     }
 
     public static String getX509CertificateString(javax.security.cert.X509Certificate cert)
             throws javax.security.cert.CertificateEncodingException, IOException {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        BASE64Encoder encoder = new BASE64Encoder();
-        encoder.encodeBuffer(cert.getEncoded(), out);
-        return EncryptionUtil.trimNewLines(new String(out.toByteArray(), "UTF-8"));
+        Base64Utils.encodeToString(cert.getEncoded());
+        return EncryptionUtil.trimNewLines(Base64Utils.encodeToString(cert.getEncoded()));
     }
 }