Commit 84cb471e0d7e2e49807a232b87c949d44e377fb2

Authored by ShvaykaD
Committed by Andrew Shvayka
1 parent cc0e5417

Sql timeseries improvements (#2033)

* init commit

* cleaned code and add test-properties

* cleaned code

* psql-update

* timescale-update

* code-refactoring

* fix typo

* renamed dao

* revert indents

* refactored code

* fix typo

* init-partitioning

* code updated

* cleaned code

* fixed license

* fix typo

* fixed code after review

* add annotation to repository

* update psql version for docker

* postgres-10

* postgres-10

* update docker compose config

* fixed partition saving

* change key_id to serial column definition

* upgrade psql added

* add separate upgrade service

* added upgrade script

* change image on k8s

* change logs

* resolve conflict after merge with master

* revert datasource url in yml

* fix typo

* license header fix

* remove old methods for the timeseries inserts

* clean up code

* fix saveOrUpdate for PostgreSQL

* refactoring & revert Timescale to use latest table

* added PsqlTsAnyDao

* duplicated code method removed

* remove unused invert dictionary map

* change the upgrade directory from 2.4.1 to 2.4.3

* refactor JpaPsqlTimeseriesDao
Showing 72 changed files with 2497 additions and 1245 deletions
  1 +--
  2 +-- Copyright © 2016-2020 The Thingsboard Authors
  3 +--
  4 +-- Licensed under the Apache License, Version 2.0 (the "License");
  5 +-- you may not use this file except in compliance with the License.
  6 +-- You may obtain a copy of the License at
  7 +--
  8 +-- http://www.apache.org/licenses/LICENSE-2.0
  9 +--
  10 +-- Unless required by applicable law or agreed to in writing, software
  11 +-- distributed under the License is distributed on an "AS IS" BASIS,
  12 +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +-- See the License for the specific language governing permissions and
  14 +-- limitations under the License.
  15 +--
  16 +
  17 +-- load function check_version()
  18 +
  19 +CREATE OR REPLACE FUNCTION check_version() RETURNS boolean AS $$
  20 +DECLARE
  21 + current_version integer;
  22 + valid_version boolean;
  23 +BEGIN
  24 + RAISE NOTICE 'Check the current installed PostgreSQL version...';
  25 + SELECT current_setting('server_version_num') INTO current_version;
  26 + IF current_version < 100000 THEN
  27 + valid_version := FALSE;
  28 + ELSE
  29 + valid_version := TRUE;
  30 + END IF;
  31 + IF valid_version = FALSE THEN
  32 + RAISE NOTICE 'Postgres version should be at least more than 10!';
  33 + ELSE
  34 + RAISE NOTICE 'PostgreSQL version is valid!';
  35 + RAISE NOTICE 'Schema update started...';
  36 + END IF;
  37 + RETURN valid_version;
  38 +END;
  39 +$$ LANGUAGE 'plpgsql';
  40 +
  41 +-- load function create_partition_table()
  42 +
  43 +CREATE OR REPLACE FUNCTION create_partition_table() RETURNS VOID AS $$
  44 +
  45 +BEGIN
  46 + ALTER TABLE ts_kv
  47 + RENAME TO ts_kv_old;
  48 + CREATE TABLE IF NOT EXISTS ts_kv
  49 + (
  50 + LIKE ts_kv_old
  51 + )
  52 + PARTITION BY RANGE (ts);
  53 + ALTER TABLE ts_kv
  54 + DROP COLUMN entity_type;
  55 + ALTER TABLE ts_kv
  56 + ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid;
  57 + ALTER TABLE ts_kv
  58 + ALTER COLUMN key TYPE integer USING key::integer;
  59 +END;
  60 +$$ LANGUAGE 'plpgsql';
  61 +
  62 +
  63 +-- load function create_partitions()
  64 +
  65 +CREATE OR REPLACE FUNCTION create_partitions() RETURNS VOID AS
  66 +$$
  67 +DECLARE
  68 + partition_date varchar;
  69 + from_ts bigint;
  70 + to_ts bigint;
  71 + key_cursor CURSOR FOR select SUBSTRING(month_date.first_date, 1, 7) AS partition_date,
  72 + extract(epoch from (month_date.first_date)::timestamp) * 1000 as from_ts,
  73 + extract(epoch from (month_date.first_date::date + INTERVAL '1 MONTH')::timestamp) *
  74 + 1000 as to_ts
  75 + FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_MM_01') AS first_date
  76 + FROM ts_kv_old) AS month_date;
  77 +BEGIN
  78 + OPEN key_cursor;
  79 + LOOP
  80 + FETCH key_cursor INTO partition_date, from_ts, to_ts;
  81 + EXIT WHEN NOT FOUND;
  82 + EXECUTE 'CREATE TABLE IF NOT EXISTS ts_kv_' || partition_date ||
  83 + ' PARTITION OF ts_kv(PRIMARY KEY (entity_id, key, ts)) FOR VALUES FROM (' || from_ts ||
  84 + ') TO (' || to_ts || ');';
  85 + RAISE NOTICE 'A partition % has been created!',CONCAT('ts_kv_', partition_date);
  86 + END LOOP;
  87 +
  88 + CLOSE key_cursor;
  89 +END;
  90 +$$ language 'plpgsql';
  91 +
  92 +-- load function create_ts_kv_dictionary_table()
  93 +
  94 +CREATE OR REPLACE FUNCTION create_ts_kv_dictionary_table() RETURNS VOID AS $$
  95 +
  96 +BEGIN
  97 + CREATE TABLE IF NOT EXISTS ts_kv_dictionary
  98 + (
  99 + key varchar(255) NOT NULL,
  100 + key_id serial UNIQUE,
  101 + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key)
  102 + );
  103 +END;
  104 +$$ LANGUAGE 'plpgsql';
  105 +
  106 +-- load function insert_into_dictionary()
  107 +
  108 +CREATE OR REPLACE FUNCTION insert_into_dictionary() RETURNS VOID AS
  109 +$$
  110 +DECLARE
  111 + insert_record RECORD;
  112 + key_cursor CURSOR FOR SELECT DISTINCT key
  113 + FROM ts_kv_old
  114 + ORDER BY key;
  115 +BEGIN
  116 + OPEN key_cursor;
  117 + LOOP
  118 + FETCH key_cursor INTO insert_record;
  119 + EXIT WHEN NOT FOUND;
  120 + IF NOT EXISTS(SELECT key FROM ts_kv_dictionary WHERE key = insert_record.key) THEN
  121 + INSERT INTO ts_kv_dictionary(key) VALUES (insert_record.key);
  122 + RAISE NOTICE 'Key: % has been inserted into the dictionary!',insert_record.key;
  123 + ELSE
  124 + RAISE NOTICE 'Key: % already exists in the dictionary!',insert_record.key;
  125 + END IF;
  126 + END LOOP;
  127 + CLOSE key_cursor;
  128 +END;
  129 +$$ language 'plpgsql';
  130 +
  131 +-- load function insert_into_ts_kv()
  132 +
  133 +CREATE OR REPLACE FUNCTION insert_into_ts_kv() RETURNS void AS
  134 +$$
  135 +DECLARE
  136 + insert_size CONSTANT integer := 10000;
  137 + insert_counter integer DEFAULT 0;
  138 + insert_record RECORD;
  139 + insert_cursor CURSOR FOR SELECT CONCAT(first, '-', second, '-1', third, '-', fourth, '-', fifth)::uuid AS entity_id,
  140 + substrings.key AS key,
  141 + substrings.ts AS ts,
  142 + substrings.bool_v AS bool_v,
  143 + substrings.str_v AS str_v,
  144 + substrings.long_v AS long_v,
  145 + substrings.dbl_v AS dbl_v
  146 + FROM (SELECT SUBSTRING(entity_id, 8, 8) AS first,
  147 + SUBSTRING(entity_id, 4, 4) AS second,
  148 + SUBSTRING(entity_id, 1, 3) AS third,
  149 + SUBSTRING(entity_id, 16, 4) AS fourth,
  150 + SUBSTRING(entity_id, 20) AS fifth,
  151 + key_id AS key,
  152 + ts,
  153 + bool_v,
  154 + str_v,
  155 + long_v,
  156 + dbl_v
  157 + FROM ts_kv_old
  158 + INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS substrings;
  159 +BEGIN
  160 + OPEN insert_cursor;
  161 + LOOP
  162 + insert_counter := insert_counter + 1;
  163 + FETCH insert_cursor INTO insert_record;
  164 + IF NOT FOUND THEN
  165 + RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter - 1;
  166 + EXIT;
  167 + END IF;
  168 + INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
  169 + VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
  170 + insert_record.long_v, insert_record.dbl_v);
  171 + IF MOD(insert_counter, insert_size) = 0 THEN
  172 + RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter;
  173 + END IF;
  174 + END LOOP;
  175 + CLOSE insert_cursor;
  176 +END;
  177 +$$ LANGUAGE 'plpgsql';
  178 +
  179 +
... ...
... ... @@ -23,7 +23,8 @@ import org.springframework.context.ApplicationContext;
23 23 import org.springframework.context.annotation.Profile;
24 24 import org.springframework.stereotype.Service;
25 25 import org.thingsboard.server.service.component.ComponentDiscoveryService;
26   -import org.thingsboard.server.service.install.DatabaseUpgradeService;
  26 +import org.thingsboard.server.service.install.DatabaseTsUpgradeService;
  27 +import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService;
27 28 import org.thingsboard.server.service.install.EntityDatabaseSchemaService;
28 29 import org.thingsboard.server.service.install.SystemDataLoaderService;
29 30 import org.thingsboard.server.service.install.TsDatabaseSchemaService;
... ... @@ -50,7 +51,10 @@ public class ThingsboardInstallService {
50 51 private TsDatabaseSchemaService tsDatabaseSchemaService;
51 52
52 53 @Autowired
53   - private DatabaseUpgradeService databaseUpgradeService;
  54 + private DatabaseEntitiesUpgradeService databaseEntitiesUpgradeService;
  55 +
  56 + @Autowired
  57 + private DatabaseTsUpgradeService databaseTsUpgradeService;
54 58
55 59 @Autowired
56 60 private ComponentDiscoveryService componentDiscoveryService;
... ... @@ -73,48 +77,48 @@ public class ThingsboardInstallService {
73 77 case "1.2.3": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion
74 78 log.info("Upgrading ThingsBoard from version 1.2.3 to 1.3.0 ...");
75 79
76   - databaseUpgradeService.upgradeDatabase("1.2.3");
  80 + databaseEntitiesUpgradeService.upgradeDatabase("1.2.3");
77 81
78 82 case "1.3.0": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion
79 83 log.info("Upgrading ThingsBoard from version 1.3.0 to 1.3.1 ...");
80 84
81   - databaseUpgradeService.upgradeDatabase("1.3.0");
  85 + databaseEntitiesUpgradeService.upgradeDatabase("1.3.0");
82 86
83 87 case "1.3.1": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion
84 88 log.info("Upgrading ThingsBoard from version 1.3.1 to 1.4.0 ...");
85 89
86   - databaseUpgradeService.upgradeDatabase("1.3.1");
  90 + databaseEntitiesUpgradeService.upgradeDatabase("1.3.1");
87 91
88 92 case "1.4.0":
89 93 log.info("Upgrading ThingsBoard from version 1.4.0 to 2.0.0 ...");
90 94
91   - databaseUpgradeService.upgradeDatabase("1.4.0");
  95 + databaseEntitiesUpgradeService.upgradeDatabase("1.4.0");
92 96
93 97 dataUpdateService.updateData("1.4.0");
94 98
95 99 case "2.0.0":
96 100 log.info("Upgrading ThingsBoard from version 2.0.0 to 2.1.1 ...");
97 101
98   - databaseUpgradeService.upgradeDatabase("2.0.0");
  102 + databaseEntitiesUpgradeService.upgradeDatabase("2.0.0");
99 103
100 104 case "2.1.1":
101 105 log.info("Upgrading ThingsBoard from version 2.1.1 to 2.1.2 ...");
102 106
103   - databaseUpgradeService.upgradeDatabase("2.1.1");
  107 + databaseEntitiesUpgradeService.upgradeDatabase("2.1.1");
104 108 case "2.1.3":
105 109 log.info("Upgrading ThingsBoard from version 2.1.3 to 2.2.0 ...");
106 110
107   - databaseUpgradeService.upgradeDatabase("2.1.3");
  111 + databaseEntitiesUpgradeService.upgradeDatabase("2.1.3");
108 112
109 113 case "2.3.0":
110 114 log.info("Upgrading ThingsBoard from version 2.3.0 to 2.3.1 ...");
111 115
112   - databaseUpgradeService.upgradeDatabase("2.3.0");
  116 + databaseEntitiesUpgradeService.upgradeDatabase("2.3.0");
113 117
114 118 case "2.3.1":
115 119 log.info("Upgrading ThingsBoard from version 2.3.1 to 2.4.0 ...");
116 120
117   - databaseUpgradeService.upgradeDatabase("2.3.1");
  121 + databaseEntitiesUpgradeService.upgradeDatabase("2.3.1");
118 122
119 123 case "2.4.0":
120 124 log.info("Upgrading ThingsBoard from version 2.4.0 to 2.4.1 ...");
... ... @@ -122,11 +126,16 @@ public class ThingsboardInstallService {
122 126 case "2.4.1":
123 127 log.info("Upgrading ThingsBoard from version 2.4.1 to 2.4.2 ...");
124 128
125   - databaseUpgradeService.upgradeDatabase("2.4.1");
  129 + databaseEntitiesUpgradeService.upgradeDatabase("2.4.1");
126 130 case "2.4.2":
127 131 log.info("Upgrading ThingsBoard from version 2.4.2 to 2.4.3 ...");
128 132
129   - databaseUpgradeService.upgradeDatabase("2.4.2");
  133 + databaseEntitiesUpgradeService.upgradeDatabase("2.4.2");
  134 +
  135 + case "2.4.3":
  136 + log.info("Upgrading ThingsBoard from version 2.4.3 to 2.5 ...");
  137 +
  138 + databaseTsUpgradeService.upgradeDatabase("2.4.3");
130 139
131 140 log.info("Updating system data...");
132 141
... ...
... ... @@ -59,7 +59,7 @@ import static org.thingsboard.server.service.install.DatabaseHelper.TYPE;
59 59 @NoSqlDao
60 60 @Profile("install")
61 61 @Slf4j
62   -public class CassandraDatabaseUpgradeService implements DatabaseUpgradeService {
  62 +public class CassandraDatabaseUpgradeService implements DatabaseEntitiesUpgradeService {
63 63
64 64 private static final String SCHEMA_UPDATE_CQL = "schema_update.cql";
65 65
... ...
application/src/main/java/org/thingsboard/server/service/install/DatabaseEntitiesUpgradeService.java renamed from application/src/main/java/org/thingsboard/server/service/install/DatabaseUpgradeService.java
... ... @@ -15,7 +15,7 @@
15 15 */
16 16 package org.thingsboard.server.service.install;
17 17
18   -public interface DatabaseUpgradeService {
  18 +public interface DatabaseEntitiesUpgradeService {
19 19
20 20 void upgradeDatabase(String fromVersion) throws Exception;
21 21
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.install;
  17 +
  18 +public interface DatabaseTsUpgradeService {
  19 +
  20 + void upgradeDatabase(String fromVersion) throws Exception;
  21 +
  22 +}
\ No newline at end of file
... ...
application/src/main/java/org/thingsboard/server/service/install/HsqlTsDatabaseSchemaService.java renamed from application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java
... ... @@ -17,14 +17,16 @@ package org.thingsboard.server.service.install;
17 17
18 18 import org.springframework.context.annotation.Profile;
19 19 import org.springframework.stereotype.Service;
  20 +import org.thingsboard.server.dao.util.HsqlDao;
20 21 import org.thingsboard.server.dao.util.SqlTsDao;
21 22
22 23 @Service
23 24 @SqlTsDao
  25 +@HsqlDao
24 26 @Profile("install")
25   -public class SqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService
  27 +public class HsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService
26 28 implements TsDatabaseSchemaService {
27   - public SqlTsDatabaseSchemaService() {
28   - super("schema-ts.sql", null);
  29 + public HsqlTsDatabaseSchemaService() {
  30 + super("schema-ts-hsql.sql", null);
29 31 }
30 32 }
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.install;
  17 +
  18 +import org.springframework.context.annotation.Profile;
  19 +import org.springframework.stereotype.Service;
  20 +import org.thingsboard.server.dao.util.PsqlDao;
  21 +import org.thingsboard.server.dao.util.SqlTsDao;
  22 +
  23 +@Service
  24 +@SqlTsDao
  25 +@PsqlDao
  26 +@Profile("install")
  27 +public class PsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService
  28 + implements TsDatabaseSchemaService {
  29 + public PsqlTsDatabaseSchemaService() {
  30 + super("schema-ts-psql.sql", null);
  31 + }
  32 +}
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.install;
  17 +
  18 +import lombok.extern.slf4j.Slf4j;
  19 +import org.springframework.beans.factory.annotation.Autowired;
  20 +import org.springframework.beans.factory.annotation.Value;
  21 +import org.springframework.context.annotation.Profile;
  22 +import org.springframework.stereotype.Service;
  23 +import org.thingsboard.server.dao.util.PsqlDao;
  24 +import org.thingsboard.server.dao.util.SqlTsDao;
  25 +
  26 +import java.nio.charset.StandardCharsets;
  27 +import java.nio.file.Files;
  28 +import java.nio.file.Path;
  29 +import java.nio.file.Paths;
  30 +import java.sql.CallableStatement;
  31 +import java.sql.Connection;
  32 +import java.sql.DriverManager;
  33 +import java.sql.SQLException;
  34 +import java.sql.Types;
  35 +
  36 +@Service
  37 +@Profile("install")
  38 +@Slf4j
  39 +@SqlTsDao
  40 +@PsqlDao
  41 +public class PsqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService {
  42 +
  43 + private static final String CALL_REGEX = "call ";
  44 + private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql";
  45 + private static final String CHECK_VERSION = CALL_REGEX + "check_version()";
  46 + private static final String CREATE_PARTITION_TABLE = CALL_REGEX + "create_partition_table()";
  47 + private static final String CREATE_PARTITIONS = CALL_REGEX + "create_partitions()";
  48 + private static final String CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + "create_ts_kv_dictionary_table()";
  49 + private static final String INSERT_INTO_DICTIONARY = CALL_REGEX + "insert_into_dictionary()";
  50 + private static final String INSERT_INTO_TS_KV = CALL_REGEX + "insert_into_ts_kv()";
  51 + private static final String DROP_OLD_TABLE = "DROP TABLE ts_kv_old;";
  52 +
  53 + @Value("${spring.datasource.url}")
  54 + private String dbUrl;
  55 +
  56 + @Value("${spring.datasource.username}")
  57 + private String dbUserName;
  58 +
  59 + @Value("${spring.datasource.password}")
  60 + private String dbPassword;
  61 +
  62 + @Autowired
  63 + private InstallScripts installScripts;
  64 +
  65 + @Override
  66 + public void upgradeDatabase(String fromVersion) throws Exception {
  67 + switch (fromVersion) {
  68 + case "2.4.3":
  69 + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
  70 + log.info("Updating timeseries schema ...");
  71 + log.info("Load upgrade functions ...");
  72 + loadSql(conn);
  73 + log.info("Upgrade functions successfully loaded!");
  74 + boolean versionValid = checkVersion(conn);
  75 + if (!versionValid) {
  76 + log.info("PostgreSQL version should be at least more than 10!");
  77 + log.info("Please upgrade your PostgreSQL and restart the script!");
  78 + } else {
  79 + log.info("PostgreSQL version is valid!");
  80 + log.info("Updating schema ...");
  81 + executeFunction(conn, CREATE_PARTITION_TABLE);
  82 + executeFunction(conn, CREATE_PARTITIONS);
  83 + executeFunction(conn, CREATE_TS_KV_DICTIONARY_TABLE);
  84 + executeFunction(conn, INSERT_INTO_DICTIONARY);
  85 + executeFunction(conn, INSERT_INTO_TS_KV);
  86 + dropOldTable(conn, DROP_OLD_TABLE);
  87 + log.info("schema timeseries updated!");
  88 + }
  89 + }
  90 + break;
  91 + default:
  92 + throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
  93 + }
  94 + }
  95 +
  96 + private void loadSql(Connection conn) {
  97 + Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", LOAD_FUNCTIONS_SQL);
  98 + try {
  99 + loadFunctions(schemaUpdateFile, conn);
  100 + } catch (Exception e) {
  101 + log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage());
  102 + }
  103 + }
  104 +
  105 + private void loadFunctions(Path sqlFile, Connection conn) throws Exception {
  106 + String sql = new String(Files.readAllBytes(sqlFile), StandardCharsets.UTF_8);
  107 + conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
  108 + }
  109 +
  110 + private boolean checkVersion(Connection conn) {
  111 + log.info("Check the current PostgreSQL version...");
  112 + boolean versionValid = false;
  113 + try {
  114 + CallableStatement callableStatement = conn.prepareCall("{? = " + CHECK_VERSION + " }");
  115 + callableStatement.registerOutParameter(1, Types.BOOLEAN);
  116 + callableStatement.execute();
  117 + versionValid = callableStatement.getBoolean(1);
  118 + callableStatement.close();
  119 + } catch (Exception e) {
  120 + log.info("Failed to check current PostgreSQL version due to: {}", e.getMessage());
  121 + }
  122 + return versionValid;
  123 + }
  124 +
  125 + private void executeFunction(Connection conn, String query) {
  126 + log.info("{} ... ", query);
  127 + try {
  128 + CallableStatement callableStatement = conn.prepareCall("{" + query + "}");
  129 + callableStatement.execute();
  130 + callableStatement.close();
  131 + log.info("Successfully executed: {}", query.replace(CALL_REGEX, ""));
  132 + } catch (Exception e) {
  133 + log.info("Failed to execute {} due to: {}", query, e.getMessage());
  134 + }
  135 + }
  136 +
  137 + private void dropOldTable(Connection conn, String query) {
  138 + try {
  139 + conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
  140 + Thread.sleep(5000);
  141 + } catch (InterruptedException | SQLException e) {
  142 + log.info("Failed to drop table {} due to: {}", query.replace("DROP TABLE ", ""), e.getMessage());
  143 + }
  144 + }
  145 +}
\ No newline at end of file
... ...
... ... @@ -54,7 +54,7 @@ import static org.thingsboard.server.service.install.DatabaseHelper.TYPE;
54 54 @Profile("install")
55 55 @Slf4j
56 56 @SqlDao
57   -public class SqlDatabaseUpgradeService implements DatabaseUpgradeService {
  57 +public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService {
58 58
59 59 private static final String SCHEMA_UPDATE_SQL = "schema_update.sql";
60 60
... ... @@ -172,7 +172,8 @@ public class SqlDatabaseUpgradeService implements DatabaseUpgradeService {
172 172 loadSql(schemaUpdateFile, conn);
173 173 try {
174 174 conn.createStatement().execute("ALTER TABLE device ADD COLUMN label varchar(255)"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
175   - } catch (Exception e) {}
  175 + } catch (Exception e) {
  176 + }
176 177 log.info("Schema updated.");
177 178 }
178 179 break;
... ... @@ -201,7 +202,8 @@ public class SqlDatabaseUpgradeService implements DatabaseUpgradeService {
201 202 log.info("Updating schema ...");
202 203 try {
203 204 conn.createStatement().execute("ALTER TABLE alarm ADD COLUMN propagate_relation_types varchar"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
204   - } catch (Exception e) {}
  205 + } catch (Exception e) {
  206 + }
205 207 log.info("Schema updated.");
206 208 }
207 209 break;
... ...
... ... @@ -171,7 +171,7 @@ cassandra:
171 171 read_consistency_level: "${CASSANDRA_READ_CONSISTENCY_LEVEL:ONE}"
172 172 write_consistency_level: "${CASSANDRA_WRITE_CONSISTENCY_LEVEL:ONE}"
173 173 default_fetch_size: "${CASSANDRA_DEFAULT_FETCH_SIZE:2000}"
174   - # Specify partitioning size for timestamp key-value storage. Example MINUTES, HOURS, DAYS, MONTHS,INDEFINITE
  174 + # Specify partitioning size for timestamp key-value storage. Example: MINUTES, HOURS, DAYS, MONTHS,INDEFINITE
175 175 ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}"
176 176 ts_key_value_ttl: "${TS_KV_TTL:0}"
177 177 events_ttl: "${TS_EVENTS_TTL:0}"
... ... @@ -214,6 +214,8 @@ sql:
214 214 stats_print_interval_ms: "${SQL_TS_TIMESCALE_BATCH_STATS_PRINT_MS:10000}"
215 215 # Specify whether to remove null characters from strValue of attributes and timeseries before insert
216 216 remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}"
  217 + # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE
  218 + ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}"
217 219
218 220 # Actor system parameters
219 221 actors:
... ...
... ... @@ -30,7 +30,7 @@ public class ControllerSqlTestSuite {
30 30
31 31 @ClassRule
32 32 public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
33   - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"),
  33 + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"),
34 34 "sql/drop-all-tables.sql",
35 35 "sql-test.properties");
36 36 }
... ...
... ... @@ -29,7 +29,7 @@ public class MqttSqlTestSuite {
29 29
30 30 @ClassRule
31 31 public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
32   - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
  32 + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
33 33 "sql/drop-all-tables.sql",
34 34 "sql-test.properties");
35 35 }
... ...
... ... @@ -30,7 +30,7 @@ public class RuleEngineSqlTestSuite {
30 30
31 31 @ClassRule
32 32 public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
33   - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
  33 + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
34 34 "sql/drop-all-tables.sql",
35 35 "sql-test.properties");
36 36 }
... ...
... ... @@ -31,7 +31,7 @@ public class SystemSqlTestSuite {
31 31
32 32 @ClassRule
33 33 public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
34   - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
  34 + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
35 35 "sql/drop-all-tables.sql",
36 36 "sql-test.properties");
37 37
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.util;
  17 +
  18 +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  19 +
  20 +@ConditionalOnExpression("('${database.ts.type}'=='sql' || '${database.entities.type}'=='timescale') " +
  21 + "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'")
  22 +public @interface PsqlTsAnyDao {
  23 +}
... ...
dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java renamed from dao/src/main/java/org/thingsboard/server/dao/SqlTsDaoConfig.java
... ... @@ -21,15 +21,17 @@ import org.springframework.context.annotation.ComponentScan;
21 21 import org.springframework.context.annotation.Configuration;
22 22 import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
23 23 import org.springframework.transaction.annotation.EnableTransactionManagement;
  24 +import org.thingsboard.server.dao.util.HsqlDao;
24 25 import org.thingsboard.server.dao.util.SqlTsDao;
25 26
26 27 @Configuration
27 28 @EnableAutoConfiguration
28   -@ComponentScan("org.thingsboard.server.dao.sqlts.ts")
29   -@EnableJpaRepositories("org.thingsboard.server.dao.sqlts.ts")
30   -@EntityScan("org.thingsboard.server.dao.model.sqlts.ts")
  29 +@ComponentScan({"org.thingsboard.server.dao.sqlts.hsql", "org.thingsboard.server.dao.sqlts.latest"})
  30 +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.hsql", "org.thingsboard.server.dao.sqlts.latest"})
  31 +@EntityScan({"org.thingsboard.server.dao.model.sqlts.hsql", "org.thingsboard.server.dao.model.sqlts.latest"})
31 32 @EnableTransactionManagement
32 33 @SqlTsDao
33   -public class SqlTsDaoConfig {
  34 +@HsqlDao
  35 +public class HsqlTsDaoConfig {
34 36
35 37 }
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao;
  17 +
  18 +import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
  19 +import org.springframework.boot.autoconfigure.domain.EntityScan;
  20 +import org.springframework.context.annotation.ComponentScan;
  21 +import org.springframework.context.annotation.Configuration;
  22 +import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
  23 +import org.springframework.transaction.annotation.EnableTransactionManagement;
  24 +import org.thingsboard.server.dao.util.PsqlDao;
  25 +import org.thingsboard.server.dao.util.SqlTsDao;
  26 +
  27 +@Configuration
  28 +@EnableAutoConfiguration
  29 +@ComponentScan({"org.thingsboard.server.dao.sqlts.psql", "org.thingsboard.server.dao.sqlts.latest"})
  30 +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.psql", "org.thingsboard.server.dao.sqlts.latest", "org.thingsboard.server.dao.sqlts.dictionary"})
  31 +@EntityScan({"org.thingsboard.server.dao.model.sqlts.psql", "org.thingsboard.server.dao.model.sqlts.latest", "org.thingsboard.server.dao.model.sqlts.dictionary"})
  32 +@EnableTransactionManagement
  33 +@SqlTsDao
  34 +@PsqlDao
  35 +public class PsqlTsDaoConfig {
  36 +
  37 +}
\ No newline at end of file
... ...
... ... @@ -25,9 +25,9 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao;
25 25
26 26 @Configuration
27 27 @EnableAutoConfiguration
28   -@ComponentScan("org.thingsboard.server.dao.sqlts.timescale")
29   -@EnableJpaRepositories("org.thingsboard.server.dao.sqlts.timescale")
30   -@EntityScan("org.thingsboard.server.dao.model.sqlts.timescale")
  28 +@ComponentScan({"org.thingsboard.server.dao.sqlts.timescale", "org.thingsboard.server.dao.sqlts.latest"})
  29 +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.timescale", "org.thingsboard.server.dao.sqlts.dictionary", "org.thingsboard.server.dao.sqlts.latest"})
  30 +@EntityScan({"org.thingsboard.server.dao.model.sqlts.timescale", "org.thingsboard.server.dao.model.sqlts.dictionary", "org.thingsboard.server.dao.model.sqlts.latest"})
31 31 @EnableTransactionManagement
32 32 @TimescaleDBTsDao
33 33 public class TimescaleDaoConfig {
... ...
... ... @@ -41,7 +41,7 @@ import org.thingsboard.server.dao.DaoUtil;
41 41 import org.thingsboard.server.dao.model.ModelConstants;
42 42 import org.thingsboard.server.dao.model.nosql.AuditLogEntity;
43 43 import org.thingsboard.server.dao.nosql.CassandraAbstractSearchTimeDao;
44   -import org.thingsboard.server.dao.timeseries.TsPartitionDate;
  44 +import org.thingsboard.server.dao.timeseries.NoSqlTsPartitionDate;
45 45 import org.thingsboard.server.dao.util.NoSqlDao;
46 46
47 47 import javax.annotation.Nullable;
... ... @@ -92,7 +92,7 @@ public class CassandraAuditLogDao extends CassandraAbstractSearchTimeDao<AuditLo
92 92
93 93 @Value("${audit-log.by_tenant_partitioning}")
94 94 private String partitioning;
95   - private TsPartitionDate tsFormat;
  95 + private NoSqlTsPartitionDate tsFormat;
96 96
97 97 @Value("${audit-log.default_query_period}")
98 98 private Integer defaultQueryPeriodInDays;
... ... @@ -110,7 +110,7 @@ public class CassandraAuditLogDao extends CassandraAbstractSearchTimeDao<AuditLo
110 110 @PostConstruct
111 111 public void init() {
112 112 if (!isInstall()) {
113   - Optional<TsPartitionDate> partition = TsPartitionDate.parse(partitioning);
  113 + Optional<NoSqlTsPartitionDate> partition = NoSqlTsPartitionDate.parse(partitioning);
114 114 if (partition.isPresent()) {
115 115 tsFormat = partition.get();
116 116 } else {
... ...
... ... @@ -359,6 +359,7 @@ public class ModelConstants {
359 359
360 360 public static final String PARTITION_COLUMN = "partition";
361 361 public static final String KEY_COLUMN = "key";
  362 + public static final String KEY_ID_COLUMN = "key_id";
362 363 public static final String TS_COLUMN = "ts";
363 364
364 365 /**
... ...
... ... @@ -16,11 +16,6 @@
16 16 package org.thingsboard.server.dao.model.sql;
17 17
18 18 import lombok.Data;
19   -import org.thingsboard.server.common.data.kv.BooleanDataEntry;
20   -import org.thingsboard.server.common.data.kv.DoubleDataEntry;
21   -import org.thingsboard.server.common.data.kv.KvEntry;
22   -import org.thingsboard.server.common.data.kv.LongDataEntry;
23   -import org.thingsboard.server.common.data.kv.StringDataEntry;
24 19
25 20 import javax.persistence.Column;
26 21 import javax.persistence.Id;
... ... @@ -28,10 +23,9 @@ import javax.persistence.MappedSuperclass;
28 23
29 24 import static org.thingsboard.server.dao.model.ModelConstants.BOOLEAN_VALUE_COLUMN;
30 25 import static org.thingsboard.server.dao.model.ModelConstants.DOUBLE_VALUE_COLUMN;
31   -import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN;
32   -import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
33 26 import static org.thingsboard.server.dao.model.ModelConstants.LONG_VALUE_COLUMN;
34 27 import static org.thingsboard.server.dao.model.ModelConstants.STRING_VALUE_COLUMN;
  28 +import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN;
35 29
36 30 @Data
37 31 @MappedSuperclass
... ... @@ -43,12 +37,8 @@ public abstract class AbstractTsKvEntity {
43 37 protected static final String MAX = "MAX";
44 38
45 39 @Id
46   - @Column(name = ENTITY_ID_COLUMN)
47   - protected String entityId;
48   -
49   - @Id
50   - @Column(name = KEY_COLUMN)
51   - protected String key;
  40 + @Column(name = TS_COLUMN)
  41 + protected Long ts;
52 42
53 43 @Column(name = BOOLEAN_VALUE_COLUMN)
54 44 protected Boolean booleanValue;
... ... @@ -62,20 +52,6 @@ public abstract class AbstractTsKvEntity {
62 52 @Column(name = DOUBLE_VALUE_COLUMN)
63 53 protected Double doubleValue;
64 54
65   - protected KvEntry getKvEntry() {
66   - KvEntry kvEntry = null;
67   - if (strValue != null) {
68   - kvEntry = new StringDataEntry(key, strValue);
69   - } else if (longValue != null) {
70   - kvEntry = new LongDataEntry(key, longValue);
71   - } else if (doubleValue != null) {
72   - kvEntry = new DoubleDataEntry(key, doubleValue);
73   - } else if (booleanValue != null) {
74   - kvEntry = new BooleanDataEntry(key, booleanValue);
75   - }
76   - return kvEntry;
77   - }
78   -
79 55 public abstract boolean isNotEmpty();
80 56
81 57 protected static boolean isAllNull(Object... args) {
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.model.sqlts.dictionary;
  17 +
  18 +import lombok.Data;
  19 +import org.hibernate.annotations.Generated;
  20 +import org.hibernate.annotations.GenerationTime;
  21 +
  22 +import javax.persistence.Column;
  23 +import javax.persistence.Entity;
  24 +import javax.persistence.Id;
  25 +import javax.persistence.IdClass;
  26 +import javax.persistence.Table;
  27 +
  28 +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
  29 +import static org.thingsboard.server.dao.model.ModelConstants.KEY_ID_COLUMN;
  30 +
  31 +@Data
  32 +@Entity
  33 +@Table(name = "ts_kv_dictionary")
  34 +@IdClass(TsKvDictionaryCompositeKey.class)
  35 +public final class TsKvDictionary {
  36 +
  37 + @Id
  38 + @Column(name = KEY_COLUMN)
  39 + private String key;
  40 +
  41 + @Column(name = KEY_ID_COLUMN, unique = true, columnDefinition="serial")
  42 + @Generated(GenerationTime.INSERT)
  43 + private int keyId;
  44 +
  45 +}
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.model.sqlts.dictionary;
  17 +
  18 +import lombok.AllArgsConstructor;
  19 +import lombok.Data;
  20 +import lombok.NoArgsConstructor;
  21 +
  22 +import javax.persistence.Transient;
  23 +import java.io.Serializable;
  24 +
  25 +@Data
  26 +@NoArgsConstructor
  27 +@AllArgsConstructor
  28 +public class TsKvDictionaryCompositeKey implements Serializable{
  29 +
  30 + @Transient
  31 + private static final long serialVersionUID = -4089175869616037523L;
  32 +
  33 + private String key;
  34 +}
\ No newline at end of file
... ...
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/hsql/TsKvCompositeKey.java renamed from dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvCompositeKey.java
... ... @@ -13,7 +13,7 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.dao.model.sqlts.ts;
  16 +package org.thingsboard.server.dao.model.sqlts.hsql;
17 17
18 18 import lombok.AllArgsConstructor;
19 19 import lombok.Data;
... ... @@ -35,4 +35,5 @@ public class TsKvCompositeKey implements Serializable {
35 35 private String entityId;
36 36 private String key;
37 37 private long ts;
  38 +
38 39 }
\ No newline at end of file
... ...
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/hsql/TsKvEntity.java renamed from dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvEntity.java
... ... @@ -13,11 +13,16 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.dao.model.sqlts.ts;
  16 +package org.thingsboard.server.dao.model.sqlts.hsql;
17 17
18 18 import lombok.Data;
19 19 import org.thingsboard.server.common.data.EntityType;
20 20 import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
  21 +import org.thingsboard.server.common.data.kv.BooleanDataEntry;
  22 +import org.thingsboard.server.common.data.kv.DoubleDataEntry;
  23 +import org.thingsboard.server.common.data.kv.KvEntry;
  24 +import org.thingsboard.server.common.data.kv.LongDataEntry;
  25 +import org.thingsboard.server.common.data.kv.StringDataEntry;
21 26 import org.thingsboard.server.common.data.kv.TsKvEntry;
22 27 import org.thingsboard.server.dao.model.ToData;
23 28 import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity;
... ... @@ -30,8 +35,9 @@ import javax.persistence.Id;
30 35 import javax.persistence.IdClass;
31 36 import javax.persistence.Table;
32 37
  38 +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN;
33 39 import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_TYPE_COLUMN;
34   -import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN;
  40 +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
35 41
36 42 @Data
37 43 @Entity
... ... @@ -45,8 +51,12 @@ public final class TsKvEntity extends AbstractTsKvEntity implements ToData<TsKvE
45 51 private EntityType entityType;
46 52
47 53 @Id
48   - @Column(name = TS_COLUMN)
49   - protected Long ts;
  54 + @Column(name = ENTITY_ID_COLUMN)
  55 + private String entityId;
  56 +
  57 + @Id
  58 + @Column(name = KEY_COLUMN)
  59 + private String key;
50 60
51 61 public TsKvEntity() {
52 62 }
... ... @@ -113,6 +123,16 @@ public final class TsKvEntity extends AbstractTsKvEntity implements ToData<TsKvE
113 123
114 124 @Override
115 125 public TsKvEntry toData() {
116   - return new BasicTsKvEntry(ts, getKvEntry());
  126 + KvEntry kvEntry = null;
  127 + if (strValue != null) {
  128 + kvEntry = new StringDataEntry(key, strValue);
  129 + } else if (longValue != null) {
  130 + kvEntry = new LongDataEntry(key, longValue);
  131 + } else if (doubleValue != null) {
  132 + kvEntry = new DoubleDataEntry(key, doubleValue);
  133 + } else if (booleanValue != null) {
  134 + kvEntry = new BooleanDataEntry(key, booleanValue);
  135 + }
  136 + return new BasicTsKvEntry(ts, kvEntry);
117 137 }
118   -}
  138 +}
\ No newline at end of file
... ...
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestCompositeKey.java renamed from dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvLatestCompositeKey.java
... ... @@ -13,7 +13,7 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.dao.model.sqlts.ts;
  16 +package org.thingsboard.server.dao.model.sqlts.latest;
17 17
18 18 import lombok.AllArgsConstructor;
19 19 import lombok.Data;
... ...
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java renamed from dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvLatestEntity.java
... ... @@ -13,11 +13,16 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.dao.model.sqlts.ts;
  16 +package org.thingsboard.server.dao.model.sqlts.latest;
17 17
18 18 import lombok.Data;
19 19 import org.thingsboard.server.common.data.EntityType;
20 20 import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
  21 +import org.thingsboard.server.common.data.kv.BooleanDataEntry;
  22 +import org.thingsboard.server.common.data.kv.DoubleDataEntry;
  23 +import org.thingsboard.server.common.data.kv.KvEntry;
  24 +import org.thingsboard.server.common.data.kv.LongDataEntry;
  25 +import org.thingsboard.server.common.data.kv.StringDataEntry;
21 26 import org.thingsboard.server.common.data.kv.TsKvEntry;
22 27 import org.thingsboard.server.dao.model.ToData;
23 28 import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity;
... ... @@ -30,8 +35,9 @@ import javax.persistence.Id;
30 35 import javax.persistence.IdClass;
31 36 import javax.persistence.Table;
32 37
  38 +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN;
33 39 import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_TYPE_COLUMN;
34   -import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN;
  40 +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
35 41
36 42 @Data
37 43 @Entity
... ... @@ -44,16 +50,32 @@ public final class TsKvLatestEntity extends AbstractTsKvEntity implements ToData
44 50 @Column(name = ENTITY_TYPE_COLUMN)
45 51 private EntityType entityType;
46 52
47   - @Column(name = TS_COLUMN)
48   - private long ts;
  53 + @Id
  54 + @Column(name = ENTITY_ID_COLUMN)
  55 + private String entityId;
49 56
50   - @Override
51   - public TsKvEntry toData() {
52   - return new BasicTsKvEntry(ts, getKvEntry());
53   - }
  57 + @Id
  58 + @Column(name = KEY_COLUMN)
  59 + private String key;
54 60
55 61 @Override
56 62 public boolean isNotEmpty() {
57 63 return strValue != null || longValue != null || doubleValue != null || booleanValue != null;
58 64 }
  65 +
  66 + @Override
  67 + public TsKvEntry toData() {
  68 + KvEntry kvEntry = null;
  69 + if (strValue != null) {
  70 + kvEntry = new StringDataEntry(key, strValue);
  71 + } else if (longValue != null) {
  72 + kvEntry = new LongDataEntry(key, longValue);
  73 + } else if (doubleValue != null) {
  74 + kvEntry = new DoubleDataEntry(key, doubleValue);
  75 + } else if (booleanValue != null) {
  76 + kvEntry = new BooleanDataEntry(key, booleanValue);
  77 + }
  78 + return new BasicTsKvEntry(ts, kvEntry);
  79 + }
  80 +
59 81 }
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.model.sqlts.psql;
  17 +
  18 +import lombok.AllArgsConstructor;
  19 +import lombok.Data;
  20 +import lombok.NoArgsConstructor;
  21 +
  22 +import javax.persistence.Transient;
  23 +import java.io.Serializable;
  24 +import java.util.UUID;
  25 +
  26 +@Data
  27 +@AllArgsConstructor
  28 +@NoArgsConstructor
  29 +public class TsKvCompositeKey implements Serializable {
  30 +
  31 + @Transient
  32 + private static final long serialVersionUID = -4089175869616037523L;
  33 +
  34 + private UUID entityId;
  35 + private int key;
  36 + private long ts;
  37 +}
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.model.sqlts.psql;
  17 +
  18 +import lombok.Data;
  19 +import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
  20 +import org.thingsboard.server.common.data.kv.BooleanDataEntry;
  21 +import org.thingsboard.server.common.data.kv.DoubleDataEntry;
  22 +import org.thingsboard.server.common.data.kv.KvEntry;
  23 +import org.thingsboard.server.common.data.kv.LongDataEntry;
  24 +import org.thingsboard.server.common.data.kv.StringDataEntry;
  25 +import org.thingsboard.server.common.data.kv.TsKvEntry;
  26 +import org.thingsboard.server.dao.model.ToData;
  27 +import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity;
  28 +
  29 +import javax.persistence.Column;
  30 +import javax.persistence.Entity;
  31 +import javax.persistence.Id;
  32 +import javax.persistence.IdClass;
  33 +import javax.persistence.Table;
  34 +import javax.persistence.Transient;
  35 +import java.util.UUID;
  36 +
  37 +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN;
  38 +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
  39 +
  40 +@Data
  41 +@Entity
  42 +@Table(name = "ts_kv")
  43 +@IdClass(TsKvCompositeKey.class)
  44 +public final class TsKvEntity extends AbstractTsKvEntity implements ToData<TsKvEntry> {
  45 +
  46 + @Id
  47 + @Column(name = ENTITY_ID_COLUMN, columnDefinition = "uuid")
  48 + protected UUID entityId;
  49 +
  50 + @Id
  51 + @Column(name = KEY_COLUMN)
  52 + protected int key;
  53 +
  54 + @Transient
  55 + protected String strKey;
  56 +
  57 + public TsKvEntity() {
  58 + }
  59 +
  60 + public TsKvEntity(String strValue) {
  61 + this.strValue = strValue;
  62 + }
  63 +
  64 + public TsKvEntity(Long longValue, Double doubleValue, Long longCountValue, Long doubleCountValue, String aggType) {
  65 + if (!isAllNull(longValue, doubleValue, longCountValue, doubleCountValue)) {
  66 + switch (aggType) {
  67 + case AVG:
  68 + double sum = 0.0;
  69 + if (longValue != null) {
  70 + sum += longValue;
  71 + }
  72 + if (doubleValue != null) {
  73 + sum += doubleValue;
  74 + }
  75 + long totalCount = longCountValue + doubleCountValue;
  76 + if (totalCount > 0) {
  77 + this.doubleValue = sum / (longCountValue + doubleCountValue);
  78 + } else {
  79 + this.doubleValue = 0.0;
  80 + }
  81 + break;
  82 + case SUM:
  83 + if (doubleCountValue > 0) {
  84 + this.doubleValue = doubleValue + (longValue != null ? longValue.doubleValue() : 0.0);
  85 + } else {
  86 + this.longValue = longValue;
  87 + }
  88 + break;
  89 + case MIN:
  90 + case MAX:
  91 + if (longCountValue > 0 && doubleCountValue > 0) {
  92 + this.doubleValue = MAX.equals(aggType) ? Math.max(doubleValue, longValue.doubleValue()) : Math.min(doubleValue, longValue.doubleValue());
  93 + } else if (doubleCountValue > 0) {
  94 + this.doubleValue = doubleValue;
  95 + } else if (longCountValue > 0) {
  96 + this.longValue = longValue;
  97 + }
  98 + break;
  99 + }
  100 + }
  101 + }
  102 +
  103 + public TsKvEntity(Long booleanValueCount, Long strValueCount, Long longValueCount, Long doubleValueCount) {
  104 + if (!isAllNull(booleanValueCount, strValueCount, longValueCount, doubleValueCount)) {
  105 + if (booleanValueCount != 0) {
  106 + this.longValue = booleanValueCount;
  107 + } else if (strValueCount != 0) {
  108 + this.longValue = strValueCount;
  109 + } else {
  110 + this.longValue = longValueCount + doubleValueCount;
  111 + }
  112 + }
  113 + }
  114 +
  115 + @Override
  116 + public boolean isNotEmpty() {
  117 + return strValue != null || longValue != null || doubleValue != null || booleanValue != null;
  118 + }
  119 +
  120 + @Override
  121 + public TsKvEntry toData() {
  122 + KvEntry kvEntry = null;
  123 + if (strValue != null) {
  124 + kvEntry = new StringDataEntry(strKey, strValue);
  125 + } else if (longValue != null) {
  126 + kvEntry = new LongDataEntry(strKey, longValue);
  127 + } else if (doubleValue != null) {
  128 + kvEntry = new DoubleDataEntry(strKey, doubleValue);
  129 + } else if (booleanValue != null) {
  130 + kvEntry = new BooleanDataEntry(strKey, booleanValue);
  131 + }
  132 + return new BasicTsKvEntry(ts, kvEntry);
  133 + }
  134 +
  135 +}
... ...
... ... @@ -21,6 +21,7 @@ import lombok.NoArgsConstructor;
21 21
22 22 import javax.persistence.Transient;
23 23 import java.io.Serializable;
  24 +import java.util.UUID;
24 25
25 26 @Data
26 27 @AllArgsConstructor
... ... @@ -30,8 +31,8 @@ public class TimescaleTsKvCompositeKey implements Serializable {
30 31 @Transient
31 32 private static final long serialVersionUID = -4089175869616037523L;
32 33
33   - private String tenantId;
34   - private String entityId;
35   - private String key;
  34 + private UUID tenantId;
  35 + private UUID entityId;
  36 + private int key;
36 37 private long ts;
37 38 }
\ No newline at end of file
... ...
... ... @@ -19,6 +19,11 @@ import lombok.Data;
19 19 import lombok.EqualsAndHashCode;
20 20 import org.springframework.util.StringUtils;
21 21 import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
  22 +import org.thingsboard.server.common.data.kv.BooleanDataEntry;
  23 +import org.thingsboard.server.common.data.kv.DoubleDataEntry;
  24 +import org.thingsboard.server.common.data.kv.KvEntry;
  25 +import org.thingsboard.server.common.data.kv.LongDataEntry;
  26 +import org.thingsboard.server.common.data.kv.StringDataEntry;
22 27 import org.thingsboard.server.common.data.kv.TsKvEntry;
23 28 import org.thingsboard.server.dao.model.ToData;
24 29 import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity;
... ... @@ -34,9 +39,12 @@ import javax.persistence.NamedNativeQuery;
34 39 import javax.persistence.SqlResultSetMapping;
35 40 import javax.persistence.SqlResultSetMappings;
36 41 import javax.persistence.Table;
  42 +import javax.persistence.Transient;
  43 +import java.util.UUID;
37 44
  45 +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN;
  46 +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
38 47 import static org.thingsboard.server.dao.model.ModelConstants.TENANT_ID_COLUMN;
39   -import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN;
40 48 import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_AVG;
41 49 import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_AVG_QUERY;
42 50 import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_COUNT;
... ... @@ -118,21 +126,30 @@ import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.F
118 126 public final class TimescaleTsKvEntity extends AbstractTsKvEntity implements ToData<TsKvEntry> {
119 127
120 128 @Id
121   - @Column(name = TENANT_ID_COLUMN)
122   - private String tenantId;
  129 + @Column(name = TENANT_ID_COLUMN, columnDefinition = "uuid")
  130 + private UUID tenantId;
123 131
124 132 @Id
125   - @Column(name = TS_COLUMN)
126   - protected Long ts;
  133 + @Column(name = ENTITY_ID_COLUMN, columnDefinition = "uuid")
  134 + protected UUID entityId;
127 135
128   - public TimescaleTsKvEntity() { }
  136 + @Id
  137 + @Column(name = KEY_COLUMN)
  138 + protected int key;
  139 +
  140 + @Transient
  141 + protected String strKey;
  142 +
  143 +
  144 + public TimescaleTsKvEntity() {
  145 + }
129 146
130 147 public TimescaleTsKvEntity(Long tsBucket, Long interval, Long longValue, Double doubleValue, Long longCountValue, Long doubleCountValue, String strValue, String aggType) {
131 148 if (!StringUtils.isEmpty(strValue)) {
132 149 this.strValue = strValue;
133 150 }
134 151 if (!isAllNull(tsBucket, interval, longValue, doubleValue, longCountValue, doubleCountValue)) {
135   - this.ts = tsBucket + interval/2;
  152 + this.ts = tsBucket + interval / 2;
136 153 switch (aggType) {
137 154 case AVG:
138 155 double sum = 0.0;
... ... @@ -172,7 +189,7 @@ public final class TimescaleTsKvEntity extends AbstractTsKvEntity implements ToD
172 189
173 190 public TimescaleTsKvEntity(Long tsBucket, Long interval, Long booleanValueCount, Long strValueCount, Long longValueCount, Long doubleValueCount) {
174 191 if (!isAllNull(tsBucket, interval, booleanValueCount, strValueCount, longValueCount, doubleValueCount)) {
175   - this.ts = tsBucket + interval/2;
  192 + this.ts = tsBucket + interval / 2;
176 193 if (booleanValueCount != 0) {
177 194 this.longValue = booleanValueCount;
178 195 } else if (strValueCount != 0) {
... ... @@ -190,6 +207,17 @@ public final class TimescaleTsKvEntity extends AbstractTsKvEntity implements ToD
190 207
191 208 @Override
192 209 public TsKvEntry toData() {
193   - return new BasicTsKvEntry(ts, getKvEntry());
  210 + KvEntry kvEntry = null;
  211 + if (strValue != null) {
  212 + kvEntry = new StringDataEntry(strKey, strValue);
  213 + } else if (longValue != null) {
  214 + kvEntry = new LongDataEntry(strKey, longValue);
  215 + } else if (doubleValue != null) {
  216 + kvEntry = new DoubleDataEntry(strKey, doubleValue);
  217 + } else if (booleanValue != null) {
  218 + kvEntry = new BooleanDataEntry(strKey, booleanValue);
  219 + }
  220 + return new BasicTsKvEntry(ts, kvEntry);
194 221 }
  222 +
195 223 }
\ No newline at end of file
... ...
... ... @@ -15,13 +15,8 @@
15 15 */
16 16 package org.thingsboard.server.dao.sql;
17 17
18   -import com.google.common.util.concurrent.ListeningExecutorService;
19   -import com.google.common.util.concurrent.MoreExecutors;
20 18 import org.springframework.beans.factory.annotation.Autowired;
21 19
22   -import javax.annotation.PreDestroy;
23   -import java.util.concurrent.Executors;
24   -
25 20 public abstract class JpaAbstractDaoListeningExecutorService {
26 21
27 22 @Autowired
... ...
... ... @@ -21,8 +21,6 @@ import org.springframework.jdbc.core.JdbcTemplate;
21 21 import org.springframework.stereotype.Repository;
22 22 import org.springframework.transaction.support.TransactionTemplate;
23 23
24   -import javax.persistence.EntityManager;
25   -import javax.persistence.PersistenceContext;
26 24 import java.util.regex.Pattern;
27 25
28 26 @Repository
... ... @@ -31,64 +29,15 @@ public abstract class AbstractInsertRepository {
31 29 private static final ThreadLocal<Pattern> PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE)));
32 30 private static final String EMPTY_STR = "";
33 31
34   - protected static final String BOOL_V = "bool_v";
35   - protected static final String STR_V = "str_v";
36   - protected static final String LONG_V = "long_v";
37   - protected static final String DBL_V = "dbl_v";
38   -
39   - protected static final String TS_KV_LATEST_TABLE = "ts_kv_latest";
40   - protected static final String TS_KV_TABLE = "ts_kv";
41   -
42   - protected static final String HSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_TABLE, BOOL_V);
43   - protected static final String HSQL_ON_STR_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_TABLE, STR_V);
44   - protected static final String HSQL_ON_LONG_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_TABLE, LONG_V);
45   - protected static final String HSQL_ON_DBL_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_TABLE, DBL_V);
46   -
47   - protected static final String HSQL_LATEST_ON_BOOL_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_LATEST_TABLE, BOOL_V);
48   - protected static final String HSQL_LATEST_ON_STR_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_LATEST_TABLE, STR_V);
49   - protected static final String HSQL_LATEST_ON_LONG_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_LATEST_TABLE, LONG_V);
50   - protected static final String HSQL_LATEST_ON_DBL_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_LATEST_TABLE, DBL_V);
51   -
52   - protected static final String PSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS = "str_v = null, long_v = null, dbl_v = null";
53   - protected static final String PSQL_ON_STR_VALUE_UPDATE_SET_NULLS = "bool_v = null, long_v = null, dbl_v = null";
54   - protected static final String PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS = "str_v = null, bool_v = null, dbl_v = null";
55   - protected static final String PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS = "str_v = null, long_v = null, bool_v = null";
56   -
57 32 @Value("${sql.remove_null_chars}")
58 33 private boolean removeNullChars;
59 34
60   - @PersistenceContext
61   - protected EntityManager entityManager;
62   -
63 35 @Autowired
64 36 protected JdbcTemplate jdbcTemplate;
65 37
66 38 @Autowired
67 39 protected TransactionTemplate transactionTemplate;
68 40
69   - protected static String getInsertOrUpdateStringHsql(String tableName, String constraint, String value, String nullValues) {
70   - return "MERGE INTO " + tableName + " USING(VALUES :entity_type, :entity_id, :key, :ts, :" + value + ") A (entity_type, entity_id, key, ts, " + value + ") ON " + constraint + " WHEN MATCHED THEN UPDATE SET " + tableName + "." + value + " = A." + value + ", " + tableName + ".ts = A.ts," + nullValues + "WHEN NOT MATCHED THEN INSERT (entity_type, entity_id, key, ts, " + value + ") VALUES (A.entity_type, A.entity_id, A.key, A.ts, A." + value + ")";
71   - }
72   -
73   - protected static String getInsertOrUpdateStringPsql(String tableName, String constraint, String value, String nullValues) {
74   - return "INSERT INTO " + tableName + " (entity_type, entity_id, key, ts, " + value + ") VALUES (:entity_type, :entity_id, :key, :ts, :" + value + ") ON CONFLICT " + constraint + " DO UPDATE SET " + value + " = :" + value + ", ts = :ts," + nullValues;
75   - }
76   -
77   - private static String getHsqlNullValues(String tableName, String notNullValue) {
78   - switch (notNullValue) {
79   - case BOOL_V:
80   - return " " + tableName + ".str_v = null, " + tableName + ".long_v = null, " + tableName + ".dbl_v = null ";
81   - case STR_V:
82   - return " " + tableName + ".bool_v = null, " + tableName + ".long_v = null, " + tableName + ".dbl_v = null ";
83   - case LONG_V:
84   - return " " + tableName + ".str_v = null, " + tableName + ".bool_v = null, " + tableName + ".dbl_v = null ";
85   - case DBL_V:
86   - return " " + tableName + ".str_v = null, " + tableName + ".long_v = null, " + tableName + ".bool_v = null ";
87   - default:
88   - throw new RuntimeException("Unsupported insert value: [" + notNullValue + "]");
89   - }
90   - }
91   -
92 41 protected String replaceNullChars(String strValue) {
93 42 if (removeNullChars && strValue != null) {
94 43 return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR);
... ...
1   -/**
2   - * Copyright © 2016-2020 The Thingsboard Authors
3   - *
4   - * Licensed under the Apache License, Version 2.0 (the "License");
5   - * you may not use this file except in compliance with the License.
6   - * You may obtain a copy of the License at
7   - *
8   - * http://www.apache.org/licenses/LICENSE-2.0
9   - *
10   - * Unless required by applicable law or agreed to in writing, software
11   - * distributed under the License is distributed on an "AS IS" BASIS,
12   - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   - * See the License for the specific language governing permissions and
14   - * limitations under the License.
15   - */
16   -package org.thingsboard.server.dao.sqlts;
17   -
18   -import org.springframework.data.jpa.repository.Modifying;
19   -import org.springframework.stereotype.Repository;
20   -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity;
21   -
22   -import java.util.List;
23   -
24   -@Repository
25   -public abstract class AbstractLatestInsertRepository extends AbstractInsertRepository {
26   -
27   - public abstract void saveOrUpdate(TsKvLatestEntity entity);
28   -
29   - public abstract void saveOrUpdate(List<TsKvLatestEntity> entities);
30   -
31   - protected void processSaveOrUpdate(TsKvLatestEntity entity, String requestBoolValue, String requestStrValue, String requestLongValue, String requestDblValue) {
32   - if (entity.getBooleanValue() != null) {
33   - saveOrUpdateBoolean(entity, requestBoolValue);
34   - }
35   - if (entity.getStrValue() != null) {
36   - saveOrUpdateString(entity, requestStrValue);
37   - }
38   - if (entity.getLongValue() != null) {
39   - saveOrUpdateLong(entity, requestLongValue);
40   - }
41   - if (entity.getDoubleValue() != null) {
42   - saveOrUpdateDouble(entity, requestDblValue);
43   - }
44   - }
45   -
46   - @Modifying
47   - protected abstract void saveOrUpdateBoolean(TsKvLatestEntity entity, String query);
48   -
49   - @Modifying
50   - protected abstract void saveOrUpdateString(TsKvLatestEntity entity, String query);
51   -
52   - @Modifying
53   - protected abstract void saveOrUpdateLong(TsKvLatestEntity entity, String query);
54   -
55   - @Modifying
56   - protected abstract void saveOrUpdateDouble(TsKvLatestEntity entity, String query);
57   -
58   -}
\ No newline at end of file
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.sqlts;
  17 +
  18 +import com.google.common.util.concurrent.Futures;
  19 +import com.google.common.util.concurrent.ListenableFuture;
  20 +import com.google.common.util.concurrent.SettableFuture;
  21 +import lombok.extern.slf4j.Slf4j;
  22 +import org.springframework.beans.factory.annotation.Autowired;
  23 +import org.springframework.beans.factory.annotation.Value;
  24 +import org.thingsboard.server.common.data.id.EntityId;
  25 +import org.thingsboard.server.common.data.id.TenantId;
  26 +import org.thingsboard.server.common.data.kv.Aggregation;
  27 +import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
  28 +import org.thingsboard.server.common.data.kv.TsKvEntry;
  29 +import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity;
  30 +import org.thingsboard.server.dao.sql.TbSqlBlockingQueue;
  31 +import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams;
  32 +
  33 +import javax.annotation.PostConstruct;
  34 +import javax.annotation.PreDestroy;
  35 +import java.util.ArrayList;
  36 +import java.util.List;
  37 +import java.util.Optional;
  38 +import java.util.concurrent.CompletableFuture;
  39 +import java.util.stream.Collectors;
  40 +
  41 +@Slf4j
  42 +public abstract class AbstractSimpleSqlTimeseriesDao<T extends AbstractTsKvEntity> extends AbstractSqlTimeseriesDao {
  43 +
  44 + @Autowired
  45 + private InsertTsRepository<T> insertRepository;
  46 +
  47 + @Value("${sql.ts.batch_size:1000}")
  48 + private int tsBatchSize;
  49 +
  50 + @Value("${sql.ts.batch_max_delay:100}")
  51 + private long tsMaxDelay;
  52 +
  53 + @Value("${sql.ts.stats_print_interval_ms:1000}")
  54 + private long tsStatsPrintIntervalMs;
  55 +
  56 + protected TbSqlBlockingQueue<EntityContainer<T>> tsQueue;
  57 +
  58 + @PostConstruct
  59 + protected void init() {
  60 + super.init();
  61 + TbSqlBlockingQueueParams tsParams = TbSqlBlockingQueueParams.builder()
  62 + .logName("TS")
  63 + .batchSize(tsBatchSize)
  64 + .maxDelay(tsMaxDelay)
  65 + .statsPrintIntervalMs(tsStatsPrintIntervalMs)
  66 + .build();
  67 + tsQueue = new TbSqlBlockingQueue<>(tsParams);
  68 + tsQueue.init(logExecutor, v -> insertRepository.saveOrUpdate(v));
  69 + }
  70 +
  71 + @PreDestroy
  72 + protected void destroy() {
  73 + super.init();
  74 + if (tsQueue != null) {
  75 + tsQueue.destroy();
  76 + }
  77 + }
  78 +
  79 + protected ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
  80 + if (query.getAggregation() == Aggregation.NONE) {
  81 + return findAllAsyncWithLimit(entityId, query);
  82 + } else {
  83 + long stepTs = query.getStartTs();
  84 + List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>();
  85 + while (stepTs < query.getEndTs()) {
  86 + long startTs = stepTs;
  87 + long endTs = stepTs + query.getInterval();
  88 + long ts = startTs + (endTs - startTs) / 2;
  89 + futures.add(findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()));
  90 + stepTs = endTs;
  91 + }
  92 + return getTskvEntriesFuture(Futures.allAsList(futures));
  93 + }
  94 + }
  95 +
  96 + protected abstract ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation);
  97 +
  98 + protected abstract ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query);
  99 +
  100 + protected SettableFuture<T> setFutures(List<CompletableFuture<T>> entitiesFutures) {
  101 + SettableFuture<T> listenableFuture = SettableFuture.create();
  102 + CompletableFuture<List<T>> entities =
  103 + CompletableFuture.allOf(entitiesFutures.toArray(new CompletableFuture[entitiesFutures.size()]))
  104 + .thenApply(v -> entitiesFutures.stream()
  105 + .map(CompletableFuture::join)
  106 + .collect(Collectors.toList()));
  107 +
  108 + entities.whenComplete((tsKvEntities, throwable) -> {
  109 + if (throwable != null) {
  110 + listenableFuture.setException(throwable);
  111 + } else {
  112 + T result = null;
  113 + for (T entity : tsKvEntities) {
  114 + if (entity.isNotEmpty()) {
  115 + result = entity;
  116 + break;
  117 + }
  118 + }
  119 + listenableFuture.set(result);
  120 + }
  121 + });
  122 + return listenableFuture;
  123 + }
  124 +
  125 + protected void switchAgregation(EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation, List<CompletableFuture<T>> entitiesFutures) {
  126 + switch (aggregation) {
  127 + case AVG:
  128 + findAvg(entityId, key, startTs, endTs, entitiesFutures);
  129 + break;
  130 + case MAX:
  131 + findMax(entityId, key, startTs, endTs, entitiesFutures);
  132 + break;
  133 + case MIN:
  134 + findMin(entityId, key, startTs, endTs, entitiesFutures);
  135 + break;
  136 + case SUM:
  137 + findSum(entityId, key, startTs, endTs, entitiesFutures);
  138 + break;
  139 + case COUNT:
  140 + findCount(entityId, key, startTs, endTs, entitiesFutures);
  141 + break;
  142 + default:
  143 + throw new IllegalArgumentException("Not supported aggregation type: " + aggregation);
  144 + }
  145 + }
  146 +
  147 + protected abstract void findCount(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<T>> entitiesFutures);
  148 +
  149 + protected abstract void findSum(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<T>> entitiesFutures);
  150 +
  151 + protected abstract void findMin(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<T>> entitiesFutures);
  152 +
  153 + protected abstract void findMax(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<T>> entitiesFutures);
  154 +
  155 + protected abstract void findAvg(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<T>> entitiesFutures);
  156 +}
\ No newline at end of file
... ...
... ... @@ -16,20 +16,32 @@
16 16 package org.thingsboard.server.dao.sqlts;
17 17
18 18 import com.google.common.base.Function;
  19 +import com.google.common.collect.Lists;
  20 +import com.google.common.util.concurrent.FutureCallback;
19 21 import com.google.common.util.concurrent.Futures;
20 22 import com.google.common.util.concurrent.ListenableFuture;
21   -import com.google.common.util.concurrent.ListeningExecutorService;
22   -import com.google.common.util.concurrent.MoreExecutors;
  23 +import lombok.extern.slf4j.Slf4j;
  24 +import org.springframework.beans.factory.annotation.Autowired;
23 25 import org.springframework.beans.factory.annotation.Value;
  26 +import org.thingsboard.server.common.data.UUIDConverter;
24 27 import org.thingsboard.server.common.data.id.EntityId;
25 28 import org.thingsboard.server.common.data.id.TenantId;
26 29 import org.thingsboard.server.common.data.kv.Aggregation;
27 30 import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
  31 +import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
28 32 import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
29 33 import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
  34 +import org.thingsboard.server.common.data.kv.StringDataEntry;
30 35 import org.thingsboard.server.common.data.kv.TsKvEntry;
  36 +import org.thingsboard.server.dao.DaoUtil;
  37 +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey;
  38 +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
31 39 import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService;
32   -import org.thingsboard.server.dao.timeseries.TsInsertExecutorType;
  40 +import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent;
  41 +import org.thingsboard.server.dao.sql.TbSqlBlockingQueue;
  42 +import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams;
  43 +import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository;
  44 +import org.thingsboard.server.dao.timeseries.SimpleListenableFuture;
33 45
34 46 import javax.annotation.Nullable;
35 47 import javax.annotation.PostConstruct;
... ... @@ -37,13 +49,55 @@ import javax.annotation.PreDestroy;
37 49 import java.util.List;
38 50 import java.util.Objects;
39 51 import java.util.Optional;
40   -import java.util.concurrent.Executors;
  52 +import java.util.concurrent.ExecutionException;
41 53 import java.util.stream.Collectors;
42 54
  55 +import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID;
  56 +
  57 +@Slf4j
43 58 public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService {
44 59
45 60 private static final String DESC_ORDER = "DESC";
46 61
  62 + @Autowired
  63 + private TsKvLatestRepository tsKvLatestRepository;
  64 +
  65 + @Autowired
  66 + private InsertLatestRepository insertLatestRepository;
  67 +
  68 + @Autowired
  69 + protected ScheduledLogExecutorComponent logExecutor;
  70 +
  71 + @Value("${sql.ts_latest.batch_size:1000}")
  72 + private int tsLatestBatchSize;
  73 +
  74 + @Value("${sql.ts_latest.batch_max_delay:100}")
  75 + private long tsLatestMaxDelay;
  76 +
  77 + @Value("${sql.ts_latest.stats_print_interval_ms:1000}")
  78 + private long tsLatestStatsPrintIntervalMs;
  79 +
  80 + private TbSqlBlockingQueue<TsKvLatestEntity> tsLatestQueue;
  81 +
  82 + @PostConstruct
  83 + protected void init() {
  84 + TbSqlBlockingQueueParams tsLatestParams = TbSqlBlockingQueueParams.builder()
  85 + .logName("TS Latest")
  86 + .batchSize(tsLatestBatchSize)
  87 + .maxDelay(tsLatestMaxDelay)
  88 + .statsPrintIntervalMs(tsLatestStatsPrintIntervalMs)
  89 + .build();
  90 + tsLatestQueue = new TbSqlBlockingQueue<>(tsLatestParams);
  91 + tsLatestQueue.init(logExecutor, v -> insertLatestRepository.saveOrUpdate(v));
  92 + }
  93 +
  94 + @PreDestroy
  95 + protected void destroy() {
  96 + if (tsLatestQueue != null) {
  97 + tsLatestQueue.destroy();
  98 + }
  99 + }
  100 +
47 101 protected ListenableFuture<List<TsKvEntry>> processFindAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) {
48 102 List<ListenableFuture<List<TsKvEntry>>> futures = queries
49 103 .stream()
... ... @@ -89,4 +143,105 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
89 143 Aggregation.NONE, DESC_ORDER);
90 144 return findAllAsync(tenantId, entityId, findNewLatestQuery);
91 145 }
  146 +
  147 + protected ListenableFuture<TsKvEntry> getFindLatestFuture(EntityId entityId, String key) {
  148 + TsKvLatestCompositeKey compositeKey =
  149 + new TsKvLatestCompositeKey(
  150 + entityId.getEntityType(),
  151 + fromTimeUUID(entityId.getId()),
  152 + key);
  153 + Optional<TsKvLatestEntity> entry = tsKvLatestRepository.findById(compositeKey);
  154 + TsKvEntry result;
  155 + if (entry.isPresent()) {
  156 + result = DaoUtil.getData(entry.get());
  157 + } else {
  158 + result = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null));
  159 + }
  160 + return Futures.immediateFuture(result);
  161 + }
  162 +
  163 + protected ListenableFuture<Void> getRemoveLatestFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
  164 + ListenableFuture<TsKvEntry> latestFuture = getFindLatestFuture(entityId, query.getKey());
  165 +
  166 + ListenableFuture<Boolean> booleanFuture = Futures.transform(latestFuture, tsKvEntry -> {
  167 + long ts = tsKvEntry.getTs();
  168 + return ts > query.getStartTs() && ts <= query.getEndTs();
  169 + }, service);
  170 +
  171 + ListenableFuture<Void> removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
  172 + if (isRemove) {
  173 + TsKvLatestEntity latestEntity = new TsKvLatestEntity();
  174 + latestEntity.setEntityType(entityId.getEntityType());
  175 + latestEntity.setEntityId(fromTimeUUID(entityId.getId()));
  176 + latestEntity.setKey(query.getKey());
  177 + return service.submit(() -> {
  178 + tsKvLatestRepository.delete(latestEntity);
  179 + return null;
  180 + });
  181 + }
  182 + return Futures.immediateFuture(null);
  183 + }, service);
  184 +
  185 + final SimpleListenableFuture<Void> resultFuture = new SimpleListenableFuture<>();
  186 + Futures.addCallback(removedLatestFuture, new FutureCallback<Void>() {
  187 + @Override
  188 + public void onSuccess(@Nullable Void result) {
  189 + if (query.getRewriteLatestIfDeleted()) {
  190 + ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
  191 + if (isRemove) {
  192 + return getNewLatestEntryFuture(tenantId, entityId, query);
  193 + }
  194 + return Futures.immediateFuture(null);
  195 + }, service);
  196 +
  197 + try {
  198 + resultFuture.set(savedLatestFuture.get());
  199 + } catch (InterruptedException | ExecutionException e) {
  200 + log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e);
  201 + }
  202 + } else {
  203 + resultFuture.set(null);
  204 + }
  205 + }
  206 +
  207 + @Override
  208 + public void onFailure(Throwable t) {
  209 + log.warn("[{}] Failed to process remove of the latest value", entityId, t);
  210 + }
  211 + });
  212 + return resultFuture;
  213 + }
  214 +
  215 + protected ListenableFuture<List<TsKvEntry>> getFindAllLatestFuture(EntityId entityId) {
  216 + return Futures.immediateFuture(
  217 + DaoUtil.convertDataList(Lists.newArrayList(
  218 + tsKvLatestRepository.findAllByEntityTypeAndEntityId(
  219 + entityId.getEntityType(),
  220 + UUIDConverter.fromTimeUUID(entityId.getId())))));
  221 + }
  222 +
  223 + protected ListenableFuture<Void> getSaveLatestFuture(EntityId entityId, TsKvEntry tsKvEntry) {
  224 + TsKvLatestEntity latestEntity = new TsKvLatestEntity();
  225 + latestEntity.setEntityType(entityId.getEntityType());
  226 + latestEntity.setEntityId(fromTimeUUID(entityId.getId()));
  227 + latestEntity.setTs(tsKvEntry.getTs());
  228 + latestEntity.setKey(tsKvEntry.getKey());
  229 + latestEntity.setStrValue(tsKvEntry.getStrValue().orElse(null));
  230 + latestEntity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null));
  231 + latestEntity.setLongValue(tsKvEntry.getLongValue().orElse(null));
  232 + latestEntity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null));
  233 + return tsLatestQueue.add(latestEntity);
  234 + }
  235 +
  236 + private ListenableFuture<Void> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
  237 + ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(tenantId, entityId, query);
  238 + return Futures.transformAsync(future, entryList -> {
  239 + if (entryList.size() == 1) {
  240 + return getSaveLatestFuture(entityId, entryList.get(0));
  241 + } else {
  242 + log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey());
  243 + }
  244 + return Futures.immediateFuture(null);
  245 + }, service);
  246 + }
92 247 }
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.sqlts;
  17 +
  18 +import lombok.AllArgsConstructor;
  19 +import lombok.Data;
  20 +import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity;
  21 +
  22 +@Data
  23 +@AllArgsConstructor
  24 +public class EntityContainer<T extends AbstractTsKvEntity> {
  25 +
  26 + private T entity;
  27 + private String partitionDate;
  28 +
  29 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.sqlts;
  17 +
  18 +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
  19 +
  20 +import java.util.List;
  21 +
  22 +public interface InsertLatestRepository {
  23 +
  24 + void saveOrUpdate(List<TsKvLatestEntity> entities);
  25 +
  26 +}
... ...
dao/src/main/java/org/thingsboard/server/dao/sqlts/InsertTsRepository.java renamed from dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractTimeseriesInsertRepository.java
... ... @@ -15,44 +15,12 @@
15 15 */
16 16 package org.thingsboard.server.dao.sqlts;
17 17
18   -import org.springframework.data.jpa.repository.Modifying;
19   -import org.springframework.stereotype.Repository;
20 18 import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity;
21 19
22 20 import java.util.List;
23 21
24   -@Repository
25   -public abstract class AbstractTimeseriesInsertRepository<T extends AbstractTsKvEntity> extends AbstractInsertRepository {
  22 +public interface InsertTsRepository<T extends AbstractTsKvEntity> {
26 23
27   - public abstract void saveOrUpdate(T entity);
  24 + void saveOrUpdate(List<EntityContainer<T>> entities);
28 25
29   - public abstract void saveOrUpdate(List<T> entities);
30   -
31   - protected void processSaveOrUpdate(T entity, String requestBoolValue, String requestStrValue, String requestLongValue, String requestDblValue) {
32   - if (entity.getBooleanValue() != null) {
33   - saveOrUpdateBoolean(entity, requestBoolValue);
34   - }
35   - if (entity.getStrValue() != null) {
36   - saveOrUpdateString(entity, requestStrValue);
37   - }
38   - if (entity.getLongValue() != null) {
39   - saveOrUpdateLong(entity, requestLongValue);
40   - }
41   - if (entity.getDoubleValue() != null) {
42   - saveOrUpdateDouble(entity, requestDblValue);
43   - }
44   - }
45   -
46   - @Modifying
47   - protected abstract void saveOrUpdateBoolean(T entity, String query);
48   -
49   - @Modifying
50   - protected abstract void saveOrUpdateString(T entity, String query);
51   -
52   - @Modifying
53   - protected abstract void saveOrUpdateLong(T entity, String query);
54   -
55   - @Modifying
56   - protected abstract void saveOrUpdateDouble(T entity, String query);
57   -
58   -}
\ No newline at end of file
  26 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.sqlts.dictionary;
  17 +
  18 +import org.springframework.data.repository.CrudRepository;
  19 +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary;
  20 +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey;
  21 +import org.thingsboard.server.dao.util.PsqlDao;
  22 +
  23 +import java.util.Optional;
  24 +
  25 +@PsqlDao
  26 +public interface TsKvDictionaryRepository extends CrudRepository<TsKvDictionary, TsKvDictionaryCompositeKey> {
  27 +
  28 + Optional<TsKvDictionary> findByKeyId(int keyId);
  29 +
  30 +}
\ No newline at end of file
... ...
dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/HsqlTimeseriesInsertRepository.java renamed from dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/HsqlTimeseriesInsertRepository.java
... ... @@ -13,13 +13,15 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.dao.sqlts.ts;
  16 +package org.thingsboard.server.dao.sqlts.hsql;
17 17
18 18 import org.springframework.jdbc.core.BatchPreparedStatementSetter;
19 19 import org.springframework.stereotype.Repository;
20 20 import org.springframework.transaction.annotation.Transactional;
21   -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity;
22   -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository;
  21 +import org.thingsboard.server.dao.model.sqlts.hsql.TsKvEntity;
  22 +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository;
  23 +import org.thingsboard.server.dao.sqlts.EntityContainer;
  24 +import org.thingsboard.server.dao.sqlts.InsertTsRepository;
23 25 import org.thingsboard.server.dao.util.HsqlDao;
24 26 import org.thingsboard.server.dao.util.SqlTsDao;
25 27
... ... @@ -32,14 +34,7 @@ import java.util.List;
32 34 @HsqlDao
33 35 @Repository
34 36 @Transactional
35   -public class HsqlTimeseriesInsertRepository extends AbstractTimeseriesInsertRepository<TsKvEntity> {
36   -
37   - private static final String TS_KV_CONSTRAINT = "(ts_kv.entity_type=A.entity_type AND ts_kv.entity_id=A.entity_id AND ts_kv.key=A.key AND ts_kv.ts=A.ts)";
38   -
39   - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_TABLE, TS_KV_CONSTRAINT, BOOL_V, HSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS);
40   - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_TABLE, TS_KV_CONSTRAINT, STR_V, HSQL_ON_STR_VALUE_UPDATE_SET_NULLS);
41   - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_TABLE, TS_KV_CONSTRAINT, LONG_V, HSQL_ON_LONG_VALUE_UPDATE_SET_NULLS);
42   - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_TABLE, TS_KV_CONSTRAINT, DBL_V, HSQL_ON_DBL_VALUE_UPDATE_SET_NULLS);
  37 +public class HsqlTimeseriesInsertRepository extends AbstractInsertRepository implements InsertTsRepository<TsKvEntity> {
43 38
44 39 private static final String INSERT_OR_UPDATE =
45 40 "MERGE INTO ts_kv USING(VALUES ?, ?, ?, ?, ?, ?, ?, ?) " +
... ... @@ -53,36 +48,33 @@ public class HsqlTimeseriesInsertRepository extends AbstractTimeseriesInsertRepo
53 48 "VALUES (T.entity_type, T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v);";
54 49
55 50 @Override
56   - public void saveOrUpdate(TsKvEntity entity) {
57   - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT);
58   - }
59   -
60   - @Override
61   - public void saveOrUpdate(List<TsKvEntity> entities) {
  51 + public void saveOrUpdate(List<EntityContainer<TsKvEntity>> entities) {
62 52 jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() {
63 53 @Override
64 54 public void setValues(PreparedStatement ps, int i) throws SQLException {
65   - ps.setString(1, entities.get(i).getEntityType().name());
66   - ps.setString(2, entities.get(i).getEntityId());
67   - ps.setString(3, entities.get(i).getKey());
68   - ps.setLong(4, entities.get(i).getTs());
69   -
70   - if (entities.get(i).getBooleanValue() != null) {
71   - ps.setBoolean(5, entities.get(i).getBooleanValue());
  55 + EntityContainer<TsKvEntity> tsKvEntityEntityContainer = entities.get(i);
  56 + TsKvEntity tsKvEntity = tsKvEntityEntityContainer.getEntity();
  57 + ps.setString(1, tsKvEntity.getEntityType().name());
  58 + ps.setString(2, tsKvEntity.getEntityId());
  59 + ps.setString(3, tsKvEntity.getKey());
  60 + ps.setLong(4, tsKvEntity.getTs());
  61 +
  62 + if (tsKvEntity.getBooleanValue() != null) {
  63 + ps.setBoolean(5, tsKvEntity.getBooleanValue());
72 64 } else {
73 65 ps.setNull(5, Types.BOOLEAN);
74 66 }
75 67
76   - ps.setString(6, entities.get(i).getStrValue());
  68 + ps.setString(6, tsKvEntity.getStrValue());
77 69
78   - if (entities.get(i).getLongValue() != null) {
79   - ps.setLong(7, entities.get(i).getLongValue());
  70 + if (tsKvEntity.getLongValue() != null) {
  71 + ps.setLong(7, tsKvEntity.getLongValue());
80 72 } else {
81 73 ps.setNull(7, Types.BIGINT);
82 74 }
83 75
84   - if (entities.get(i).getDoubleValue() != null) {
85   - ps.setDouble(8, entities.get(i).getDoubleValue());
  76 + if (tsKvEntity.getDoubleValue() != null) {
  77 + ps.setDouble(8, tsKvEntity.getDoubleValue());
86 78 } else {
87 79 ps.setNull(8, Types.DOUBLE);
88 80 }
... ... @@ -94,48 +86,4 @@ public class HsqlTimeseriesInsertRepository extends AbstractTimeseriesInsertRepo
94 86 }
95 87 });
96 88 }
97   -
98   - @Override
99   - protected void saveOrUpdateBoolean(TsKvEntity entity, String query) {
100   - entityManager.createNativeQuery(query)
101   - .setParameter("entity_type", entity.getEntityType().name())
102   - .setParameter("entity_id", entity.getEntityId())
103   - .setParameter("key", entity.getKey())
104   - .setParameter("ts", entity.getTs())
105   - .setParameter("bool_v", entity.getBooleanValue())
106   - .executeUpdate();
107   - }
108   -
109   - @Override
110   - protected void saveOrUpdateString(TsKvEntity entity, String query) {
111   - entityManager.createNativeQuery(query)
112   - .setParameter("entity_type", entity.getEntityType().name())
113   - .setParameter("entity_id", entity.getEntityId())
114   - .setParameter("key", entity.getKey())
115   - .setParameter("ts", entity.getTs())
116   - .setParameter("str_v", entity.getStrValue())
117   - .executeUpdate();
118   - }
119   -
120   - @Override
121   - protected void saveOrUpdateLong(TsKvEntity entity, String query) {
122   - entityManager.createNativeQuery(query)
123   - .setParameter("entity_type", entity.getEntityType().name())
124   - .setParameter("entity_id", entity.getEntityId())
125   - .setParameter("key", entity.getKey())
126   - .setParameter("ts", entity.getTs())
127   - .setParameter("long_v", entity.getLongValue())
128   - .executeUpdate();
129   - }
130   -
131   - @Override
132   - protected void saveOrUpdateDouble(TsKvEntity entity, String query) {
133   - entityManager.createNativeQuery(query)
134   - .setParameter("entity_type", entity.getEntityType().name())
135   - .setParameter("entity_id", entity.getEntityId())
136   - .setParameter("key", entity.getKey())
137   - .setParameter("ts", entity.getTs())
138   - .setParameter("dbl_v", entity.getDoubleValue())
139   - .executeUpdate();
140   - }
141 89 }
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.sqlts.hsql;
  17 +
  18 +import com.google.common.util.concurrent.Futures;
  19 +import com.google.common.util.concurrent.ListenableFuture;
  20 +import lombok.extern.slf4j.Slf4j;
  21 +import org.springframework.beans.factory.annotation.Autowired;
  22 +import org.springframework.data.domain.PageRequest;
  23 +import org.springframework.data.domain.Sort;
  24 +import org.springframework.stereotype.Component;
  25 +import org.thingsboard.server.common.data.id.EntityId;
  26 +import org.thingsboard.server.common.data.id.TenantId;
  27 +import org.thingsboard.server.common.data.kv.Aggregation;
  28 +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
  29 +import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
  30 +import org.thingsboard.server.common.data.kv.TsKvEntry;
  31 +import org.thingsboard.server.dao.DaoUtil;
  32 +import org.thingsboard.server.dao.model.sqlts.hsql.TsKvEntity;
  33 +import org.thingsboard.server.dao.sqlts.AbstractSimpleSqlTimeseriesDao;
  34 +import org.thingsboard.server.dao.sqlts.EntityContainer;
  35 +import org.thingsboard.server.dao.timeseries.TimeseriesDao;
  36 +import org.thingsboard.server.dao.util.HsqlDao;
  37 +import org.thingsboard.server.dao.util.SqlTsDao;
  38 +
  39 +import java.util.ArrayList;
  40 +import java.util.List;
  41 +import java.util.Optional;
  42 +import java.util.concurrent.CompletableFuture;
  43 +
  44 +import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID;
  45 +
  46 +
  47 +@Component
  48 +@Slf4j
  49 +@SqlTsDao
  50 +@HsqlDao
  51 +public class JpaHsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao<TsKvEntity> implements TimeseriesDao {
  52 +
  53 + @Autowired
  54 + private TsKvHsqlRepository tsKvRepository;
  55 +
  56 + @Override
  57 + public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) {
  58 + return processFindAllAsync(tenantId, entityId, queries);
  59 + }
  60 +
  61 + @Override
  62 + public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) {
  63 + TsKvEntity entity = new TsKvEntity();
  64 + entity.setEntityType(entityId.getEntityType());
  65 + entity.setEntityId(fromTimeUUID(entityId.getId()));
  66 + entity.setTs(tsKvEntry.getTs());
  67 + entity.setKey(tsKvEntry.getKey());
  68 + entity.setStrValue(tsKvEntry.getStrValue().orElse(null));
  69 + entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null));
  70 + entity.setLongValue(tsKvEntry.getLongValue().orElse(null));
  71 + entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null));
  72 + log.trace("Saving entity: {}", entity);
  73 + return tsQueue.add(new EntityContainer(entity, null));
  74 + }
  75 +
  76 + @Override
  77 + public ListenableFuture<Void> remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
  78 + return service.submit(() -> {
  79 + tsKvRepository.delete(
  80 + fromTimeUUID(entityId.getId()),
  81 + entityId.getEntityType(),
  82 + query.getKey(),
  83 + query.getStartTs(),
  84 + query.getEndTs());
  85 + return null;
  86 + });
  87 + }
  88 +
  89 + @Override
  90 + public ListenableFuture<Void> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
  91 + return getSaveLatestFuture(entityId, tsKvEntry);
  92 + }
  93 +
  94 + @Override
  95 + public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
  96 + return getRemoveLatestFuture(tenantId, entityId, query);
  97 + }
  98 +
  99 + @Override
  100 + public ListenableFuture<TsKvEntry> findLatest(TenantId tenantId, EntityId entityId, String key) {
  101 + return getFindLatestFuture(entityId, key);
  102 + }
  103 +
  104 + @Override
  105 + public ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId) {
  106 + return getFindAllLatestFuture(entityId);
  107 + }
  108 +
  109 + @Override
  110 + public ListenableFuture<Void> savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) {
  111 + return Futures.immediateFuture(null);
  112 + }
  113 +
  114 + @Override
  115 + public ListenableFuture<Void> removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
  116 + return Futures.immediateFuture(null);
  117 + }
  118 +
  119 + protected ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) {
  120 + List<CompletableFuture<TsKvEntity>> entitiesFutures = new ArrayList<>();
  121 + switchAgregation(entityId, key, startTs, endTs, aggregation, entitiesFutures);
  122 + return Futures.transform(setFutures(entitiesFutures), entity -> {
  123 + if (entity != null && entity.isNotEmpty()) {
  124 + entity.setEntityId(fromTimeUUID(entityId.getId()));
  125 + entity.setEntityType(entityId.getEntityType());
  126 + entity.setKey(key);
  127 + entity.setTs(ts);
  128 + return Optional.of(DaoUtil.getData(entity));
  129 + } else {
  130 + return Optional.empty();
  131 + }
  132 + });
  133 + }
  134 +
  135 + protected ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) {
  136 + return Futures.immediateFuture(
  137 + DaoUtil.convertDataList(
  138 + tsKvRepository.findAllWithLimit(
  139 + fromTimeUUID(entityId.getId()),
  140 + entityId.getEntityType(),
  141 + query.getKey(),
  142 + query.getStartTs(),
  143 + query.getEndTs(),
  144 + new PageRequest(0, query.getLimit(),
  145 + new Sort(Sort.Direction.fromString(
  146 + query.getOrderBy()), "ts")))));
  147 + }
  148 +
  149 + protected void findCount(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  150 + entitiesFutures.add(tsKvRepository.findCount(
  151 + fromTimeUUID(entityId.getId()),
  152 + entityId.getEntityType(),
  153 + key,
  154 + startTs,
  155 + endTs));
  156 + }
  157 +
  158 + protected void findSum(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  159 + entitiesFutures.add(tsKvRepository.findSum(
  160 + fromTimeUUID(entityId.getId()),
  161 + entityId.getEntityType(),
  162 + key,
  163 + startTs,
  164 + endTs));
  165 + }
  166 +
  167 + protected void findMin(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  168 + entitiesFutures.add(tsKvRepository.findStringMin(
  169 + fromTimeUUID(entityId.getId()),
  170 + entityId.getEntityType(),
  171 + key,
  172 + startTs,
  173 + endTs));
  174 + entitiesFutures.add(tsKvRepository.findNumericMin(
  175 + fromTimeUUID(entityId.getId()),
  176 + entityId.getEntityType(),
  177 + key,
  178 + startTs,
  179 + endTs));
  180 + }
  181 +
  182 + protected void findMax(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  183 + entitiesFutures.add(tsKvRepository.findStringMax(
  184 + fromTimeUUID(entityId.getId()),
  185 + entityId.getEntityType(),
  186 + key,
  187 + startTs,
  188 + endTs));
  189 + entitiesFutures.add(tsKvRepository.findNumericMax(
  190 + fromTimeUUID(entityId.getId()),
  191 + entityId.getEntityType(),
  192 + key,
  193 + startTs,
  194 + endTs));
  195 + }
  196 +
  197 + protected void findAvg(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  198 + entitiesFutures.add(tsKvRepository.findAvg(
  199 + fromTimeUUID(entityId.getId()),
  200 + entityId.getEntityType(),
  201 + key,
  202 + startTs,
  203 + endTs));
  204 + }
  205 +
  206 +}
\ No newline at end of file
... ...
dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/TsKvHsqlRepository.java renamed from dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java
... ... @@ -13,7 +13,7 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.dao.sqlts.ts;
  16 +package org.thingsboard.server.dao.sqlts.hsql;
17 17
18 18 import org.springframework.data.domain.Pageable;
19 19 import org.springframework.data.jpa.repository.Modifying;
... ... @@ -23,15 +23,15 @@ import org.springframework.data.repository.query.Param;
23 23 import org.springframework.scheduling.annotation.Async;
24 24 import org.springframework.transaction.annotation.Transactional;
25 25 import org.thingsboard.server.common.data.EntityType;
26   -import org.thingsboard.server.dao.model.sqlts.ts.TsKvCompositeKey;
27   -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity;
  26 +import org.thingsboard.server.dao.model.sqlts.hsql.TsKvCompositeKey;
  27 +import org.thingsboard.server.dao.model.sqlts.hsql.TsKvEntity;
28 28 import org.thingsboard.server.dao.util.SqlDao;
29 29
30 30 import java.util.List;
31 31 import java.util.concurrent.CompletableFuture;
32 32
33 33 @SqlDao
34   -public interface TsKvRepository extends CrudRepository<TsKvEntity, TsKvCompositeKey> {
  34 +public interface TsKvHsqlRepository extends CrudRepository<TsKvEntity, TsKvCompositeKey> {
35 35
36 36 @Query("SELECT tskv FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " +
37 37 "AND tskv.entityType = :entityType AND tskv.key = :entityKey " +
... ... @@ -146,4 +146,4 @@ public interface TsKvRepository extends CrudRepository<TsKvEntity, TsKvComposite
146 146 @Param("startTs") long startTs,
147 147 @Param("endTs") long endTs);
148 148
149   -}
  149 +}
\ No newline at end of file
... ...
dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/HsqlLatestInsertRepository.java renamed from dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/HsqlLatestInsertRepository.java
... ... @@ -13,13 +13,14 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.dao.sqlts.ts;
  16 +package org.thingsboard.server.dao.sqlts.latest;
17 17
18 18 import org.springframework.jdbc.core.BatchPreparedStatementSetter;
19 19 import org.springframework.stereotype.Repository;
20 20 import org.springframework.transaction.annotation.Transactional;
21   -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity;
22   -import org.thingsboard.server.dao.sqlts.AbstractLatestInsertRepository;
  21 +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
  22 +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository;
  23 +import org.thingsboard.server.dao.sqlts.InsertLatestRepository;
23 24 import org.thingsboard.server.dao.util.HsqlDao;
24 25 import org.thingsboard.server.dao.util.SqlTsDao;
25 26
... ... @@ -32,14 +33,7 @@ import java.util.List;
32 33 @HsqlDao
33 34 @Repository
34 35 @Transactional
35   -public class HsqlLatestInsertRepository extends AbstractLatestInsertRepository {
36   -
37   - private static final String TS_KV_LATEST_CONSTRAINT = "(ts_kv_latest.entity_type=A.entity_type AND ts_kv_latest.entity_id=A.entity_id AND ts_kv_latest.key=A.key)";
38   -
39   - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, BOOL_V, HSQL_LATEST_ON_BOOL_VALUE_UPDATE_SET_NULLS);
40   - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, STR_V, HSQL_LATEST_ON_STR_VALUE_UPDATE_SET_NULLS);
41   - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, LONG_V, HSQL_LATEST_ON_LONG_VALUE_UPDATE_SET_NULLS);
42   - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, DBL_V, HSQL_LATEST_ON_DBL_VALUE_UPDATE_SET_NULLS);
  36 +public class HsqlLatestInsertRepository extends AbstractInsertRepository implements InsertLatestRepository {
43 37
44 38 private static final String INSERT_OR_UPDATE =
45 39 "MERGE INTO ts_kv_latest USING(VALUES ?, ?, ?, ?, ?, ?, ?, ?) " +
... ... @@ -52,11 +46,6 @@ public class HsqlLatestInsertRepository extends AbstractLatestInsertRepository {
52 46 "VALUES (T.entity_type, T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v);";
53 47
54 48 @Override
55   - public void saveOrUpdate(TsKvLatestEntity entity) {
56   - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT);
57   - }
58   -
59   - @Override
60 49 public void saveOrUpdate(List<TsKvLatestEntity> entities) {
61 50 jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() {
62 51 @Override
... ... @@ -93,48 +82,4 @@ public class HsqlLatestInsertRepository extends AbstractLatestInsertRepository {
93 82 }
94 83 });
95 84 }
96   -
97   - @Override
98   - protected void saveOrUpdateBoolean(TsKvLatestEntity entity, String query) {
99   - entityManager.createNativeQuery(query)
100   - .setParameter("entity_type", entity.getEntityType().name())
101   - .setParameter("entity_id", entity.getEntityId())
102   - .setParameter("key", entity.getKey())
103   - .setParameter("ts", entity.getTs())
104   - .setParameter("bool_v", entity.getBooleanValue())
105   - .executeUpdate();
106   - }
107   -
108   - @Override
109   - protected void saveOrUpdateString(TsKvLatestEntity entity, String query) {
110   - entityManager.createNativeQuery(query)
111   - .setParameter("entity_type", entity.getEntityType().name())
112   - .setParameter("entity_id", entity.getEntityId())
113   - .setParameter("key", entity.getKey())
114   - .setParameter("ts", entity.getTs())
115   - .setParameter("str_v", entity.getStrValue())
116   - .executeUpdate();
117   - }
118   -
119   - @Override
120   - protected void saveOrUpdateLong(TsKvLatestEntity entity, String query) {
121   - entityManager.createNativeQuery(query)
122   - .setParameter("entity_type", entity.getEntityType().name())
123   - .setParameter("entity_id", entity.getEntityId())
124   - .setParameter("key", entity.getKey())
125   - .setParameter("ts", entity.getTs())
126   - .setParameter("long_v", entity.getLongValue())
127   - .executeUpdate();
128   - }
129   -
130   - @Override
131   - protected void saveOrUpdateDouble(TsKvLatestEntity entity, String query) {
132   - entityManager.createNativeQuery(query)
133   - .setParameter("entity_type", entity.getEntityType().name())
134   - .setParameter("entity_id", entity.getEntityId())
135   - .setParameter("key", entity.getKey())
136   - .setParameter("ts", entity.getTs())
137   - .setParameter("dbl_v", entity.getDoubleValue())
138   - .executeUpdate();
139   - }
140 85 }
\ No newline at end of file
... ...
dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/PsqlLatestInsertRepository.java renamed from dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlLatestInsertRepository.java
... ... @@ -13,17 +13,17 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.dao.sqlts.ts;
  16 +package org.thingsboard.server.dao.sqlts.latest;
17 17
18 18 import org.springframework.jdbc.core.BatchPreparedStatementSetter;
19 19 import org.springframework.stereotype.Repository;
20 20 import org.springframework.transaction.TransactionStatus;
21 21 import org.springframework.transaction.annotation.Transactional;
22 22 import org.springframework.transaction.support.TransactionCallbackWithoutResult;
23   -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity;
24   -import org.thingsboard.server.dao.sqlts.AbstractLatestInsertRepository;
25   -import org.thingsboard.server.dao.util.PsqlDao;
26   -import org.thingsboard.server.dao.util.SqlTsDao;
  23 +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
  24 +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository;
  25 +import org.thingsboard.server.dao.sqlts.InsertLatestRepository;
  26 +import org.thingsboard.server.dao.util.PsqlTsAnyDao;
27 27
28 28 import java.sql.PreparedStatement;
29 29 import java.sql.SQLException;
... ... @@ -31,18 +31,11 @@ import java.sql.Types;
31 31 import java.util.ArrayList;
32 32 import java.util.List;
33 33
34   -@SqlTsDao
35   -@PsqlDao
  34 +
  35 +@PsqlTsAnyDao
36 36 @Repository
37 37 @Transactional
38   -public class PsqlLatestInsertRepository extends AbstractLatestInsertRepository {
39   -
40   - private static final String TS_KV_LATEST_CONSTRAINT = "(entity_type, entity_id, key)";
41   -
42   - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, BOOL_V, PSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS);
43   - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, STR_V, PSQL_ON_STR_VALUE_UPDATE_SET_NULLS);
44   - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, LONG_V, PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS);
45   - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, DBL_V, PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS);
  38 +public class PsqlLatestInsertRepository extends AbstractInsertRepository implements InsertLatestRepository {
46 39
47 40 private static final String BATCH_UPDATE =
48 41 "UPDATE ts_kv_latest SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ? WHERE entity_type = ? AND entity_id = ? and key = ?";
... ... @@ -53,11 +46,6 @@ public class PsqlLatestInsertRepository extends AbstractLatestInsertRepository {
53 46 "ON CONFLICT (entity_type, entity_id, key) DO UPDATE SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?;";
54 47
55 48 @Override
56   - public void saveOrUpdate(TsKvLatestEntity entity) {
57   - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT);
58   - }
59   -
60   - @Override
61 49 public void saveOrUpdate(List<TsKvLatestEntity> entities) {
62 50 transactionTemplate.execute(new TransactionCallbackWithoutResult() {
63 51 @Override
... ... @@ -160,48 +148,4 @@ public class PsqlLatestInsertRepository extends AbstractLatestInsertRepository {
160 148 }
161 149 });
162 150 }
163   -
164   - @Override
165   - protected void saveOrUpdateBoolean(TsKvLatestEntity entity, String query) {
166   - entityManager.createNativeQuery(query)
167   - .setParameter("entity_type", entity.getEntityType().name())
168   - .setParameter("entity_id", entity.getEntityId())
169   - .setParameter("key", entity.getKey())
170   - .setParameter("ts", entity.getTs())
171   - .setParameter("bool_v", entity.getBooleanValue())
172   - .executeUpdate();
173   - }
174   -
175   - @Override
176   - protected void saveOrUpdateString(TsKvLatestEntity entity, String query) {
177   - entityManager.createNativeQuery(query)
178   - .setParameter("entity_type", entity.getEntityType().name())
179   - .setParameter("entity_id", entity.getEntityId())
180   - .setParameter("key", entity.getKey())
181   - .setParameter("ts", entity.getTs())
182   - .setParameter("str_v", replaceNullChars(entity.getStrValue()))
183   - .executeUpdate();
184   - }
185   -
186   - @Override
187   - protected void saveOrUpdateLong(TsKvLatestEntity entity, String query) {
188   - entityManager.createNativeQuery(query)
189   - .setParameter("entity_type", entity.getEntityType().name())
190   - .setParameter("entity_id", entity.getEntityId())
191   - .setParameter("key", entity.getKey())
192   - .setParameter("ts", entity.getTs())
193   - .setParameter("long_v", entity.getLongValue())
194   - .executeUpdate();
195   - }
196   -
197   - @Override
198   - protected void saveOrUpdateDouble(TsKvLatestEntity entity, String query) {
199   - entityManager.createNativeQuery(query)
200   - .setParameter("entity_type", entity.getEntityType().name())
201   - .setParameter("entity_id", entity.getEntityId())
202   - .setParameter("key", entity.getKey())
203   - .setParameter("ts", entity.getTs())
204   - .setParameter("dbl_v", entity.getDoubleValue())
205   - .executeUpdate();
206   - }
207 151 }
\ No newline at end of file
... ...
dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java renamed from dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvLatestRepository.java
... ... @@ -13,12 +13,12 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.dao.sqlts.ts;
  16 +package org.thingsboard.server.dao.sqlts.latest;
17 17
18 18 import org.springframework.data.repository.CrudRepository;
19 19 import org.thingsboard.server.common.data.EntityType;
20   -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestCompositeKey;
21   -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity;
  20 +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey;
  21 +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
22 22 import org.thingsboard.server.dao.util.SqlDao;
23 23
24 24 import java.util.List;
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.sqlts.psql;
  17 +
  18 +import com.google.common.util.concurrent.Futures;
  19 +import com.google.common.util.concurrent.ListenableFuture;
  20 +import lombok.extern.slf4j.Slf4j;
  21 +import org.hibernate.exception.ConstraintViolationException;
  22 +import org.springframework.beans.factory.annotation.Autowired;
  23 +import org.springframework.beans.factory.annotation.Value;
  24 +import org.springframework.data.domain.PageRequest;
  25 +import org.springframework.data.domain.Sort;
  26 +import org.springframework.stereotype.Component;
  27 +import org.thingsboard.server.common.data.id.EntityId;
  28 +import org.thingsboard.server.common.data.id.TenantId;
  29 +import org.thingsboard.server.common.data.kv.Aggregation;
  30 +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
  31 +import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
  32 +import org.thingsboard.server.common.data.kv.TsKvEntry;
  33 +import org.thingsboard.server.dao.DaoUtil;
  34 +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary;
  35 +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey;
  36 +import org.thingsboard.server.dao.model.sqlts.psql.TsKvEntity;
  37 +import org.thingsboard.server.dao.sqlts.AbstractSimpleSqlTimeseriesDao;
  38 +import org.thingsboard.server.dao.sqlts.EntityContainer;
  39 +import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository;
  40 +import org.thingsboard.server.dao.timeseries.PsqlPartition;
  41 +import org.thingsboard.server.dao.timeseries.SqlTsPartitionDate;
  42 +import org.thingsboard.server.dao.timeseries.TimeseriesDao;
  43 +import org.thingsboard.server.dao.util.PsqlDao;
  44 +import org.thingsboard.server.dao.util.SqlTsDao;
  45 +
  46 +import java.time.Instant;
  47 +import java.time.LocalDateTime;
  48 +import java.time.ZoneOffset;
  49 +import java.util.ArrayList;
  50 +import java.util.List;
  51 +import java.util.Optional;
  52 +import java.util.Set;
  53 +import java.util.concurrent.CompletableFuture;
  54 +import java.util.concurrent.ConcurrentHashMap;
  55 +import java.util.concurrent.ConcurrentMap;
  56 +import java.util.concurrent.locks.ReentrantLock;
  57 +
  58 +import static org.thingsboard.server.dao.timeseries.SqlTsPartitionDate.EPOCH_START;
  59 +
  60 +
  61 +@Component
  62 +@Slf4j
  63 +@SqlTsDao
  64 +@PsqlDao
  65 +public class JpaPsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao<TsKvEntity> implements TimeseriesDao {
  66 +
  67 + private final ConcurrentMap<String, Integer> tsKvDictionaryMap = new ConcurrentHashMap<>();
  68 + private final Set<PsqlPartition> partitions = ConcurrentHashMap.newKeySet();
  69 +
  70 + private static final ReentrantLock tsCreationLock = new ReentrantLock();
  71 + private static final ReentrantLock partitionCreationLock = new ReentrantLock();
  72 +
  73 + @Autowired
  74 + private TsKvDictionaryRepository dictionaryRepository;
  75 +
  76 + @Autowired
  77 + private TsKvPsqlRepository tsKvRepository;
  78 +
  79 + @Autowired
  80 + private PsqlPartitioningRepository partitioningRepository;
  81 +
  82 + private SqlTsPartitionDate tsFormat;
  83 +
  84 + @Value("${sql.ts_key_value_partitioning}")
  85 + private String partitioning;
  86 +
  87 + @Override
  88 + protected void init() {
  89 + super.init();
  90 + Optional<SqlTsPartitionDate> partition = SqlTsPartitionDate.parse(partitioning);
  91 + if (partition.isPresent()) {
  92 + tsFormat = partition.get();
  93 + } else {
  94 + log.warn("Incorrect configuration of partitioning {}", partitioning);
  95 + throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!");
  96 + }
  97 + }
  98 +
  99 + @Override
  100 + public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) {
  101 + return processFindAllAsync(tenantId, entityId, queries);
  102 + }
  103 +
  104 + @Override
  105 + public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) {
  106 + String strKey = tsKvEntry.getKey();
  107 + Integer keyId = getOrSaveKeyId(strKey);
  108 + TsKvEntity entity = new TsKvEntity();
  109 + entity.setEntityId(entityId.getId());
  110 + entity.setTs(tsKvEntry.getTs());
  111 + entity.setKey(keyId);
  112 + entity.setStrValue(tsKvEntry.getStrValue().orElse(null));
  113 + entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null));
  114 + entity.setLongValue(tsKvEntry.getLongValue().orElse(null));
  115 + entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null));
  116 + PsqlPartition psqlPartition = toPartition(tsKvEntry.getTs());
  117 + savePartition(psqlPartition);
  118 + log.trace("Saving entity: {}", entity);
  119 + return tsQueue.add(new EntityContainer(entity, psqlPartition.getPartitionDate()));
  120 + }
  121 +
  122 + @Override
  123 + public ListenableFuture<Void> remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
  124 + return service.submit(() -> {
  125 + String strKey = query.getKey();
  126 + Integer keyId = getOrSaveKeyId(strKey);
  127 + tsKvRepository.delete(
  128 + entityId.getId(),
  129 + keyId,
  130 + query.getStartTs(),
  131 + query.getEndTs());
  132 + return null;
  133 + });
  134 + }
  135 +
  136 + @Override
  137 + public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
  138 + return getRemoveLatestFuture(tenantId, entityId, query);
  139 + }
  140 +
  141 + @Override
  142 + public ListenableFuture<Void> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
  143 + return getSaveLatestFuture(entityId, tsKvEntry);
  144 + }
  145 +
  146 + @Override
  147 + public ListenableFuture<TsKvEntry> findLatest(TenantId tenantId, EntityId entityId, String key) {
  148 + return getFindLatestFuture(entityId, key);
  149 + }
  150 +
  151 + @Override
  152 + public ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId) {
  153 + return getFindAllLatestFuture(entityId);
  154 + }
  155 +
  156 + @Override
  157 + public ListenableFuture<Void> savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) {
  158 + return Futures.immediateFuture(null);
  159 + }
  160 +
  161 + @Override
  162 + public ListenableFuture<Void> removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
  163 + return Futures.immediateFuture(null);
  164 + }
  165 +
  166 + protected ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) {
  167 + List<CompletableFuture<TsKvEntity>> entitiesFutures = new ArrayList<>();
  168 + switchAgregation(entityId, key, startTs, endTs, aggregation, entitiesFutures);
  169 + return Futures.transform(setFutures(entitiesFutures), entity -> {
  170 + if (entity != null && entity.isNotEmpty()) {
  171 + entity.setEntityId(entityId.getId());
  172 + entity.setStrKey(key);
  173 + entity.setTs(ts);
  174 + return Optional.of(DaoUtil.getData(entity));
  175 + } else {
  176 + return Optional.empty();
  177 + }
  178 + });
  179 + }
  180 +
  181 + protected ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) {
  182 + Integer keyId = getOrSaveKeyId(query.getKey());
  183 + List<TsKvEntity> tsKvEntities = tsKvRepository.findAllWithLimit(
  184 + entityId.getId(),
  185 + keyId,
  186 + query.getStartTs(),
  187 + query.getEndTs(),
  188 + new PageRequest(0, query.getLimit(),
  189 + new Sort(Sort.Direction.fromString(
  190 + query.getOrderBy()), "ts")));
  191 + tsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(query.getKey()));
  192 + return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities));
  193 + }
  194 +
  195 + protected void findCount(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  196 + Integer keyId = getOrSaveKeyId(key);
  197 + entitiesFutures.add(tsKvRepository.findCount(
  198 + entityId.getId(),
  199 + keyId,
  200 + startTs,
  201 + endTs));
  202 + }
  203 +
  204 + protected void findSum(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  205 + Integer keyId = getOrSaveKeyId(key);
  206 + entitiesFutures.add(tsKvRepository.findSum(
  207 + entityId.getId(),
  208 + keyId,
  209 + startTs,
  210 + endTs));
  211 + }
  212 +
  213 + protected void findMin(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  214 + Integer keyId = getOrSaveKeyId(key);
  215 + entitiesFutures.add(tsKvRepository.findStringMin(
  216 + entityId.getId(),
  217 + keyId,
  218 + startTs,
  219 + endTs));
  220 + entitiesFutures.add(tsKvRepository.findNumericMin(
  221 + entityId.getId(),
  222 + keyId,
  223 + startTs,
  224 + endTs));
  225 + }
  226 +
  227 + protected void findMax(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  228 + Integer keyId = getOrSaveKeyId(key);
  229 + entitiesFutures.add(tsKvRepository.findStringMax(
  230 + entityId.getId(),
  231 + keyId,
  232 + startTs,
  233 + endTs));
  234 + entitiesFutures.add(tsKvRepository.findNumericMax(
  235 + entityId.getId(),
  236 + keyId,
  237 + startTs,
  238 + endTs));
  239 + }
  240 +
  241 + protected void findAvg(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  242 + Integer keyId = getOrSaveKeyId(key);
  243 + entitiesFutures.add(tsKvRepository.findAvg(
  244 + entityId.getId(),
  245 + keyId,
  246 + startTs,
  247 + endTs));
  248 + }
  249 +
  250 + private Integer getOrSaveKeyId(String strKey) {
  251 + Integer keyId = tsKvDictionaryMap.get(strKey);
  252 + if (keyId == null) {
  253 + Optional<TsKvDictionary> tsKvDictionaryOptional;
  254 + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey));
  255 + if (!tsKvDictionaryOptional.isPresent()) {
  256 + tsCreationLock.lock();
  257 + try {
  258 + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey));
  259 + if (!tsKvDictionaryOptional.isPresent()) {
  260 + TsKvDictionary tsKvDictionary = new TsKvDictionary();
  261 + tsKvDictionary.setKey(strKey);
  262 + try {
  263 + TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary);
  264 + tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId());
  265 + keyId = saved.getKeyId();
  266 + } catch (ConstraintViolationException e) {
  267 + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey));
  268 + TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!"));
  269 + tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId());
  270 + keyId = dictionary.getKeyId();
  271 + }
  272 + } else {
  273 + keyId = tsKvDictionaryOptional.get().getKeyId();
  274 + }
  275 + } finally {
  276 + tsCreationLock.unlock();
  277 + }
  278 + } else {
  279 + keyId = tsKvDictionaryOptional.get().getKeyId();
  280 + tsKvDictionaryMap.put(strKey, keyId);
  281 + }
  282 + }
  283 + return keyId;
  284 + }
  285 +
  286 + private void savePartition(PsqlPartition psqlPartition) {
  287 + if (!partitions.contains(psqlPartition)) {
  288 + partitionCreationLock.lock();
  289 + try {
  290 + log.trace("Saving partition: {}", psqlPartition);
  291 + partitioningRepository.save(psqlPartition);
  292 + log.trace("Adding partition to Set: {}", psqlPartition);
  293 + partitions.add(psqlPartition);
  294 + } finally {
  295 + partitionCreationLock.unlock();
  296 + }
  297 + }
  298 + }
  299 +
  300 + private PsqlPartition toPartition(long ts) {
  301 + LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC);
  302 + LocalDateTime localDateTimeStart = tsFormat.trancateTo(time);
  303 + if (localDateTimeStart == SqlTsPartitionDate.EPOCH_START) {
  304 + return new PsqlPartition(toMills(EPOCH_START), Long.MAX_VALUE, tsFormat.getPattern());
  305 + } else {
  306 + LocalDateTime localDateTimeEnd = tsFormat.plusTo(localDateTimeStart);
  307 + return new PsqlPartition(toMills(localDateTimeStart), toMills(localDateTimeEnd), tsFormat.getPattern());
  308 + }
  309 + }
  310 +
  311 + private long toMills(LocalDateTime time) { return time.toInstant(ZoneOffset.UTC).toEpochMilli(); }
  312 +}
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.sqlts.psql;
  17 +
  18 +import org.springframework.stereotype.Repository;
  19 +import org.springframework.transaction.annotation.Transactional;
  20 +import org.thingsboard.server.dao.timeseries.PsqlPartition;
  21 +import org.thingsboard.server.dao.util.PsqlDao;
  22 +import org.thingsboard.server.dao.util.SqlTsDao;
  23 +
  24 +import javax.persistence.EntityManager;
  25 +import javax.persistence.PersistenceContext;
  26 +
  27 +@SqlTsDao
  28 +@PsqlDao
  29 +@Repository
  30 +@Transactional
  31 +public class PsqlPartitioningRepository {
  32 +
  33 + @PersistenceContext
  34 + private EntityManager entityManager;
  35 +
  36 + public void save(PsqlPartition partition) {
  37 + entityManager.createNativeQuery(partition.getQuery())
  38 + .executeUpdate();
  39 + }
  40 +
  41 +}
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.sqlts.psql;
  17 +
  18 +import org.springframework.jdbc.core.BatchPreparedStatementSetter;
  19 +import org.springframework.stereotype.Repository;
  20 +import org.springframework.transaction.annotation.Transactional;
  21 +import org.thingsboard.server.dao.model.sqlts.psql.TsKvEntity;
  22 +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository;
  23 +import org.thingsboard.server.dao.sqlts.EntityContainer;
  24 +import org.thingsboard.server.dao.sqlts.InsertTsRepository;
  25 +import org.thingsboard.server.dao.util.PsqlDao;
  26 +import org.thingsboard.server.dao.util.SqlTsDao;
  27 +
  28 +import java.sql.PreparedStatement;
  29 +import java.sql.SQLException;
  30 +import java.sql.Types;
  31 +import java.util.ArrayList;
  32 +import java.util.HashMap;
  33 +import java.util.List;
  34 +import java.util.Map;
  35 +
  36 +@SqlTsDao
  37 +@PsqlDao
  38 +@Repository
  39 +@Transactional
  40 +public class PsqlTimeseriesInsertRepository extends AbstractInsertRepository implements InsertTsRepository<TsKvEntity> {
  41 +
  42 + private static final String INSERT_INTO_TS_KV = "INSERT INTO ts_kv_";
  43 +
  44 + private static final String VALUES_ON_CONFLICT_DO_UPDATE = " (entity_id, key, ts, bool_v, str_v, long_v, dbl_v) VALUES (?, ?, ?, ?, ?, ?, ?) " +
  45 + "ON CONFLICT (entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?;";
  46 +
  47 + @Override
  48 + public void saveOrUpdate(List<EntityContainer<TsKvEntity>> entities) {
  49 + Map<String, List<TsKvEntity>> partitionMap = new HashMap<>();
  50 + for (EntityContainer<TsKvEntity> entityContainer : entities) {
  51 + List<TsKvEntity> tsKvEntities = partitionMap.computeIfAbsent(entityContainer.getPartitionDate(), k -> new ArrayList<>());
  52 + tsKvEntities.add(entityContainer.getEntity());
  53 + }
  54 + partitionMap.forEach((partition, entries) -> jdbcTemplate.batchUpdate(getInsertOrUpdateQuery(partition), new BatchPreparedStatementSetter() {
  55 + @Override
  56 + public void setValues(PreparedStatement ps, int i) throws SQLException {
  57 + TsKvEntity tsKvEntity = entries.get(i);
  58 + ps.setObject(1, tsKvEntity.getEntityId());
  59 + ps.setInt(2, tsKvEntity.getKey());
  60 + ps.setLong(3, tsKvEntity.getTs());
  61 +
  62 + if (tsKvEntity.getBooleanValue() != null) {
  63 + ps.setBoolean(4, tsKvEntity.getBooleanValue());
  64 + ps.setBoolean(8, tsKvEntity.getBooleanValue());
  65 + } else {
  66 + ps.setNull(4, Types.BOOLEAN);
  67 + ps.setNull(8, Types.BOOLEAN);
  68 + }
  69 +
  70 + ps.setString(5, replaceNullChars(tsKvEntity.getStrValue()));
  71 + ps.setString(9, replaceNullChars(tsKvEntity.getStrValue()));
  72 +
  73 +
  74 + if (tsKvEntity.getLongValue() != null) {
  75 + ps.setLong(6, tsKvEntity.getLongValue());
  76 + ps.setLong(10, tsKvEntity.getLongValue());
  77 + } else {
  78 + ps.setNull(6, Types.BIGINT);
  79 + ps.setNull(10, Types.BIGINT);
  80 + }
  81 +
  82 + if (tsKvEntity.getDoubleValue() != null) {
  83 + ps.setDouble(7, tsKvEntity.getDoubleValue());
  84 + ps.setDouble(11, tsKvEntity.getDoubleValue());
  85 + } else {
  86 + ps.setNull(7, Types.DOUBLE);
  87 + ps.setNull(11, Types.DOUBLE);
  88 + }
  89 + }
  90 +
  91 + @Override
  92 + public int getBatchSize() {
  93 + return entries.size();
  94 + }
  95 + }));
  96 + }
  97 +
  98 + private String getInsertOrUpdateQuery(String partitionDate) {
  99 + return INSERT_INTO_TS_KV + partitionDate + VALUES_ON_CONFLICT_DO_UPDATE;
  100 + }
  101 +}
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.sqlts.psql;
  17 +
  18 +import org.springframework.data.domain.Pageable;
  19 +import org.springframework.data.jpa.repository.Modifying;
  20 +import org.springframework.data.jpa.repository.Query;
  21 +import org.springframework.data.repository.CrudRepository;
  22 +import org.springframework.data.repository.query.Param;
  23 +import org.springframework.scheduling.annotation.Async;
  24 +import org.springframework.transaction.annotation.Transactional;
  25 +import org.thingsboard.server.dao.model.sqlts.psql.TsKvCompositeKey;
  26 +import org.thingsboard.server.dao.model.sqlts.psql.TsKvEntity;
  27 +import org.thingsboard.server.dao.util.SqlDao;
  28 +
  29 +import java.util.List;
  30 +import java.util.UUID;
  31 +import java.util.concurrent.CompletableFuture;
  32 +
  33 +@SqlDao
  34 +public interface TsKvPsqlRepository extends CrudRepository<TsKvEntity, TsKvCompositeKey> {
  35 +
  36 + @Query("SELECT tskv FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " +
  37 + "AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs")
  38 + List<TsKvEntity> findAllWithLimit(@Param("entityId") UUID entityId,
  39 + @Param("entityKey") int key,
  40 + @Param("startTs") long startTs,
  41 + @Param("endTs") long endTs,
  42 + Pageable pageable);
  43 +
  44 + @Transactional
  45 + @Modifying
  46 + @Query("DELETE FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " +
  47 + "AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs")
  48 + void delete(@Param("entityId") UUID entityId,
  49 + @Param("entityKey") int key,
  50 + @Param("startTs") long startTs,
  51 + @Param("endTs") long endTs);
  52 +
  53 + @Async
  54 + @Query("SELECT new TsKvEntity(MAX(tskv.strValue)) FROM TsKvEntity tskv " +
  55 + "WHERE tskv.strValue IS NOT NULL " +
  56 + "AND tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs")
  57 + CompletableFuture<TsKvEntity> findStringMax(@Param("entityId") UUID entityId,
  58 + @Param("entityKey") int entityKey,
  59 + @Param("startTs") long startTs,
  60 + @Param("endTs") long endTs);
  61 +
  62 + @Async
  63 + @Query("SELECT new TsKvEntity(MAX(COALESCE(tskv.longValue, -9223372036854775807)), " +
  64 + "MAX(COALESCE(tskv.doubleValue, -1.79769E+308)), " +
  65 + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " +
  66 + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " +
  67 + "'MAX') FROM TsKvEntity tskv " +
  68 + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs")
  69 + CompletableFuture<TsKvEntity> findNumericMax(@Param("entityId") UUID entityId,
  70 + @Param("entityKey") int entityKey,
  71 + @Param("startTs") long startTs,
  72 + @Param("endTs") long endTs);
  73 +
  74 +
  75 + @Async
  76 + @Query("SELECT new TsKvEntity(MIN(tskv.strValue)) FROM TsKvEntity tskv " +
  77 + "WHERE tskv.strValue IS NOT NULL " +
  78 + "AND tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs")
  79 + CompletableFuture<TsKvEntity> findStringMin(@Param("entityId") UUID entityId,
  80 + @Param("entityKey") int entityKey,
  81 + @Param("startTs") long startTs,
  82 + @Param("endTs") long endTs);
  83 +
  84 + @Async
  85 + @Query("SELECT new TsKvEntity(MIN(COALESCE(tskv.longValue, 9223372036854775807)), " +
  86 + "MIN(COALESCE(tskv.doubleValue, 1.79769E+308)), " +
  87 + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " +
  88 + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " +
  89 + "'MIN') FROM TsKvEntity tskv " +
  90 + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs")
  91 + CompletableFuture<TsKvEntity> findNumericMin(
  92 + @Param("entityId") UUID entityId,
  93 + @Param("entityKey") int entityKey,
  94 + @Param("startTs") long startTs,
  95 + @Param("endTs") long endTs);
  96 +
  97 + @Async
  98 + @Query("SELECT new TsKvEntity(SUM(CASE WHEN tskv.booleanValue IS NULL THEN 0 ELSE 1 END), " +
  99 + "SUM(CASE WHEN tskv.strValue IS NULL THEN 0 ELSE 1 END), " +
  100 + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " +
  101 + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END)) FROM TsKvEntity tskv " +
  102 + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs")
  103 + CompletableFuture<TsKvEntity> findCount(@Param("entityId") UUID entityId,
  104 + @Param("entityKey") int entityKey,
  105 + @Param("startTs") long startTs,
  106 + @Param("endTs") long endTs);
  107 +
  108 + @Async
  109 + @Query("SELECT new TsKvEntity(SUM(COALESCE(tskv.longValue, 0)), " +
  110 + "SUM(COALESCE(tskv.doubleValue, 0.0)), " +
  111 + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " +
  112 + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " +
  113 + "'AVG') FROM TsKvEntity tskv " +
  114 + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs")
  115 + CompletableFuture<TsKvEntity> findAvg(@Param("entityId") UUID entityId,
  116 + @Param("entityKey") int entityKey,
  117 + @Param("startTs") long startTs,
  118 + @Param("endTs") long endTs);
  119 +
  120 + @Async
  121 + @Query("SELECT new TsKvEntity(SUM(COALESCE(tskv.longValue, 0)), " +
  122 + "SUM(COALESCE(tskv.doubleValue, 0.0)), " +
  123 + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " +
  124 + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " +
  125 + "'SUM') FROM TsKvEntity tskv " +
  126 + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs")
  127 + CompletableFuture<TsKvEntity> findSum(@Param("entityId") UUID entityId,
  128 + @Param("entityKey") int entityKey,
  129 + @Param("startTs") long startTs,
  130 + @Param("endTs") long endTs);
  131 +
  132 +}
... ...
... ... @@ -23,6 +23,7 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao;
23 23 import javax.persistence.EntityManager;
24 24 import javax.persistence.PersistenceContext;
25 25 import java.util.List;
  26 +import java.util.UUID;
26 27 import java.util.concurrent.CompletableFuture;
27 28
28 29 @Repository
... ... @@ -36,7 +37,7 @@ public class AggregationRepository {
36 37 public static final String FIND_COUNT = "findCount";
37 38
38 39
39   - public static final String FROM_WHERE_CLAUSE = "FROM tenant_ts_kv tskv WHERE tskv.tenant_id = cast(:tenantId AS varchar) AND tskv.entity_id = cast(:entityId AS varchar) AND tskv.key= cast(:entityKey AS varchar) AND tskv.ts > :startTs AND tskv.ts <= :endTs GROUP BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket ORDER BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket";
  40 + public static final String FROM_WHERE_CLAUSE = "FROM tenant_ts_kv tskv WHERE tskv.tenant_id = cast(:tenantId AS uuid) AND tskv.entity_id = cast(:entityId AS uuid) AND tskv.key= cast(:entityKey AS int) AND tskv.ts > :startTs AND tskv.ts <= :endTs GROUP BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket ORDER BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket";
40 41
41 42 public static final String FIND_AVG_QUERY = "SELECT time_bucket(:timeBucket, tskv.ts) AS tsBucket, :timeBucket AS interval, SUM(COALESCE(tskv.long_v, 0)) AS longValue, SUM(COALESCE(tskv.dbl_v, 0.0)) AS doubleValue, SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, null AS strValue, 'AVG' AS aggType ";
42 43
... ... @@ -52,41 +53,41 @@ public class AggregationRepository {
52 53 private EntityManager entityManager;
53 54
54 55 @Async
55   - public CompletableFuture<List<TimescaleTsKvEntity>> findAvg(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) {
  56 + public CompletableFuture<List<TimescaleTsKvEntity>> findAvg(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
56 57 @SuppressWarnings("unchecked")
57 58 List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_AVG);
58 59 return CompletableFuture.supplyAsync(() -> resultList);
59 60 }
60 61
61 62 @Async
62   - public CompletableFuture<List<TimescaleTsKvEntity>> findMax(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) {
  63 + public CompletableFuture<List<TimescaleTsKvEntity>> findMax(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
63 64 @SuppressWarnings("unchecked")
64 65 List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_MAX);
65 66 return CompletableFuture.supplyAsync(() -> resultList);
66 67 }
67 68
68 69 @Async
69   - public CompletableFuture<List<TimescaleTsKvEntity>> findMin(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) {
  70 + public CompletableFuture<List<TimescaleTsKvEntity>> findMin(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
70 71 @SuppressWarnings("unchecked")
71 72 List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_MIN);
72 73 return CompletableFuture.supplyAsync(() -> resultList);
73 74 }
74 75
75 76 @Async
76   - public CompletableFuture<List<TimescaleTsKvEntity>> findSum(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) {
  77 + public CompletableFuture<List<TimescaleTsKvEntity>> findSum(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
77 78 @SuppressWarnings("unchecked")
78 79 List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_SUM);
79 80 return CompletableFuture.supplyAsync(() -> resultList);
80 81 }
81 82
82 83 @Async
83   - public CompletableFuture<List<TimescaleTsKvEntity>> findCount(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) {
  84 + public CompletableFuture<List<TimescaleTsKvEntity>> findCount(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
84 85 @SuppressWarnings("unchecked")
85 86 List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_COUNT);
86 87 return CompletableFuture.supplyAsync(() -> resultList);
87 88 }
88 89
89   - private List getResultList(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs, String query) {
  90 + private List getResultList(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs, String query) {
90 91 return entityManager.createNamedQuery(query)
91 92 .setParameter("tenantId", tenantId)
92 93 .setParameter("entityId", entityId)
... ...
... ... @@ -19,7 +19,9 @@ import org.springframework.jdbc.core.BatchPreparedStatementSetter;
19 19 import org.springframework.stereotype.Repository;
20 20 import org.springframework.transaction.annotation.Transactional;
21 21 import org.thingsboard.server.dao.model.sqlts.timescale.TimescaleTsKvEntity;
22   -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository;
  22 +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository;
  23 +import org.thingsboard.server.dao.sqlts.EntityContainer;
  24 +import org.thingsboard.server.dao.sqlts.InsertTsRepository;
23 25 import org.thingsboard.server.dao.util.PsqlDao;
24 26 import org.thingsboard.server.dao.util.TimescaleDBTsDao;
25 27
... ... @@ -32,35 +34,21 @@ import java.util.List;
32 34 @PsqlDao
33 35 @Repository
34 36 @Transactional
35   -public class TimescaleInsertRepository extends AbstractTimeseriesInsertRepository<TimescaleTsKvEntity> {
36   -
37   - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateString(BOOL_V, PSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS);
38   - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateString(STR_V, PSQL_ON_STR_VALUE_UPDATE_SET_NULLS);
39   - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateString(LONG_V, PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS);
40   - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateString(DBL_V, PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS);
41   -
42   - private static final String BATCH_UPDATE =
43   - "UPDATE tenant_ts_kv SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ? WHERE entity_type = ? AND entity_id = ? and key = ? and ts = ?";
44   -
  37 +public class TimescaleInsertRepository extends AbstractInsertRepository implements InsertTsRepository<TimescaleTsKvEntity> {
45 38
46 39 private static final String INSERT_OR_UPDATE =
47 40 "INSERT INTO tenant_ts_kv (tenant_id, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) VALUES(?, ?, ?, ?, ?, ?, ?, ?) " +
48 41 "ON CONFLICT (tenant_id, entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?;";
49 42
50 43 @Override
51   - public void saveOrUpdate(TimescaleTsKvEntity entity) {
52   - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT);
53   - }
54   -
55   - @Override
56   - public void saveOrUpdate(List<TimescaleTsKvEntity> entities) {
  44 + public void saveOrUpdate(List<EntityContainer<TimescaleTsKvEntity>> entities) {
57 45 jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() {
58 46 @Override
59 47 public void setValues(PreparedStatement ps, int i) throws SQLException {
60   - TimescaleTsKvEntity tsKvEntity = entities.get(i);
61   - ps.setString(1, tsKvEntity.getTenantId());
62   - ps.setString(2, tsKvEntity.getEntityId());
63   - ps.setString(3, tsKvEntity.getKey());
  48 + TimescaleTsKvEntity tsKvEntity = entities.get(i).getEntity();
  49 + ps.setObject(1, tsKvEntity.getTenantId());
  50 + ps.setObject(2, tsKvEntity.getEntityId());
  51 + ps.setInt(3, tsKvEntity.getKey());
64 52 ps.setLong(4, tsKvEntity.getTs());
65 53
66 54 if (tsKvEntity.getBooleanValue() != null) {
... ... @@ -98,52 +86,4 @@ public class TimescaleInsertRepository extends AbstractTimeseriesInsertRepositor
98 86 }
99 87 });
100 88 }
101   -
102   - @Override
103   - protected void saveOrUpdateBoolean(TimescaleTsKvEntity entity, String query) {
104   - entityManager.createNativeQuery(query)
105   - .setParameter("tenant_id", entity.getTenantId())
106   - .setParameter("entity_id", entity.getEntityId())
107   - .setParameter("key", entity.getKey())
108   - .setParameter("ts", entity.getTs())
109   - .setParameter("bool_v", entity.getBooleanValue())
110   - .executeUpdate();
111   - }
112   -
113   - @Override
114   - protected void saveOrUpdateString(TimescaleTsKvEntity entity, String query) {
115   - entityManager.createNativeQuery(query)
116   - .setParameter("tenant_id", entity.getTenantId())
117   - .setParameter("entity_id", entity.getEntityId())
118   - .setParameter("key", entity.getKey())
119   - .setParameter("ts", entity.getTs())
120   - .setParameter("str_v", replaceNullChars(entity.getStrValue()))
121   - .executeUpdate();
122   - }
123   -
124   - @Override
125   - protected void saveOrUpdateLong(TimescaleTsKvEntity entity, String query) {
126   - entityManager.createNativeQuery(query)
127   - .setParameter("tenant_id", entity.getTenantId())
128   - .setParameter("entity_id", entity.getEntityId())
129   - .setParameter("key", entity.getKey())
130   - .setParameter("ts", entity.getTs())
131   - .setParameter("long_v", entity.getLongValue())
132   - .executeUpdate();
133   - }
134   -
135   - @Override
136   - protected void saveOrUpdateDouble(TimescaleTsKvEntity entity, String query) {
137   - entityManager.createNativeQuery(query)
138   - .setParameter("tenant_id", entity.getTenantId())
139   - .setParameter("entity_id", entity.getEntityId())
140   - .setParameter("key", entity.getKey())
141   - .setParameter("ts", entity.getTs())
142   - .setParameter("dbl_v", entity.getDoubleValue())
143   - .executeUpdate();
144   - }
145   -
146   - private static String getInsertOrUpdateString(String value, String nullValues) {
147   - return "INSERT INTO tenant_ts_kv(tenant_id, entity_id, key, ts, " + value + ") VALUES (:tenant_id, :entity_id, :key, :ts, :" + value + ") ON CONFLICT (tenant_id, entity_id, key, ts) DO UPDATE SET " + value + " = :" + value + ", ts = :ts," + nullValues;
148   - }
149 89 }
\ No newline at end of file
... ...
... ... @@ -15,11 +15,11 @@
15 15 */
16 16 package org.thingsboard.server.dao.sqlts.timescale;
17 17
18   -import com.google.common.collect.Lists;
19 18 import com.google.common.util.concurrent.Futures;
20 19 import com.google.common.util.concurrent.ListenableFuture;
21 20 import com.google.common.util.concurrent.SettableFuture;
22 21 import lombok.extern.slf4j.Slf4j;
  22 +import org.hibernate.exception.ConstraintViolationException;
23 23 import org.springframework.beans.factory.annotation.Autowired;
24 24 import org.springframework.beans.factory.annotation.Value;
25 25 import org.springframework.data.domain.PageRequest;
... ... @@ -29,19 +29,19 @@ import org.springframework.util.CollectionUtils;
29 29 import org.thingsboard.server.common.data.id.EntityId;
30 30 import org.thingsboard.server.common.data.id.TenantId;
31 31 import org.thingsboard.server.common.data.kv.Aggregation;
32   -import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
33 32 import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
34 33 import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
35   -import org.thingsboard.server.common.data.kv.StringDataEntry;
36 34 import org.thingsboard.server.common.data.kv.TsKvEntry;
37   -import org.thingsboard.server.common.data.kv.TsKvQuery;
38 35 import org.thingsboard.server.dao.DaoUtil;
  36 +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary;
  37 +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey;
39 38 import org.thingsboard.server.dao.model.sqlts.timescale.TimescaleTsKvEntity;
40   -import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent;
41 39 import org.thingsboard.server.dao.sql.TbSqlBlockingQueue;
42 40 import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams;
43 41 import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao;
44   -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository;
  42 +import org.thingsboard.server.dao.sqlts.EntityContainer;
  43 +import org.thingsboard.server.dao.sqlts.InsertTsRepository;
  44 +import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository;
45 45 import org.thingsboard.server.dao.timeseries.TimeseriesDao;
46 46 import org.thingsboard.server.dao.util.TimescaleDBTsDao;
47 47
... ... @@ -51,9 +51,11 @@ import java.util.ArrayList;
51 51 import java.util.Collections;
52 52 import java.util.List;
53 53 import java.util.Optional;
  54 +import java.util.UUID;
54 55 import java.util.concurrent.CompletableFuture;
55   -
56   -import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID;
  56 +import java.util.concurrent.ConcurrentHashMap;
  57 +import java.util.concurrent.ConcurrentMap;
  58 +import java.util.concurrent.locks.ReentrantLock;
57 59
58 60
59 61 @Component
... ... @@ -63,17 +65,21 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
63 65
64 66 private static final String TS = "ts";
65 67
  68 + private final ConcurrentMap<String, Integer> tsKvDictionaryMap = new ConcurrentHashMap<>();
  69 +
  70 + private static final ReentrantLock tsCreationLock = new ReentrantLock();
  71 +
66 72 @Autowired
67   - private TsKvTimescaleRepository tsKvRepository;
  73 + private TsKvDictionaryRepository dictionaryRepository;
68 74
69 75 @Autowired
70   - private AggregationRepository aggregationRepository;
  76 + private TsKvTimescaleRepository tsKvRepository;
71 77
72 78 @Autowired
73   - private AbstractTimeseriesInsertRepository insertRepository;
  79 + private AggregationRepository aggregationRepository;
74 80
75 81 @Autowired
76   - ScheduledLogExecutorComponent logExecutor;
  82 + private InsertTsRepository<TimescaleTsKvEntity> insertRepository;
77 83
78 84 @Value("${sql.ts_timescale.batch_size:1000}")
79 85 private int batchSize;
... ... @@ -84,10 +90,11 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
84 90 @Value("${sql.ts_timescale.stats_print_interval_ms:1000}")
85 91 private long statsPrintIntervalMs;
86 92
87   - private TbSqlBlockingQueue<TimescaleTsKvEntity> queue;
  93 + private TbSqlBlockingQueue<EntityContainer<TimescaleTsKvEntity>> queue;
88 94
89 95 @PostConstruct
90   - private void init() {
  96 + protected void init() {
  97 + super.init();
91 98 TbSqlBlockingQueueParams params = TbSqlBlockingQueueParams.builder()
92 99 .logName("TS Timescale")
93 100 .batchSize(batchSize)
... ... @@ -99,17 +106,13 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
99 106 }
100 107
101 108 @PreDestroy
102   - private void destroy() {
  109 + protected void destroy() {
  110 + super.init();
103 111 if (queue != null) {
104 112 queue.destroy();
105 113 }
106 114 }
107 115
108   - @Override
109   - public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) {
110   - return processFindAllAsync(tenantId, entityId, queries);
111   - }
112   -
113 116 protected ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
114 117 if (query.getAggregation() == Aggregation.NONE) {
115 118 return findAllAsyncWithLimit(tenantId, entityId, query);
... ... @@ -122,50 +125,36 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
122 125 }
123 126 }
124 127
125   - private ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
126   - return Futures.immediateFuture(
127   - DaoUtil.convertDataList(
128   - tsKvRepository.findAllWithLimit(
129   - fromTimeUUID(tenantId.getId()),
130   - fromTimeUUID(entityId.getId()),
131   - query.getKey(),
132   - query.getStartTs(),
133   - query.getEndTs(),
134   - new PageRequest(0, query.getLimit(),
135   - new Sort(Sort.Direction.fromString(
136   - query.getOrderBy()), "ts")))));
  128 + @Override
  129 + public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) {
  130 + return processFindAllAsync(tenantId, entityId, queries);
137 131 }
138 132
139   -
140 133 @Override
141 134 public ListenableFuture<TsKvEntry> findLatest(TenantId tenantId, EntityId entityId, String key) {
142   - ListenableFuture<List<TimescaleTsKvEntity>> future = getLatest(tenantId, entityId, key, 0L, System.currentTimeMillis());
143   - return Futures.transform(future, latest -> {
144   - if (!CollectionUtils.isEmpty(latest)) {
145   - return DaoUtil.getData(latest.get(0));
146   - } else {
147   - return new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null));
148   - }
149   - }, service);
  135 + return getFindLatestFuture(entityId, key);
150 136 }
151 137
152 138 @Override
153 139 public ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId) {
154   - return Futures.immediateFuture(DaoUtil.convertDataList(Lists.newArrayList(tsKvRepository.findAllLatestValues(fromTimeUUID(tenantId.getId()), fromTimeUUID(entityId.getId())))));
  140 + return getFindAllLatestFuture(entityId);
155 141 }
156 142
157 143 @Override
158 144 public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) {
  145 + String strKey = tsKvEntry.getKey();
  146 + Integer keyId = getOrSaveKeyId(strKey);
159 147 TimescaleTsKvEntity entity = new TimescaleTsKvEntity();
160   - entity.setTenantId(fromTimeUUID(tenantId.getId()));
161   - entity.setEntityId(fromTimeUUID(entityId.getId()));
  148 + entity.setTenantId(tenantId.getId());
  149 + entity.setEntityId(entityId.getId());
162 150 entity.setTs(tsKvEntry.getTs());
163   - entity.setKey(tsKvEntry.getKey());
  151 + entity.setKey(keyId);
164 152 entity.setStrValue(tsKvEntry.getStrValue().orElse(null));
165 153 entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null));
166 154 entity.setLongValue(tsKvEntry.getLongValue().orElse(null));
167 155 entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null));
168   - return queue.add(entity);
  156 + log.trace("Saving entity to timescale db: {}", entity);
  157 + return queue.add(new EntityContainer(entity, null));
169 158 }
170 159
171 160 @Override
... ... @@ -175,16 +164,18 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
175 164
176 165 @Override
177 166 public ListenableFuture<Void> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
178   - return Futures.immediateFuture(null);
  167 + return getSaveLatestFuture(entityId, tsKvEntry);
179 168 }
180 169
181 170 @Override
182 171 public ListenableFuture<Void> remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
  172 + String strKey = query.getKey();
  173 + Integer keyId = getOrSaveKeyId(strKey);
183 174 return service.submit(() -> {
184 175 tsKvRepository.delete(
185   - fromTimeUUID(tenantId.getId()),
186   - fromTimeUUID(entityId.getId()),
187   - query.getKey(),
  176 + tenantId.getId(),
  177 + entityId.getId(),
  178 + keyId,
188 179 query.getStartTs(),
189 180 query.getEndTs());
190 181 return null;
... ... @@ -193,7 +184,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
193 184
194 185 @Override
195 186 public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
196   - return service.submit(() -> null);
  187 + return getRemoveLatestFuture(tenantId, entityId, query);
197 188 }
198 189
199 190 @Override
... ... @@ -201,37 +192,60 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
201 192 return service.submit(() -> null);
202 193 }
203 194
204   - private ListenableFuture<Void> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
205   - ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(tenantId, entityId, query);
206   - return Futures.transformAsync(future, entryList -> {
207   - if (entryList.size() == 1) {
208   - return save(tenantId, entityId, entryList.get(0), 0L);
209   - } else {
210   - log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey());
211   - }
212   - return Futures.immediateFuture(null);
213   - }, service);
214   - }
215   -
216   - private ListenableFuture<List<TimescaleTsKvEntity>> findLatestByQuery(TenantId tenantId, EntityId entityId, TsKvQuery query) {
217   - return getLatest(tenantId, entityId, query.getKey(), query.getStartTs(), query.getEndTs());
  195 + private ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
  196 + String strKey = query.getKey();
  197 + Integer keyId = getOrSaveKeyId(strKey);
  198 + List<TimescaleTsKvEntity> timescaleTsKvEntities = tsKvRepository.findAllWithLimit(
  199 + tenantId.getId(),
  200 + entityId.getId(),
  201 + keyId,
  202 + query.getStartTs(),
  203 + query.getEndTs(),
  204 + new PageRequest(0, query.getLimit(),
  205 + new Sort(Sort.Direction.fromString(
  206 + query.getOrderBy()), TS)));
  207 + timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey));
  208 + return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities));
218 209 }
219 210
220   - private ListenableFuture<List<TimescaleTsKvEntity>> getLatest(TenantId tenantId, EntityId entityId, String key, long start, long end) {
221   - return Futures.immediateFuture(tsKvRepository.findAllWithLimit(
222   - fromTimeUUID(tenantId.getId()),
223   - fromTimeUUID(entityId.getId()),
224   - key,
225   - start,
226   - end,
227   - new PageRequest(0, 1,
228   - new Sort(Sort.Direction.DESC, TS))));
  211 + private Integer getOrSaveKeyId(String strKey) {
  212 + Integer keyId = tsKvDictionaryMap.get(strKey);
  213 + if (keyId == null) {
  214 + Optional<TsKvDictionary> tsKvDictionaryOptional;
  215 + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey));
  216 + if (!tsKvDictionaryOptional.isPresent()) {
  217 + tsCreationLock.lock();
  218 + try {
  219 + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey));
  220 + if (!tsKvDictionaryOptional.isPresent()) {
  221 + TsKvDictionary tsKvDictionary = new TsKvDictionary();
  222 + tsKvDictionary.setKey(strKey);
  223 + try {
  224 + TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary);
  225 + tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId());
  226 + keyId = saved.getKeyId();
  227 + } catch (ConstraintViolationException e) {
  228 + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey));
  229 + TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!"));
  230 + tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId());
  231 + keyId = dictionary.getKeyId();
  232 + }
  233 + } else {
  234 + keyId = tsKvDictionaryOptional.get().getKeyId();
  235 + }
  236 + } finally {
  237 + tsCreationLock.unlock();
  238 + }
  239 + } else {
  240 + keyId = tsKvDictionaryOptional.get().getKeyId();
  241 + tsKvDictionaryMap.put(strKey, keyId);
  242 + }
  243 + }
  244 + return keyId;
229 245 }
230 246
231 247 private ListenableFuture<List<Optional<TsKvEntry>>> findAndAggregateAsync(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, long timeBucket, Aggregation aggregation) {
232   - String entityIdStr = fromTimeUUID(entityId.getId());
233   - String tenantIdStr = fromTimeUUID(tenantId.getId());
234   - CompletableFuture<List<TimescaleTsKvEntity>> listCompletableFuture = switchAgregation(key, startTs, endTs, timeBucket, aggregation, entityIdStr, tenantIdStr);
  248 + CompletableFuture<List<TimescaleTsKvEntity>> listCompletableFuture = switchAgregation(key, startTs, endTs, timeBucket, aggregation, entityId.getId(), tenantId.getId());
235 249 SettableFuture<List<TimescaleTsKvEntity>> listenableFuture = SettableFuture.create();
236 250 listCompletableFuture.whenComplete((timescaleTsKvEntities, throwable) -> {
237 251 if (throwable != null) {
... ... @@ -245,9 +259,9 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
245 259 List<Optional<TsKvEntry>> result = new ArrayList<>();
246 260 timescaleTsKvEntities.forEach(entity -> {
247 261 if (entity != null && entity.isNotEmpty()) {
248   - entity.setEntityId(entityIdStr);
249   - entity.setTenantId(tenantIdStr);
250   - entity.setKey(key);
  262 + entity.setEntityId(entityId.getId());
  263 + entity.setTenantId(tenantId.getId());
  264 + entity.setStrKey(key);
251 265 result.add(Optional.of(DaoUtil.getData(entity)));
252 266 } else {
253 267 result.add(Optional.empty());
... ... @@ -260,69 +274,74 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
260 274 });
261 275 }
262 276
263   - private CompletableFuture<List<TimescaleTsKvEntity>> switchAgregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, String entityIdStr, String tenantIdStr) {
  277 + private CompletableFuture<List<TimescaleTsKvEntity>> switchAgregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, UUID entityId, UUID tenantId) {
264 278 switch (aggregation) {
265 279 case AVG:
266   - return findAvg(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr);
  280 + return findAvg(key, startTs, endTs, timeBucket, entityId, tenantId);
267 281 case MAX:
268   - return findMax(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr);
  282 + return findMax(key, startTs, endTs, timeBucket, entityId, tenantId);
269 283 case MIN:
270   - return findMin(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr);
  284 + return findMin(key, startTs, endTs, timeBucket, entityId, tenantId);
271 285 case SUM:
272   - return findSum(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr);
  286 + return findSum(key, startTs, endTs, timeBucket, entityId, tenantId);
273 287 case COUNT:
274   - return findCount(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr);
  288 + return findCount(key, startTs, endTs, timeBucket, entityId, tenantId);
275 289 default:
276 290 throw new IllegalArgumentException("Not supported aggregation type: " + aggregation);
277 291 }
278 292 }
279 293
280   - private CompletableFuture<List<TimescaleTsKvEntity>> findAvg(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) {
  294 + private CompletableFuture<List<TimescaleTsKvEntity>> findAvg(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
  295 + Integer keyId = getOrSaveKeyId(key);
281 296 return aggregationRepository.findAvg(
282   - tenantIdStr,
283   - entityIdStr,
284   - key,
  297 + tenantId,
  298 + entityId,
  299 + keyId,
285 300 timeBucket,
286 301 startTs,
287 302 endTs);
288 303 }
289 304
290   - private CompletableFuture<List<TimescaleTsKvEntity>> findMax(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) {
  305 + private CompletableFuture<List<TimescaleTsKvEntity>> findMax(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
  306 + Integer keyId = getOrSaveKeyId(key);
291 307 return aggregationRepository.findMax(
292   - tenantIdStr,
293   - entityIdStr,
294   - key,
  308 + tenantId,
  309 + entityId,
  310 + keyId,
295 311 timeBucket,
296 312 startTs,
297 313 endTs);
298 314 }
299 315
300   - private CompletableFuture<List<TimescaleTsKvEntity>> findMin(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) {
  316 + private CompletableFuture<List<TimescaleTsKvEntity>> findMin(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
  317 + Integer keyId = getOrSaveKeyId(key);
301 318 return aggregationRepository.findMin(
302   - tenantIdStr,
303   - entityIdStr,
304   - key,
  319 + tenantId,
  320 + entityId,
  321 + keyId,
305 322 timeBucket,
306 323 startTs,
307 324 endTs);
308 325
309 326 }
310 327
311   - private CompletableFuture<List<TimescaleTsKvEntity>> findSum(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) {
  328 + private CompletableFuture<List<TimescaleTsKvEntity>> findSum(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
  329 + Integer keyId = getOrSaveKeyId(key);
312 330 return aggregationRepository.findSum(
313   - tenantIdStr,
314   - entityIdStr,
315   - key,
  331 + tenantId,
  332 + entityId,
  333 + keyId,
316 334 timeBucket,
317 335 startTs,
318 336 endTs);
319 337 }
320 338
321   - private CompletableFuture<List<TimescaleTsKvEntity>> findCount(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) {
  339 + private CompletableFuture<List<TimescaleTsKvEntity>> findCount(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
  340 + Integer keyId = getOrSaveKeyId(key);
322 341 return aggregationRepository.findCount(
323   - tenantIdStr,
324   - entityIdStr,
325   - key,
  342 + tenantId,
  343 + entityId,
  344 + keyId,
326 345 timeBucket,
327 346 startTs,
328 347 endTs);
... ...
... ... @@ -26,6 +26,7 @@ import org.thingsboard.server.dao.model.sqlts.timescale.TimescaleTsKvEntity;
26 26 import org.thingsboard.server.dao.util.TimescaleDBTsDao;
27 27
28 28 import java.util.List;
  29 +import java.util.UUID;
29 30
30 31 @TimescaleDBTsDao
31 32 public interface TsKvTimescaleRepository extends CrudRepository<TimescaleTsKvEntity, TimescaleTsKvCompositeKey> {
... ... @@ -35,31 +36,21 @@ public interface TsKvTimescaleRepository extends CrudRepository<TimescaleTsKvEnt
35 36 "AND tskv.key = :entityKey " +
36 37 "AND tskv.ts > :startTs AND tskv.ts <= :endTs")
37 38 List<TimescaleTsKvEntity> findAllWithLimit(
38   - @Param("tenantId") String tenantId,
39   - @Param("entityId") String entityId,
40   - @Param("entityKey") String key,
  39 + @Param("tenantId") UUID tenantId,
  40 + @Param("entityId") UUID entityId,
  41 + @Param("entityKey") int key,
41 42 @Param("startTs") long startTs,
42 43 @Param("endTs") long endTs, Pageable pageable);
43 44
44   - @Query(value = "SELECT tskv.tenant_id as tenant_id, tskv.entity_id as entity_id, tskv.key as key, last(tskv.ts,tskv.ts) as ts," +
45   - " last(tskv.bool_v, tskv.ts) as bool_v, last(tskv.str_v, tskv.ts) as str_v," +
46   - " last(tskv.long_v, tskv.ts) as long_v, last(tskv.dbl_v, tskv.ts) as dbl_v" +
47   - " FROM tenant_ts_kv tskv WHERE tskv.tenant_id = cast(:tenantId AS varchar) " +
48   - "AND tskv.entity_id = cast(:entityId AS varchar) " +
49   - "GROUP BY tskv.tenant_id, tskv.entity_id, tskv.key", nativeQuery = true)
50   - List<TimescaleTsKvEntity> findAllLatestValues(
51   - @Param("tenantId") String tenantId,
52   - @Param("entityId") String entityId);
53   -
54 45 @Transactional
55 46 @Modifying
56 47 @Query("DELETE FROM TimescaleTsKvEntity tskv WHERE tskv.tenantId = :tenantId " +
57 48 "AND tskv.entityId = :entityId " +
58 49 "AND tskv.key = :entityKey " +
59 50 "AND tskv.ts > :startTs AND tskv.ts <= :endTs")
60   - void delete(@Param("tenantId") String tenantId,
61   - @Param("entityId") String entityId,
62   - @Param("entityKey") String key,
  51 + void delete(@Param("tenantId") UUID tenantId,
  52 + @Param("entityId") UUID entityId,
  53 + @Param("entityKey") int key,
63 54 @Param("startTs") long startTs,
64 55 @Param("endTs") long endTs);
65 56
... ...
1   -/**
2   - * Copyright © 2016-2020 The Thingsboard Authors
3   - *
4   - * Licensed under the Apache License, Version 2.0 (the "License");
5   - * you may not use this file except in compliance with the License.
6   - * You may obtain a copy of the License at
7   - *
8   - * http://www.apache.org/licenses/LICENSE-2.0
9   - *
10   - * Unless required by applicable law or agreed to in writing, software
11   - * distributed under the License is distributed on an "AS IS" BASIS,
12   - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   - * See the License for the specific language governing permissions and
14   - * limitations under the License.
15   - */
16   -package org.thingsboard.server.dao.sqlts.ts;
17   -
18   -import com.google.common.collect.Lists;
19   -import com.google.common.util.concurrent.FutureCallback;
20   -import com.google.common.util.concurrent.Futures;
21   -import com.google.common.util.concurrent.ListenableFuture;
22   -import com.google.common.util.concurrent.SettableFuture;
23   -import lombok.extern.slf4j.Slf4j;
24   -import org.springframework.beans.factory.annotation.Autowired;
25   -import org.springframework.beans.factory.annotation.Value;
26   -import org.springframework.data.domain.PageRequest;
27   -import org.springframework.data.domain.Sort;
28   -import org.springframework.stereotype.Component;
29   -import org.thingsboard.server.common.data.UUIDConverter;
30   -import org.thingsboard.server.common.data.id.EntityId;
31   -import org.thingsboard.server.common.data.id.TenantId;
32   -import org.thingsboard.server.common.data.kv.Aggregation;
33   -import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
34   -import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
35   -import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
36   -import org.thingsboard.server.common.data.kv.StringDataEntry;
37   -import org.thingsboard.server.common.data.kv.TsKvEntry;
38   -import org.thingsboard.server.dao.DaoUtil;
39   -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity;
40   -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestCompositeKey;
41   -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity;
42   -import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent;
43   -import org.thingsboard.server.dao.sql.TbSqlBlockingQueue;
44   -import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams;
45   -import org.thingsboard.server.dao.sqlts.AbstractLatestInsertRepository;
46   -import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao;
47   -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository;
48   -import org.thingsboard.server.dao.timeseries.SimpleListenableFuture;
49   -import org.thingsboard.server.dao.timeseries.TimeseriesDao;
50   -import org.thingsboard.server.dao.util.SqlTsDao;
51   -
52   -import javax.annotation.Nullable;
53   -import javax.annotation.PostConstruct;
54   -import javax.annotation.PreDestroy;
55   -import java.util.ArrayList;
56   -import java.util.List;
57   -import java.util.Optional;
58   -import java.util.concurrent.CompletableFuture;
59   -import java.util.concurrent.ExecutionException;
60   -import java.util.stream.Collectors;
61   -
62   -import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID;
63   -
64   -
65   -@Component
66   -@Slf4j
67   -@SqlTsDao
68   -public class JpaTimeseriesDao extends AbstractSqlTimeseriesDao implements TimeseriesDao {
69   -
70   - @Autowired
71   - private TsKvRepository tsKvRepository;
72   -
73   - @Autowired
74   - private TsKvLatestRepository tsKvLatestRepository;
75   -
76   - @Autowired
77   - private AbstractTimeseriesInsertRepository insertRepository;
78   -
79   - @Autowired
80   - private AbstractLatestInsertRepository insertLatestRepository;
81   -
82   - @Autowired
83   - ScheduledLogExecutorComponent logExecutor;
84   -
85   - @Value("${sql.ts.batch_size:1000}")
86   - private int tsBatchSize;
87   -
88   - @Value("${sql.ts.batch_max_delay:100}")
89   - private long tsMaxDelay;
90   -
91   - @Value("${sql.ts.stats_print_interval_ms:1000}")
92   - private long tsStatsPrintIntervalMs;
93   -
94   - @Value("${sql.ts_latest.batch_size:1000}")
95   - private int tsLatestBatchSize;
96   -
97   - @Value("${sql.ts_latest.batch_max_delay:100}")
98   - private long tsLatestMaxDelay;
99   -
100   - @Value("${sql.ts_latest.stats_print_interval_ms:1000}")
101   - private long tsLatestStatsPrintIntervalMs;
102   -
103   - private TbSqlBlockingQueue<TsKvEntity> tsQueue;
104   - private TbSqlBlockingQueue<TsKvLatestEntity> tsLatestQueue;
105   -
106   -
107   - @PostConstruct
108   - private void init() {
109   - TbSqlBlockingQueueParams tsParams = TbSqlBlockingQueueParams.builder()
110   - .logName("TS")
111   - .batchSize(tsBatchSize)
112   - .maxDelay(tsMaxDelay)
113   - .statsPrintIntervalMs(tsStatsPrintIntervalMs)
114   - .build();
115   - tsQueue = new TbSqlBlockingQueue<>(tsParams);
116   - tsQueue.init(logExecutor, v -> insertRepository.saveOrUpdate(v));
117   -
118   - TbSqlBlockingQueueParams tsLatestParams = TbSqlBlockingQueueParams.builder()
119   - .logName("TS Latest")
120   - .batchSize(tsLatestBatchSize)
121   - .maxDelay(tsLatestMaxDelay)
122   - .statsPrintIntervalMs(tsLatestStatsPrintIntervalMs)
123   - .build();
124   - tsLatestQueue = new TbSqlBlockingQueue<>(tsLatestParams);
125   - tsLatestQueue.init(logExecutor, v -> insertLatestRepository.saveOrUpdate(v));
126   - }
127   -
128   - @PreDestroy
129   - private void destroy() {
130   - if (tsQueue != null) {
131   - tsQueue.destroy();
132   - }
133   -
134   - if (tsLatestQueue != null) {
135   - tsLatestQueue.destroy();
136   - }
137   - }
138   -
139   - @Override
140   - public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) {
141   - return processFindAllAsync(tenantId, entityId, queries);
142   - }
143   -
144   - protected ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
145   - if (query.getAggregation() == Aggregation.NONE) {
146   - return findAllAsyncWithLimit(entityId, query);
147   - } else {
148   - long stepTs = query.getStartTs();
149   - List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>();
150   - while (stepTs < query.getEndTs()) {
151   - long startTs = stepTs;
152   - long endTs = stepTs + query.getInterval();
153   - long ts = startTs + (endTs - startTs) / 2;
154   - futures.add(findAndAggregateAsync(tenantId, entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()));
155   - stepTs = endTs;
156   - }
157   - return getTskvEntriesFuture(Futures.allAsList(futures));
158   - }
159   - }
160   -
161   - private ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) {
162   - List<CompletableFuture<TsKvEntity>> entitiesFutures = new ArrayList<>();
163   - String entityIdStr = fromTimeUUID(entityId.getId());
164   - switchAgregation(entityId, key, startTs, endTs, aggregation, entitiesFutures, entityIdStr);
165   -
166   - SettableFuture<TsKvEntity> listenableFuture = SettableFuture.create();
167   -
168   - CompletableFuture<List<TsKvEntity>> entities =
169   - CompletableFuture.allOf(entitiesFutures.toArray(new CompletableFuture[entitiesFutures.size()]))
170   - .thenApply(v -> entitiesFutures.stream()
171   - .map(CompletableFuture::join)
172   - .collect(Collectors.toList()));
173   -
174   - entities.whenComplete((tsKvEntities, throwable) -> {
175   - if (throwable != null) {
176   - listenableFuture.setException(throwable);
177   - } else {
178   - TsKvEntity result = null;
179   - for (TsKvEntity entity : tsKvEntities) {
180   - if (entity.isNotEmpty()) {
181   - result = entity;
182   - break;
183   - }
184   - }
185   - listenableFuture.set(result);
186   - }
187   - });
188   - return Futures.transform(listenableFuture, entity -> {
189   - if (entity != null && entity.isNotEmpty()) {
190   - entity.setEntityId(entityIdStr);
191   - entity.setEntityType(entityId.getEntityType());
192   - entity.setKey(key);
193   - entity.setTs(ts);
194   - return Optional.of(DaoUtil.getData(entity));
195   - } else {
196   - return Optional.empty();
197   - }
198   - });
199   - }
200   -
201   - private void switchAgregation(EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation, List<CompletableFuture<TsKvEntity>> entitiesFutures, String entityIdStr) {
202   - switch (aggregation) {
203   - case AVG:
204   - findAvg(entityId, key, startTs, endTs, entitiesFutures, entityIdStr);
205   - break;
206   - case MAX:
207   - findMax(entityId, key, startTs, endTs, entitiesFutures, entityIdStr);
208   - break;
209   - case MIN:
210   - findMin(entityId, key, startTs, endTs, entitiesFutures, entityIdStr);
211   - break;
212   - case SUM:
213   - findSum(entityId, key, startTs, endTs, entitiesFutures, entityIdStr);
214   - break;
215   - case COUNT:
216   - findCount(entityId, key, startTs, endTs, entitiesFutures, entityIdStr);
217   - break;
218   - default:
219   - throw new IllegalArgumentException("Not supported aggregation type: " + aggregation);
220   - }
221   - }
222   -
223   - private void findCount(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures, String entityIdStr) {
224   - entitiesFutures.add(tsKvRepository.findCount(
225   - entityIdStr,
226   - entityId.getEntityType(),
227   - key,
228   - startTs,
229   - endTs));
230   - }
231   -
232   - private void findSum(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures, String entityIdStr) {
233   - entitiesFutures.add(tsKvRepository.findSum(
234   - entityIdStr,
235   - entityId.getEntityType(),
236   - key,
237   - startTs,
238   - endTs));
239   - }
240   -
241   - private void findMin(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures, String entityIdStr) {
242   - entitiesFutures.add(tsKvRepository.findStringMin(
243   - entityIdStr,
244   - entityId.getEntityType(),
245   - key,
246   - startTs,
247   - endTs));
248   - entitiesFutures.add(tsKvRepository.findNumericMin(
249   - entityIdStr,
250   - entityId.getEntityType(),
251   - key,
252   - startTs,
253   - endTs));
254   - }
255   -
256   - private void findMax(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures, String entityIdStr) {
257   - entitiesFutures.add(tsKvRepository.findStringMax(
258   - entityIdStr,
259   - entityId.getEntityType(),
260   - key,
261   - startTs,
262   - endTs));
263   - entitiesFutures.add(tsKvRepository.findNumericMax(
264   - entityIdStr,
265   - entityId.getEntityType(),
266   - key,
267   - startTs,
268   - endTs));
269   - }
270   -
271   - private void findAvg(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures, String entityIdStr) {
272   - entitiesFutures.add(tsKvRepository.findAvg(
273   - entityIdStr,
274   - entityId.getEntityType(),
275   - key,
276   - startTs,
277   - endTs));
278   - }
279   -
280   - private ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) {
281   - return Futures.immediateFuture(
282   - DaoUtil.convertDataList(
283   - tsKvRepository.findAllWithLimit(
284   - fromTimeUUID(entityId.getId()),
285   - entityId.getEntityType(),
286   - query.getKey(),
287   - query.getStartTs(),
288   - query.getEndTs(),
289   - new PageRequest(0, query.getLimit(),
290   - new Sort(Sort.Direction.fromString(
291   - query.getOrderBy()), "ts")))));
292   - }
293   -
294   - @Override
295   - public ListenableFuture<TsKvEntry> findLatest(TenantId tenantId, EntityId entityId, String key) {
296   - TsKvLatestCompositeKey compositeKey =
297   - new TsKvLatestCompositeKey(
298   - entityId.getEntityType(),
299   - fromTimeUUID(entityId.getId()),
300   - key);
301   - Optional<TsKvLatestEntity> entry = tsKvLatestRepository.findById(compositeKey);
302   - TsKvEntry result;
303   - if (entry.isPresent()) {
304   - result = DaoUtil.getData(entry.get());
305   - } else {
306   - result = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null));
307   - }
308   - return Futures.immediateFuture(result);
309   - }
310   -
311   - @Override
312   - public ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId) {
313   - return Futures.immediateFuture(
314   - DaoUtil.convertDataList(Lists.newArrayList(
315   - tsKvLatestRepository.findAllByEntityTypeAndEntityId(
316   - entityId.getEntityType(),
317   - UUIDConverter.fromTimeUUID(entityId.getId())))));
318   - }
319   -
320   - @Override
321   - public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) {
322   - TsKvEntity entity = new TsKvEntity();
323   - entity.setEntityType(entityId.getEntityType());
324   - entity.setEntityId(fromTimeUUID(entityId.getId()));
325   - entity.setTs(tsKvEntry.getTs());
326   - entity.setKey(tsKvEntry.getKey());
327   - entity.setStrValue(tsKvEntry.getStrValue().orElse(null));
328   - entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null));
329   - entity.setLongValue(tsKvEntry.getLongValue().orElse(null));
330   - entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null));
331   - log.trace("Saving entity: {}", entity);
332   - return tsQueue.add(entity);
333   - }
334   -
335   - @Override
336   - public ListenableFuture<Void> savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) {
337   - return Futures.immediateFuture(null);
338   - }
339   -
340   - @Override
341   - public ListenableFuture<Void> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
342   - TsKvLatestEntity latestEntity = new TsKvLatestEntity();
343   - latestEntity.setEntityType(entityId.getEntityType());
344   - latestEntity.setEntityId(fromTimeUUID(entityId.getId()));
345   - latestEntity.setTs(tsKvEntry.getTs());
346   - latestEntity.setKey(tsKvEntry.getKey());
347   - latestEntity.setStrValue(tsKvEntry.getStrValue().orElse(null));
348   - latestEntity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null));
349   - latestEntity.setLongValue(tsKvEntry.getLongValue().orElse(null));
350   - latestEntity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null));
351   - return tsLatestQueue.add(latestEntity);
352   - }
353   -
354   - @Override
355   - public ListenableFuture<Void> remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
356   - return service.submit(() -> {
357   - tsKvRepository.delete(
358   - fromTimeUUID(entityId.getId()),
359   - entityId.getEntityType(),
360   - query.getKey(),
361   - query.getStartTs(),
362   - query.getEndTs());
363   - return null;
364   - });
365   - }
366   -
367   - @Override
368   - public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
369   - ListenableFuture<TsKvEntry> latestFuture = findLatest(tenantId, entityId, query.getKey());
370   -
371   - ListenableFuture<Boolean> booleanFuture = Futures.transform(latestFuture, tsKvEntry -> {
372   - long ts = tsKvEntry.getTs();
373   - return ts > query.getStartTs() && ts <= query.getEndTs();
374   - }, service);
375   -
376   - ListenableFuture<Void> removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
377   - if (isRemove) {
378   - TsKvLatestEntity latestEntity = new TsKvLatestEntity();
379   - latestEntity.setEntityType(entityId.getEntityType());
380   - latestEntity.setEntityId(fromTimeUUID(entityId.getId()));
381   - latestEntity.setKey(query.getKey());
382   - return service.submit(() -> {
383   - tsKvLatestRepository.delete(latestEntity);
384   - return null;
385   - });
386   - }
387   - return Futures.immediateFuture(null);
388   - }, service);
389   -
390   - final SimpleListenableFuture<Void> resultFuture = new SimpleListenableFuture<>();
391   - Futures.addCallback(removedLatestFuture, new FutureCallback<Void>() {
392   - @Override
393   - public void onSuccess(@Nullable Void result) {
394   - if (query.getRewriteLatestIfDeleted()) {
395   - ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
396   - if (isRemove) {
397   - return getNewLatestEntryFuture(tenantId, entityId, query);
398   - }
399   - return Futures.immediateFuture(null);
400   - }, service);
401   -
402   - try {
403   - resultFuture.set(savedLatestFuture.get());
404   - } catch (InterruptedException | ExecutionException e) {
405   - log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e);
406   - }
407   - } else {
408   - resultFuture.set(null);
409   - }
410   - }
411   -
412   - @Override
413   - public void onFailure(Throwable t) {
414   - log.warn("[{}] Failed to process remove of the latest value", entityId, t);
415   - }
416   - });
417   - return resultFuture;
418   - }
419   -
420   - private ListenableFuture<Void> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
421   - ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(tenantId, entityId, query);
422   - return Futures.transformAsync(future, entryList -> {
423   - if (entryList.size() == 1) {
424   - return saveLatest(tenantId, entityId, entryList.get(0));
425   - } else {
426   - log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey());
427   - }
428   - return Futures.immediateFuture(null);
429   - }, service);
430   - }
431   -
432   - @Override
433   - public ListenableFuture<Void> removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
434   - return service.submit(() -> null);
435   - }
436   -}
\ No newline at end of file
1   -/**
2   - * Copyright © 2016-2020 The Thingsboard Authors
3   - *
4   - * Licensed under the Apache License, Version 2.0 (the "License");
5   - * you may not use this file except in compliance with the License.
6   - * You may obtain a copy of the License at
7   - *
8   - * http://www.apache.org/licenses/LICENSE-2.0
9   - *
10   - * Unless required by applicable law or agreed to in writing, software
11   - * distributed under the License is distributed on an "AS IS" BASIS,
12   - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   - * See the License for the specific language governing permissions and
14   - * limitations under the License.
15   - */
16   -package org.thingsboard.server.dao.sqlts.ts;
17   -
18   -import org.springframework.jdbc.core.BatchPreparedStatementSetter;
19   -import org.springframework.stereotype.Repository;
20   -import org.springframework.transaction.annotation.Transactional;
21   -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity;
22   -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository;
23   -import org.thingsboard.server.dao.util.PsqlDao;
24   -import org.thingsboard.server.dao.util.SqlTsDao;
25   -
26   -import java.sql.PreparedStatement;
27   -import java.sql.SQLException;
28   -import java.sql.Types;
29   -import java.util.List;
30   -
31   -@SqlTsDao
32   -@PsqlDao
33   -@Repository
34   -@Transactional
35   -public class PsqlTimeseriesInsertRepository extends AbstractTimeseriesInsertRepository<TsKvEntity> {
36   -
37   - private static final String TS_KV_CONSTRAINT = "(entity_type, entity_id, key, ts)";
38   -
39   - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_TABLE, TS_KV_CONSTRAINT, BOOL_V, PSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS);
40   - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_TABLE, TS_KV_CONSTRAINT, STR_V, PSQL_ON_STR_VALUE_UPDATE_SET_NULLS);
41   - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_TABLE, TS_KV_CONSTRAINT, LONG_V, PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS);
42   - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_TABLE, TS_KV_CONSTRAINT, DBL_V, PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS);
43   -
44   - private static final String INSERT_OR_UPDATE =
45   - "INSERT INTO ts_kv (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) VALUES(?, ?, ?, ?, ?, ?, ?, ?) " +
46   - "ON CONFLICT (entity_type, entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?;";
47   -
48   - @Override
49   - public void saveOrUpdate(TsKvEntity entity) {
50   - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT);
51   - }
52   -
53   - @Override
54   - protected void saveOrUpdateBoolean(TsKvEntity entity, String query) {
55   - entityManager.createNativeQuery(query)
56   - .setParameter("entity_type", entity.getEntityType().name())
57   - .setParameter("entity_id", entity.getEntityId())
58   - .setParameter("key", entity.getKey())
59   - .setParameter("ts", entity.getTs())
60   - .setParameter("bool_v", entity.getBooleanValue())
61   - .executeUpdate();
62   - }
63   -
64   - @Override
65   - protected void saveOrUpdateString(TsKvEntity entity, String query) {
66   - entityManager.createNativeQuery(query)
67   - .setParameter("entity_type", entity.getEntityType().name())
68   - .setParameter("entity_id", entity.getEntityId())
69   - .setParameter("key", entity.getKey())
70   - .setParameter("ts", entity.getTs())
71   - .setParameter("str_v", replaceNullChars(entity.getStrValue()))
72   - .executeUpdate();
73   - }
74   -
75   - @Override
76   - protected void saveOrUpdateLong(TsKvEntity entity, String query) {
77   - entityManager.createNativeQuery(query)
78   - .setParameter("entity_type", entity.getEntityType().name())
79   - .setParameter("entity_id", entity.getEntityId())
80   - .setParameter("key", entity.getKey())
81   - .setParameter("ts", entity.getTs())
82   - .setParameter("long_v", entity.getLongValue())
83   - .executeUpdate();
84   - }
85   -
86   - @Override
87   - protected void saveOrUpdateDouble(TsKvEntity entity, String query) {
88   - entityManager.createNativeQuery(query)
89   - .setParameter("entity_type", entity.getEntityType().name())
90   - .setParameter("entity_id", entity.getEntityId())
91   - .setParameter("key", entity.getKey())
92   - .setParameter("ts", entity.getTs())
93   - .setParameter("dbl_v", entity.getDoubleValue())
94   - .executeUpdate();
95   - }
96   -
97   - @Override
98   - public void saveOrUpdate(List<TsKvEntity> entities) {
99   - jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() {
100   - @Override
101   - public void setValues(PreparedStatement ps, int i) throws SQLException {
102   - TsKvEntity tsKvEntity = entities.get(i);
103   - ps.setString(1, tsKvEntity.getEntityType().name());
104   - ps.setString(2, tsKvEntity.getEntityId());
105   - ps.setString(3, tsKvEntity.getKey());
106   - ps.setLong(4, tsKvEntity.getTs());
107   -
108   - if (tsKvEntity.getBooleanValue() != null) {
109   - ps.setBoolean(5, tsKvEntity.getBooleanValue());
110   - ps.setBoolean(9, tsKvEntity.getBooleanValue());
111   - } else {
112   - ps.setNull(5, Types.BOOLEAN);
113   - ps.setNull(9, Types.BOOLEAN);
114   - }
115   -
116   - ps.setString(6, replaceNullChars(tsKvEntity.getStrValue()));
117   - ps.setString(10, replaceNullChars(tsKvEntity.getStrValue()));
118   -
119   -
120   - if (tsKvEntity.getLongValue() != null) {
121   - ps.setLong(7, tsKvEntity.getLongValue());
122   - ps.setLong(11, tsKvEntity.getLongValue());
123   - } else {
124   - ps.setNull(7, Types.BIGINT);
125   - ps.setNull(11, Types.BIGINT);
126   - }
127   -
128   - if (tsKvEntity.getDoubleValue() != null) {
129   - ps.setDouble(8, tsKvEntity.getDoubleValue());
130   - ps.setDouble(12, tsKvEntity.getDoubleValue());
131   - } else {
132   - ps.setNull(8, Types.DOUBLE);
133   - ps.setNull(12, Types.DOUBLE);
134   - }
135   - }
136   -
137   - @Override
138   - public int getBatchSize() {
139   - return entities.size();
140   - }
141   - });
142   - }
143   -}
\ No newline at end of file
... ... @@ -97,7 +97,7 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
97 97 @Value("${cassandra.query.set_null_values_enabled}")
98 98 private boolean setNullValuesEnabled;
99 99
100   - private TsPartitionDate tsFormat;
  100 + private NoSqlTsPartitionDate tsFormat;
101 101
102 102 private PreparedStatement partitionInsertStmt;
103 103 private PreparedStatement partitionInsertTtlStmt;
... ... @@ -120,7 +120,7 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
120 120 super.startExecutor();
121 121 if (!isInstall()) {
122 122 getFetchStmt(Aggregation.NONE, DESC_ORDER);
123   - Optional<TsPartitionDate> partition = TsPartitionDate.parse(partitioning);
  123 + Optional<NoSqlTsPartitionDate> partition = NoSqlTsPartitionDate.parse(partitioning);
124 124 if (partition.isPresent()) {
125 125 tsFormat = partition.get();
126 126 } else {
... ...
dao/src/main/java/org/thingsboard/server/dao/timeseries/NoSqlTsPartitionDate.java renamed from dao/src/main/java/org/thingsboard/server/dao/timeseries/TsPartitionDate.java
... ... @@ -21,7 +21,7 @@ import java.time.temporal.ChronoUnit;
21 21 import java.time.temporal.TemporalUnit;
22 22 import java.util.Optional;
23 23
24   -public enum TsPartitionDate {
  24 +public enum NoSqlTsPartitionDate {
25 25
26 26 MINUTES("yyyy-MM-dd-HH-mm", ChronoUnit.MINUTES), HOURS("yyyy-MM-dd-HH", ChronoUnit.HOURS), DAYS("yyyy-MM-dd", ChronoUnit.DAYS), MONTHS("yyyy-MM", ChronoUnit.MONTHS), YEARS("yyyy", ChronoUnit.YEARS),INDEFINITE("",ChronoUnit.FOREVER);
27 27
... ... @@ -29,7 +29,7 @@ public enum TsPartitionDate {
29 29 private final transient TemporalUnit truncateUnit;
30 30 public final static LocalDateTime EPOCH_START = LocalDateTime.ofEpochSecond(0,0, ZoneOffset.UTC);
31 31
32   - TsPartitionDate(String pattern, TemporalUnit truncateUnit) {
  32 + NoSqlTsPartitionDate(String pattern, TemporalUnit truncateUnit) {
33 33 this.pattern = pattern;
34 34 this.truncateUnit = truncateUnit;
35 35 }
... ... @@ -56,10 +56,10 @@ public enum TsPartitionDate {
56 56 }
57 57 }
58 58
59   - public static Optional<TsPartitionDate> parse(String name) {
60   - TsPartitionDate partition = null;
  59 + public static Optional<NoSqlTsPartitionDate> parse(String name) {
  60 + NoSqlTsPartitionDate partition = null;
61 61 if (name != null) {
62   - for (TsPartitionDate partitionDate : TsPartitionDate.values()) {
  62 + for (NoSqlTsPartitionDate partitionDate : NoSqlTsPartitionDate.values()) {
63 63 if (partitionDate.name().equalsIgnoreCase(name)) {
64 64 partition = partitionDate;
65 65 break;
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.timeseries;
  17 +
  18 +import lombok.Data;
  19 +
  20 +import java.text.SimpleDateFormat;
  21 +import java.util.Date;
  22 +
  23 +@Data
  24 +public class PsqlPartition {
  25 +
  26 + private static final String TABLE_REGEX = "ts_kv_";
  27 +
  28 + private long start;
  29 + private long end;
  30 + private String partitionDate;
  31 + private String query;
  32 +
  33 + public PsqlPartition(long start, long end, String pattern) {
  34 + this.start = start;
  35 + this.end = end;
  36 + this.partitionDate = new SimpleDateFormat(pattern).format(new Date(start));
  37 + this.query = createStatement(start, end, partitionDate);
  38 + }
  39 +
  40 + private String createStatement(long start, long end, String partitionDate) {
  41 + return "CREATE TABLE IF NOT EXISTS " + TABLE_REGEX + partitionDate + " PARTITION OF ts_kv(PRIMARY KEY (entity_id, key, ts)) FOR VALUES FROM (" + start + ") TO (" + end + ")";
  42 + }
  43 +}
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.timeseries;
  17 +
  18 +import java.time.LocalDateTime;
  19 +import java.time.ZoneOffset;
  20 +import java.time.temporal.ChronoUnit;
  21 +import java.time.temporal.TemporalUnit;
  22 +import java.util.Optional;
  23 +
  24 +public enum SqlTsPartitionDate {
  25 +
  26 + MINUTES("yyyy_MM_dd_HH_mm", ChronoUnit.MINUTES), HOURS("yyyy_MM_dd_HH", ChronoUnit.HOURS), DAYS("yyyy_MM_dd", ChronoUnit.DAYS), MONTHS("yyyy_MM", ChronoUnit.MONTHS), YEARS("yyyy", ChronoUnit.YEARS), INDEFINITE("indefinite", ChronoUnit.FOREVER);
  27 +
  28 + private final String pattern;
  29 + private final transient TemporalUnit truncateUnit;
  30 + public final static LocalDateTime EPOCH_START = LocalDateTime.ofEpochSecond(0, 0, ZoneOffset.UTC);
  31 +
  32 + SqlTsPartitionDate(String pattern, TemporalUnit truncateUnit) {
  33 + this.pattern = pattern;
  34 + this.truncateUnit = truncateUnit;
  35 + }
  36 +
  37 + public String getPattern() {
  38 + return pattern;
  39 + }
  40 +
  41 + public TemporalUnit getTruncateUnit() {
  42 + return truncateUnit;
  43 + }
  44 +
  45 + public LocalDateTime trancateTo(LocalDateTime time) {
  46 + switch (this) {
  47 + case MINUTES:
  48 + return time.truncatedTo(ChronoUnit.MINUTES);
  49 + case HOURS:
  50 + return time.truncatedTo(ChronoUnit.HOURS);
  51 + case DAYS:
  52 + return time.truncatedTo(ChronoUnit.DAYS);
  53 + case MONTHS:
  54 + return time.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1);
  55 + case YEARS:
  56 + return time.truncatedTo(ChronoUnit.DAYS).withDayOfYear(1);
  57 + case INDEFINITE:
  58 + return EPOCH_START;
  59 + default:
  60 + throw new RuntimeException("Failed to parse partitioning property!");
  61 + }
  62 + }
  63 +
  64 + public LocalDateTime plusTo(LocalDateTime time) {
  65 + switch (this) {
  66 + case MINUTES:
  67 + return time.plusMinutes(1);
  68 + case HOURS:
  69 + return time.plusHours(1);
  70 + case DAYS:
  71 + return time.plusDays(1);
  72 + case MONTHS:
  73 + return time.plusMonths(1);
  74 + case YEARS:
  75 + return time.plusYears(1);
  76 + default:
  77 + throw new RuntimeException("Failed to parse partitioning property!");
  78 + }
  79 + }
  80 +
  81 + public static Optional<SqlTsPartitionDate> parse(String name) {
  82 + SqlTsPartitionDate partition = null;
  83 + if (name != null) {
  84 + for (SqlTsPartitionDate partitionDate : SqlTsPartitionDate.values()) {
  85 + if (partitionDate.name().equalsIgnoreCase(name)) {
  86 + partition = partitionDate;
  87 + break;
  88 + }
  89 + }
  90 + }
  91 + return Optional.of(partition);
  92 + }
  93 +}
\ No newline at end of file
... ...
... ... @@ -17,7 +17,25 @@
17 17 CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
18 18
19 19 CREATE TABLE IF NOT EXISTS tenant_ts_kv (
20   - tenant_id varchar(31) NOT NULL,
  20 + tenant_id uuid NOT NULL,
  21 + entity_id uuid NOT NULL,
  22 + key int NOT NULL,
  23 + ts bigint NOT NULL,
  24 + bool_v boolean,
  25 + str_v varchar(10000000),
  26 + long_v bigint,
  27 + dbl_v double precision,
  28 + CONSTRAINT ts_kv_pkey PRIMARY KEY (tenant_id, entity_id, key, ts)
  29 +);
  30 +
  31 +CREATE TABLE IF NOT EXISTS ts_kv_dictionary (
  32 + key varchar(255) NOT NULL,
  33 + key_id serial UNIQUE,
  34 + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key)
  35 +);
  36 +
  37 +CREATE TABLE IF NOT EXISTS ts_kv_latest (
  38 + entity_type varchar(255) NOT NULL,
21 39 entity_id varchar(31) NOT NULL,
22 40 key varchar(255) NOT NULL,
23 41 ts bigint NOT NULL,
... ... @@ -25,7 +43,7 @@ CREATE TABLE IF NOT EXISTS tenant_ts_kv (
25 43 str_v varchar(10000000),
26 44 long_v bigint,
27 45 dbl_v double precision,
28   - CONSTRAINT ts_kv_pkey PRIMARY KEY (tenant_id, entity_id, key, ts)
  46 + CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_type, entity_id, key)
29 47 );
30 48
31 49 SELECT create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => 86400000, if_not_exists => true);
\ No newline at end of file
... ...
dao/src/main/resources/sql/schema-ts-hsql.sql renamed from dao/src/main/resources/sql/schema-ts.sql
  1 +--
  2 +-- Copyright © 2016-2020 The Thingsboard Authors
  3 +--
  4 +-- Licensed under the Apache License, Version 2.0 (the "License");
  5 +-- you may not use this file except in compliance with the License.
  6 +-- You may obtain a copy of the License at
  7 +--
  8 +-- http://www.apache.org/licenses/LICENSE-2.0
  9 +--
  10 +-- Unless required by applicable law or agreed to in writing, software
  11 +-- distributed under the License is distributed on an "AS IS" BASIS,
  12 +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +-- See the License for the specific language governing permissions and
  14 +-- limitations under the License.
  15 +--
  16 +
  17 +CREATE TABLE IF NOT EXISTS ts_kv (
  18 + entity_id uuid NOT NULL,
  19 + key int NOT NULL,
  20 + ts bigint NOT NULL,
  21 + bool_v boolean,
  22 + str_v varchar(10000000),
  23 + long_v bigint,
  24 + dbl_v double precision
  25 +) PARTITION BY RANGE (ts);
  26 +
  27 +CREATE TABLE IF NOT EXISTS ts_kv_dictionary (
  28 + key varchar(255) NOT NULL,
  29 + key_id serial UNIQUE,
  30 + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key)
  31 +);
  32 +
  33 +CREATE TABLE IF NOT EXISTS ts_kv_latest (
  34 + entity_type varchar(255) NOT NULL,
  35 + entity_id varchar(31) NOT NULL,
  36 + key varchar(255) NOT NULL,
  37 + ts bigint NOT NULL,
  38 + bool_v boolean,
  39 + str_v varchar(10000000),
  40 + long_v bigint,
  41 + dbl_v double precision,
  42 + CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_type, entity_id, key)
  43 +);
\ No newline at end of file
... ...
... ... @@ -30,7 +30,7 @@ import org.springframework.test.context.support.DirtiesContextTestExecutionListe
30 30 * Created by Valerii Sosliuk on 4/22/2017.
31 31 */
32 32 @RunWith(SpringRunner.class)
33   -@ContextConfiguration(classes = {JpaDaoConfig.class, SqlTsDaoConfig.class, JpaDbunitTestConfig.class})
  33 +@ContextConfiguration(classes = {JpaDaoConfig.class, HsqlTsDaoConfig.class, JpaDbunitTestConfig.class})
34 34 @TestPropertySource("classpath:sql-test.properties")
35 35 @TestExecutionListeners({
36 36 DependencyInjectionTestExecutionListener.class,
... ...
... ... @@ -30,9 +30,23 @@ public class JpaDaoTestSuite {
30 30
31 31 @ClassRule
32 32 public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
33   - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
  33 + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
34 34 "sql/drop-all-tables.sql",
35 35 "sql-test.properties"
36 36 );
37 37
  38 +// @ClassRule
  39 +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
  40 +// Arrays.asList("sql/schema-ts-psql.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
  41 +// "sql/drop-all-tables.sql",
  42 +// "sql-test.properties"
  43 +// );
  44 +
  45 +// @ClassRule
  46 +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
  47 +// Arrays.asList("sql/schema-timescale.sql", "sql/schema-timescale-idx.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
  48 +// "sql/timescale/drop-all-tables.sql",
  49 +// "sql-test.properties"
  50 +// );
  51 +
38 52 }
... ...
... ... @@ -30,9 +30,23 @@ public class SqlDaoServiceTestSuite {
30 30
31 31 @ClassRule
32 32 public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
33   - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"),
  33 + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"),
34 34 "sql/drop-all-tables.sql",
35 35 "sql-test.properties"
36 36 );
37 37
  38 +// @ClassRule
  39 +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
  40 +// Arrays.asList("sql/schema-ts-psql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"),
  41 +// "sql/drop-all-tables.sql",
  42 +// "sql-test.properties"
  43 +// );
  44 +
  45 +// @ClassRule
  46 +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
  47 +// Arrays.asList("sql/schema-timescale.sql", "sql/schema-timescale-idx.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"),
  48 +// "sql/timescale/drop-all-tables.sql",
  49 +// "sql-test.properties"
  50 +// );
  51 +
38 52 }
... ...
... ... @@ -2,7 +2,8 @@ database.ts.type=sql
2 2 database.entities.type=sql
3 3
4 4 sql.ts_inserts_executor_type=fixed
5   -sql.ts_inserts_fixed_thread_pool_size=10
  5 +sql.ts_inserts_fixed_thread_pool_size=200
  6 +sql.ts_key_value_partitioning=MONTHS
6 7
7 8 spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true
8 9 spring.jpa.show-sql=false
... ... @@ -13,4 +14,23 @@ spring.datasource.username=sa
13 14 spring.datasource.password=
14 15 spring.datasource.url=jdbc:hsqldb:file:/tmp/testDb;sql.enforce_size=false
15 16 spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver
16   -spring.datasource.hikari.maximumPoolSize = 50
\ No newline at end of file
  17 +spring.datasource.hikari.maximumPoolSize = 50
  18 +
  19 +#database.ts.type=timescale
  20 +#database.ts.type=sql
  21 +#database.entities.type=sql
  22 +#
  23 +#sql.ts_inserts_executor_type=fixed
  24 +#sql.ts_inserts_fixed_thread_pool_size=200
  25 +#sql.ts_key_value_partitioning=MONTHS
  26 +#
  27 +#spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true
  28 +#spring.jpa.show-sql=false
  29 +#spring.jpa.hibernate.ddl-auto=none
  30 +#spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect
  31 +#
  32 +#spring.datasource.username=postgres
  33 +#spring.datasource.password=postgres
  34 +#spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest
  35 +#spring.datasource.driverClassName=org.postgresql.Driver
  36 +#spring.datasource.hikari.maximumPoolSize = 50
\ No newline at end of file
... ...
... ... @@ -13,6 +13,7 @@ DROP TABLE IF EXISTS relation;
13 13 DROP TABLE IF EXISTS tb_user;
14 14 DROP TABLE IF EXISTS tenant;
15 15 DROP TABLE IF EXISTS tenant_ts_kv;
  16 +DROP TABLE IF EXISTS ts_kv_latest;
16 17 DROP TABLE IF EXISTS user_credentials;
17 18 DROP TABLE IF EXISTS widget_type;
18 19 DROP TABLE IF EXISTS widgets_bundle;
... ...
... ... @@ -19,7 +19,7 @@ version: '2.2'
19 19 services:
20 20 postgres:
21 21 restart: always
22   - image: "postgres:9.6"
  22 + image: "postgres:10"
23 23 ports:
24 24 - "5432"
25 25 environment:
... ...
... ... @@ -51,7 +51,7 @@ spec:
51 51 containers:
52 52 - name: postgres
53 53 imagePullPolicy: Always
54   - image: postgres:9.6
  54 + image: postgres:10
55 55 ports:
56 56 - containerPort: 5432
57 57 name: postgres
... ...
... ... @@ -20,10 +20,10 @@ firstlaunch=${DATA_FOLDER}/.firstlaunch
20 20 if [ ! -d ${PGDATA} ]; then
21 21 mkdir -p ${PGDATA}
22 22 chown -R postgres:postgres ${PGDATA}
23   - su postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl initdb -U postgres'
  23 + su postgres -c '/usr/lib/postgresql/10/bin/pg_ctl initdb -U postgres'
24 24 fi
25 25
26   -su postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl -l /var/log/postgres/postgres.log -w start'
  26 +su postgres -c '/usr/lib/postgresql/10/bin/pg_ctl -l /var/log/postgres/postgres.log -w start'
27 27
28 28 if [ ! -f ${firstlaunch} ]; then
29 29 su postgres -c 'psql -U postgres -d postgres -c "CREATE DATABASE thingsboard"'
... ...
... ... @@ -15,4 +15,4 @@
15 15 # limitations under the License.
16 16 #
17 17
18   -su postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl stop'
  18 +su postgres -c '/usr/lib/postgresql/10/bin/pg_ctl stop'
... ...