简柏林 / thingskit · Commits

Commit 3b17d14f1504e447f5cae6066ceb2b786c2579fc

Authored by Viacheslav Klimov
2 parents 35058102 2696c1b8

Merge branch 'feature/bulk-import/device-credentials' of https://github.com/vvll… 

…add28/thingsboard into feature/bulk-import/device-credentials
Inline Side-by-side
Showing 7 changed files with 101 additions and 205 deletions
... ... @@ -22,6 +22,7 @@ import { PageLink } from '@shared/models/page/page-link';
22 22 import { PageData } from '@shared/models/page/page-data';
23 23 import { EntitySubtype } from '@app/shared/models/entity-type.models';
24 24 import { Asset, AssetInfo, AssetSearchQuery } from '@app/shared/models/asset.models';
  25 +import { BulkImportRequest, BulkImportResult } from '@home/components/import-export/import-export.models';
25 26
26 27 @Injectable({
27 28 providedIn: 'root'
... ... @@ -105,4 +106,8 @@ export class AssetService {
105 106 defaultHttpOptionsFromConfig(config));
106 107 }
107 108
  109 + public bulkImportAssets(entitiesData: BulkImportRequest, config?: RequestConfig): Observable<BulkImportResult> {
  110 + return this.http.post<BulkImportResult>('/api/asset/bulk_import', entitiesData, defaultHttpOptionsFromConfig(config));
  111 + }
  112 +
108 113 }
... ...
... ... @@ -30,6 +30,7 @@ import {
30 30 } from '@app/shared/models/device.models';
31 31 import { EntitySubtype } from '@app/shared/models/entity-type.models';
32 32 import { AuthService } from '@core/auth/auth.service';
  33 +import { BulkImportRequest, BulkImportResult } from '@home/components/import-export/import-export.models';
33 34
34 35 @Injectable({
35 36 providedIn: 'root'
... ... @@ -170,7 +171,11 @@ export class DeviceService {
170 171 public getEdgeDevices(edgeId: string, pageLink: PageLink, type: string = '',
171 172 config?: RequestConfig): Observable<PageData<DeviceInfo>> {
172 173 return this.http.get<PageData<DeviceInfo>>(`/api/edge/${edgeId}/devices${pageLink.toQuery()}&type=${type}`,
173   - defaultHttpOptionsFromConfig(config))
  174 + defaultHttpOptionsFromConfig(config));
  175 + }
  176 +
  177 + public bulkImportDevices(entitiesData: BulkImportRequest, config?: RequestConfig): Observable<BulkImportResult> {
  178 + return this.http.post<BulkImportResult>('/api/device/bulk_import', entitiesData, defaultHttpOptionsFromConfig(config));
174 179 }
175 180
176 181 }
... ...
... ... @@ -23,6 +23,7 @@ import { PageData } from '@shared/models/page/page-data';
23 23 import { EntitySubtype } from '@app/shared/models/entity-type.models';
24 24 import { Edge, EdgeEvent, EdgeInfo, EdgeSearchQuery } from '@shared/models/edge.models';
25 25 import { EntityId } from '@shared/models/id/entity-id';
  26 +import { BulkImportRequest, BulkImportResult } from '@home/components/import-export/import-export.models';
26 27
27 28 @Injectable({
28 29 providedIn: 'root'
... ... @@ -59,7 +60,7 @@ export class EdgeService {
59 60 }
60 61
61 62 public getCustomerEdgeInfos(customerId: string, pageLink: PageLink, type: string = '',
62   - config?: RequestConfig): Observable<PageData<EdgeInfo>> {
  63 + config?: RequestConfig): Observable<PageData<EdgeInfo>> {
63 64 return this.http.get<PageData<EdgeInfo>>(`/api/customer/${customerId}/edgeInfos${pageLink.toQuery()}&type=${type}`,
64 65 defaultHttpOptionsFromConfig(config));
65 66 }
... ... @@ -108,4 +109,8 @@ export class EdgeService {
108 109 public findByName(edgeName: string, config?: RequestConfig): Observable<Edge> {
109 110 return this.http.get<Edge>(`/api/tenant/edges?edgeName=${edgeName}`, defaultHttpOptionsFromConfig(config));
110 111 }
  112 +
  113 + public bulkImportEdges(entitiesData: BulkImportRequest, config?: RequestConfig): Observable<BulkImportResult> {
  114 + return this.http.post<BulkImportResult>('/api/edge/bulk_import', entitiesData, defaultHttpOptionsFromConfig(config));
  115 + }
111 116 }
... ...
... ... @@ -113,23 +113,23 @@
113 113 </mat-step>
114 114 <mat-step>
115 115 <ng-template matStepLabel>{{ 'import.stepper-text.creat-entities' | translate }}</ng-template>
116   - <mat-progress-bar color="warn" class="tb-import-progress" mode="determinate" [value]="progressCreate">
  116 + <mat-progress-bar color="warn" class="tb-import-progress" mode="indeterminate">
117 117 </mat-progress-bar>
118 118 </mat-step>
119 119 <mat-step>
120 120 <ng-template matStepLabel>{{ 'import.stepper-text.done' | translate }}</ng-template>
121 121 <div fxLayout="column">
122   - <p class="mat-body-1" *ngIf="this.statistical?.create && this.statistical?.create.entity">
123   - {{ translate.instant('import.message.create-entities', {count: this.statistical.create.entity}) }}
  122 + <p class="mat-body-1" *ngIf="this.statistical?.created">
  123 + {{ translate.instant('import.message.create-entities', {count: this.statistical.created}) }}
124 124 </p>
125   - <p class="mat-body-1" *ngIf="this.statistical?.update && this.statistical?.update.entity">
126   - {{ translate.instant('import.message.update-entities', {count: this.statistical.update.entity}) }}
  125 + <p class="mat-body-1" *ngIf="this.statistical?.updated">
  126 + {{ translate.instant('import.message.update-entities', {count: this.statistical.updated}) }}
127 127 </p>
128   - <p class="mat-body-1" style="margin-bottom: 0.8em" *ngIf="this.statistical?.error && this.statistical?.error.entity">
129   - {{ translate.instant('import.message.error-entities', {count: this.statistical.error.entity}) }}
  128 + <p class="mat-body-1" style="margin-bottom: 0.8em" *ngIf="this.statistical?.errors">
  129 + {{ translate.instant('import.message.error-entities', {count: this.statistical.errors}) }}
130 130 </p>
131 131 <mat-expansion-panel class="advanced-logs" [expanded]="false"
132   - *ngIf="this.statistical?.error && this.statistical?.error.entity"
  132 + *ngIf="this.statistical?.errorsList?.length"
133 133 (opened)="initEditor()">
134 134 <mat-expansion-panel-header [collapsedHeight]="'38px'" [expandedHeight]="'38px'">
135 135 <mat-panel-title>
... ...
... ... @@ -26,18 +26,18 @@ import { TranslateService } from '@ngx-translate/core';
26 26 import { ActionNotificationShow } from '@core/notification/notification.actions';
27 27 import { MatVerticalStepper } from '@angular/material/stepper';
28 28 import {
  29 + BulkImportRequest,
  30 + BulkImportResult,
  31 + ColumnMapping,
29 32 convertCSVToJson,
30 33 CsvColumnParam,
  34 + CSVDelimiter,
31 35 CsvToJsonConfig,
32 36 CsvToJsonResult,
33 37 ImportEntityColumnType
34 38 } from '@home/components/import-export/import-export.models';
35   -import { EdgeImportEntityData, ImportEntitiesResultInfo, ImportEntityData } from '@app/shared/models/entity.models';
36 39 import { ImportExportService } from '@home/components/import-export/import-export.service';
37 40 import { TableColumnsAssignmentComponent } from '@home/components/import-export/table-columns-assignment.component';
38   -import { getDeviceCredentialMQTTDefault } from '@shared/models/device.models';
39   -import { isDefinedAndNotNull } from '@core/utils';
40   -import { getLwm2mSecurityConfigModelsDefault } from '@shared/models/lwm2m-security-config.models';
41 41 import { Ace } from 'ace-builds';
42 42 import { getAce } from '@shared/models/ace/ace.models';
43 43
... ... @@ -68,7 +68,7 @@ export class ImportDialogCsvComponent extends DialogComponent<ImportDialogCsvCom
68 68 importTitle: string;
69 69 importFileLabel: string;
70 70
71   - delimiters: { key: string, value: string }[] = [{
  71 + delimiters: { key: CSVDelimiter, value: string }[] = [{
72 72 key: ',',
73 73 value: ','
74 74 }, {
... ... @@ -89,8 +89,7 @@ export class ImportDialogCsvComponent extends DialogComponent<ImportDialogCsvCom
89 89 columnTypesFormGroup: FormGroup;
90 90
91 91 isImportData = false;
92   - progressCreate = 0;
93   - statistical: ImportEntitiesResultInfo;
  92 + statistical: BulkImportResult;
94 93
95 94 private allowAssignColumn: ImportEntityColumnType[];
96 95 private initEditorComponent = false;
... ... @@ -215,169 +214,16 @@ export class ImportDialogCsvComponent extends DialogComponent<ImportDialogCsvCom
215 214
216 215
217 216 private addEntities() {
218   - const importData = this.parseData;
219   - const isHeader: boolean = this.importParametersFormGroup.get('isHeader').value;
220   - const parameterColumns: CsvColumnParam[] = this.columnTypesFormGroup.get('columnsParam').value;
221   - const entitiesData: ImportEntityData[] = [];
222   - let sentDataLength = 0;
223   - const startLineNumber = isHeader ? 2 : 1;
224   - for (let row = 0; row < importData.rows.length; row++) {
225   - const entityData: ImportEntityData = this.constructDraftImportEntityData();
226   - const i = row;
227   - entityData.lineNumber = startLineNumber + i;
228   - for (let j = 0; j < parameterColumns.length; j++) {
229   - if (!isDefinedAndNotNull(importData.rows[i][j]) || importData.rows[i][j] === '') {
230   - continue;
231   - }
232   - switch (parameterColumns[j].type) {
233   - case ImportEntityColumnType.serverAttribute:
234   - entityData.attributes.server.push({
235   - key: parameterColumns[j].key,
236   - value: importData.rows[i][j]
237   - });
238   - break;
239   - case ImportEntityColumnType.timeseries:
240   - entityData.timeseries.push({
241   - key: parameterColumns[j].key,
242   - value: importData.rows[i][j]
243   - });
244   - break;
245   - case ImportEntityColumnType.sharedAttribute:
246   - entityData.attributes.shared.push({
247   - key: parameterColumns[j].key,
248   - value: importData.rows[i][j]
249   - });
250   - break;
251   - case ImportEntityColumnType.name:
252   - entityData.name = importData.rows[i][j];
253   - break;
254   - case ImportEntityColumnType.type:
255   - entityData.type = importData.rows[i][j];
256   - break;
257   - case ImportEntityColumnType.label:
258   - entityData.label = importData.rows[i][j];
259   - break;
260   - case ImportEntityColumnType.isGateway:
261   - entityData.gateway = importData.rows[i][j];
262   - break;
263   - case ImportEntityColumnType.description:
264   - entityData.description = importData.rows[i][j];
265   - break;
266   - case ImportEntityColumnType.accessToken:
267   - entityData.credential.accessToken = importData.rows[i][j];
268   - break;
269   - case ImportEntityColumnType.x509:
270   - entityData.credential.x509 = importData.rows[i][j];
271   - break;
272   - case ImportEntityColumnType.mqttClientId:
273   - if (!entityData.credential.mqtt) {
274   - entityData.credential.mqtt = getDeviceCredentialMQTTDefault();
275   - }
276   - entityData.credential.mqtt.clientId = importData.rows[i][j];
277   - break;
278   - case ImportEntityColumnType.mqttUserName:
279   - if (!entityData.credential.mqtt) {
280   - entityData.credential.mqtt = getDeviceCredentialMQTTDefault();
281   - }
282   - entityData.credential.mqtt.userName = importData.rows[i][j];
283   - break;
284   - case ImportEntityColumnType.mqttPassword:
285   - if (!entityData.credential.mqtt) {
286   - entityData.credential.mqtt = getDeviceCredentialMQTTDefault();
287   - }
288   - entityData.credential.mqtt.password = importData.rows[i][j];
289   - break;
290   - case ImportEntityColumnType.lwm2mClientEndpoint:
291   - if (!entityData.credential.lwm2m) {
292   - entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
293   - }
294   - entityData.credential.lwm2m.client.endpoint = importData.rows[i][j];
295   - break;
296   - case ImportEntityColumnType.lwm2mClientSecurityConfigMode:
297   - if (!entityData.credential.lwm2m) {
298   - entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
299   - }
300   - entityData.credential.lwm2m.client.securityConfigClientMode = importData.rows[i][j];
301   - break;
302   - case ImportEntityColumnType.lwm2mClientIdentity:
303   - if (!entityData.credential.lwm2m) {
304   - entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
305   - }
306   - entityData.credential.lwm2m.client.identity = importData.rows[i][j];
307   - break;
308   - case ImportEntityColumnType.lwm2mClientKey:
309   - if (!entityData.credential.lwm2m) {
310   - entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
311   - }
312   - entityData.credential.lwm2m.client.key = importData.rows[i][j];
313   - break;
314   - case ImportEntityColumnType.lwm2mClientCert:
315   - if (!entityData.credential.lwm2m) {
316   - entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
317   - }
318   - entityData.credential.lwm2m.client.cert = importData.rows[i][j];
319   - break;
320   - case ImportEntityColumnType.lwm2mBootstrapServerSecurityMode:
321   - if (!entityData.credential.lwm2m) {
322   - entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
323   - }
324   - entityData.credential.lwm2m.bootstrap.bootstrapServer.securityMode = importData.rows[i][j];
325   - break;
326   - case ImportEntityColumnType.lwm2mBootstrapServerClientPublicKeyOrId:
327   - if (!entityData.credential.lwm2m) {
328   - entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
329   - }
330   - entityData.credential.lwm2m.bootstrap.bootstrapServer.clientPublicKeyOrId = importData.rows[i][j];
331   - break;
332   - case ImportEntityColumnType.lwm2mBootstrapServerClientSecretKey:
333   - if (!entityData.credential.lwm2m) {
334   - entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
335   - }
336   - entityData.credential.lwm2m.bootstrap.bootstrapServer.clientSecretKey = importData.rows[i][j];
337   - break;
338   - case ImportEntityColumnType.lwm2mServerSecurityMode:
339   - if (!entityData.credential.lwm2m) {
340   - entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
341   - }
342   - entityData.credential.lwm2m.bootstrap.lwm2mServer.securityMode = importData.rows[i][j];
343   - break;
344   - case ImportEntityColumnType.lwm2mServerClientPublicKeyOrId:
345   - if (!entityData.credential.lwm2m) {
346   - entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
347   - }
348   - entityData.credential.lwm2m.bootstrap.lwm2mServer.clientPublicKeyOrId = importData.rows[i][j];
349   - break;
350   - case ImportEntityColumnType.lwm2mServerClientSecretKey:
351   - if (!entityData.credential.lwm2m) {
352   - entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
353   - }
354   - entityData.credential.lwm2m.bootstrap.lwm2mServer.clientSecretKey = importData.rows[i][j];
355   - break;
356   - case ImportEntityColumnType.edgeLicenseKey:
357   - (entityData as EdgeImportEntityData).edgeLicenseKey = importData.rows[i][j];
358   - break;
359   - case ImportEntityColumnType.cloudEndpoint:
360   - (entityData as EdgeImportEntityData).cloudEndpoint = importData.rows[i][j];
361   - break;
362   - case ImportEntityColumnType.routingKey:
363   - (entityData as EdgeImportEntityData).routingKey = importData.rows[i][j];
364   - break;
365   - case ImportEntityColumnType.secret:
366   - (entityData as EdgeImportEntityData).secret = importData.rows[i][j];
367   - break;
368   - }
  217 + const entitiesData: BulkImportRequest = {
  218 + file: this.selectFileFormGroup.get('importData').value,
  219 + mapping: {
  220 + columns: this.processingColumnsParams(),
  221 + delimiter: this.importParametersFormGroup.get('delim').value,
  222 + header: this.importParametersFormGroup.get('isHeader').value,
  223 + update: this.importParametersFormGroup.get('isUpdate').value
369 224 }
370   - entitiesData.push(entityData);
371   - }
372   - const createImportEntityCompleted = () => {
373   - sentDataLength++;
374   - this.progressCreate = Math.round((sentDataLength / importData.rows.length) * 100);
375 225 };
376   -
377   - const isUpdate: boolean = this.importParametersFormGroup.get('isUpdate').value;
378   -
379   - this.importExport.importEntities(entitiesData, this.entityType, isUpdate,
380   - createImportEntityCompleted, {ignoreErrors: true, resendRequest: true}).subscribe(
  226 + this.importExport.bulkImportEntities(entitiesData, this.entityType, {ignoreErrors: true}).subscribe(
381 227 (result) => {
382 228 this.statistical = result;
383 229 this.isImportData = false;
... ... @@ -386,36 +232,22 @@ export class ImportDialogCsvComponent extends DialogComponent<ImportDialogCsvCom
386 232 );
387 233 }
388 234
389   - private constructDraftImportEntityData(): ImportEntityData {
390   - const entityData: ImportEntityData = {
391   - lineNumber: 1,
392   - name: '',
393   - type: '',
394   - description: '',
395   - gateway: null,
396   - label: '',
397   - attributes: {
398   - server: [],
399   - shared: []
400   - },
401   - credential: {},
402   - timeseries: []
403   - };
404   - if (this.entityType === EntityType.EDGE) {
405   - const edgeEntityData: EdgeImportEntityData = entityData as EdgeImportEntityData;
406   - edgeEntityData.edgeLicenseKey = '';
407   - edgeEntityData.cloudEndpoint = '';
408   - edgeEntityData.routingKey = '';
409   - edgeEntityData.secret = '';
410   - return edgeEntityData;
411   - } else {
412   - return entityData;
413   - }
  235 + private processingColumnsParams(): Array<ColumnMapping> {
  236 + const parameterColumns: CsvColumnParam[] = this.columnTypesFormGroup.get('columnsParam').value;
  237 + const allowKeyForTypeColumns: ImportEntityColumnType[] = [
  238 + ImportEntityColumnType.serverAttribute,
  239 + ImportEntityColumnType.timeseries,
  240 + ImportEntityColumnType.sharedAttribute
  241 + ];
  242 + return parameterColumns.map(column => ({
  243 + type: column.type,
  244 + key: allowKeyForTypeColumns.some(type => type === column.type) ? column.key : undefined
  245 + }));
414 246 }
415 247
416 248 initEditor() {
417 249 if (!this.initEditorComponent) {
418   - this.createEditor(this.failureDetailsEditorElmRef, this.statistical.error.errors);
  250 + this.createEditor(this.failureDetailsEditorElmRef, this.statistical.errorsList.join('\n'));
419 251 }
420 252 }
421 253
... ...
... ... @@ -38,6 +38,8 @@ export interface CsvToJsonResult {
38 38 rows?: any[][];
39 39 }
40 40
  41 +export type CSVDelimiter = ',' | ';' | '|' | '\t';
  42 +
41 43 export enum ImportEntityColumnType {
42 44 name = 'NAME',
43 45 type = 'TYPE',
... ... @@ -113,6 +115,28 @@ export interface CsvColumnParam {
113 115 sampleData: any;
114 116 }
115 117
  118 +export interface ColumnMapping {
  119 + type: ImportEntityColumnType;
  120 + key?: string;
  121 +}
  122 +
  123 +export interface BulkImportRequest {
  124 + file: string;
  125 + mapping: {
  126 + columns: Array<ColumnMapping>;
  127 + delimiter: CSVDelimiter;
  128 + header: boolean;
  129 + update: boolean;
  130 + };
  131 +}
  132 +
  133 +export interface BulkImportResult {
  134 + created: number;
  135 + updated: number;
  136 + errors: number;
  137 + errorsList: Array<string>;
  138 +}
  139 +
116 140 export interface FileType {
117 141 mimeType: string;
118 142 extension: string;
... ...
... ... @@ -44,7 +44,15 @@ import {
44 44 EntityAliasesDialogData
45 45 } from '@home/components/alias/entity-aliases-dialog.component';
46 46 import { ItemBufferService, WidgetItem } from '@core/services/item-buffer.service';
47   -import { FileType, ImportWidgetResult, JSON_TYPE, WidgetsBundleItem, ZIP_TYPE } from './import-export.models';
  47 +import {
  48 + BulkImportRequest,
  49 + BulkImportResult,
  50 + FileType,
  51 + ImportWidgetResult,
  52 + JSON_TYPE,
  53 + WidgetsBundleItem,
  54 + ZIP_TYPE
  55 +} from './import-export.models';
48 56 import { AliasEntityType, EntityType } from '@shared/models/entity-type.models';
49 57 import { UtilsService } from '@core/services/utils.service';
50 58 import { WidgetService } from '@core/http/widget.service';
... ... @@ -59,6 +67,9 @@ import { DeviceProfileService } from '@core/http/device-profile.service';
59 67 import { DeviceProfile } from '@shared/models/device.models';
60 68 import { TenantProfile } from '@shared/models/tenant.model';
61 69 import { TenantProfileService } from '@core/http/tenant-profile.service';
  70 +import { DeviceService } from '@core/http/device.service';
  71 +import { AssetService } from '@core/http/asset.service';
  72 +import { EdgeService } from '@core/http/edge.service';
62 73
63 74 // @dynamic
64 75 @Injectable()
... ... @@ -75,6 +86,9 @@ export class ImportExportService {
75 86 private tenantProfileService: TenantProfileService,
76 87 private entityService: EntityService,
77 88 private ruleChainService: RuleChainService,
  89 + private deviceService: DeviceService,
  90 + private assetService: AssetService,
  91 + private edgeService: EdgeService,
78 92 private utils: UtilsService,
79 93 private itembuffer: ItemBufferService,
80 94 private dialog: MatDialog) {
... ... @@ -342,6 +356,17 @@ export class ImportExportService {
342 356 );
343 357 }
344 358
  359 + public bulkImportEntities(entitiesData: BulkImportRequest, entityType: EntityType, config?: RequestConfig): Observable<BulkImportResult> {
  360 + switch (entityType) {
  361 + case EntityType.DEVICE:
  362 + return this.deviceService.bulkImportDevices(entitiesData, config);
  363 + case EntityType.ASSET:
  364 + return this.assetService.bulkImportAssets(entitiesData, config);
  365 + case EntityType.EDGE:
  366 + return this.edgeService.bulkImportEdges(entitiesData, config);
  367 + }
  368 + }
  369 +
345 370 public importEntities(entitiesData: ImportEntityData[], entityType: EntityType, updateData: boolean,
346 371 importEntityCompleted?: () => void, config?: RequestConfig): Observable<ImportEntitiesResultInfo> {
347 372 let partSize = 100;
... ...