From 3f4f3bd73362dc8b7b897f011c361d1d7f96ef4e Mon Sep 17 00:00:00 2001 From: floxx2112 Date: Tue, 11 Jul 2023 16:04:36 +0200 Subject: [PATCH] add efficiency to fill database (approx 19 min) --- .../controller/ResetDataController.java | 2 +- .../assist/db/backend/model/InfoColumn.java | 7 + .../assist/db/backend/model/InfoTable.java | 6 + .../assist/db/backend/model/LinkInfo.java | 6 + .../assist/db/backend/model/json/Column.java | 11 ++ .../db/backend/model/json/DataEnum.java | 13 ++ .../assist/db/backend/model/json/Schema.java | 11 ++ .../repository/InfoColumnRepository.java | 2 +- .../db/backend/service/InfoColumnService.java | 8 + .../db/backend/service/InfoTableService.java | 9 + .../db/backend/service/LinkInfoService.java | 8 + .../db/backend/service/ResetDataService.java | 7 +- .../implementation/InfoColumnServiceImpl.java | 6 + .../implementation/InfoTableServiceImpl.java | 7 + .../implementation/LinkInfoServiceImpl.java | 6 + .../implementation/ResetDataServiceImpl.java | 162 +++++------------- src/main/resources/application.properties | 6 +- 17 files changed, 149 insertions(+), 128 deletions(-) create mode 100644 src/main/java/com/apside/assist/db/backend/model/json/Column.java create mode 100644 src/main/java/com/apside/assist/db/backend/model/json/DataEnum.java create mode 100644 src/main/java/com/apside/assist/db/backend/model/json/Schema.java diff --git a/src/main/java/com/apside/assist/db/backend/controller/ResetDataController.java b/src/main/java/com/apside/assist/db/backend/controller/ResetDataController.java index c5341e9..105bf46 100644 --- a/src/main/java/com/apside/assist/db/backend/controller/ResetDataController.java +++ b/src/main/java/com/apside/assist/db/backend/controller/ResetDataController.java @@ -2,6 +2,7 @@ package com.apside.assist.db.backend.controller; import java.io.IOException; import com.apside.assist.db.backend.service.ResetDataService; +import com.fasterxml.jackson.core.JsonProcessingException; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; @@ -15,7 +16,6 @@ public class ResetDataController { @Autowired private ResetDataService resetDataService; - @PostMapping("/reset") public ResponseEntity resetData() { log.debug("ResetData called"); diff --git a/src/main/java/com/apside/assist/db/backend/model/InfoColumn.java b/src/main/java/com/apside/assist/db/backend/model/InfoColumn.java index 4aceb3f..1d5ecc8 100644 --- a/src/main/java/com/apside/assist/db/backend/model/InfoColumn.java +++ b/src/main/java/com/apside/assist/db/backend/model/InfoColumn.java @@ -34,4 +34,11 @@ public class InfoColumn { this.lengthColumn = lengthColumn; this.columnText = columnText; } + + public InfoColumn(String nameColumn, String dataType, int lengthColumn, String columnText){ + this.nameColumn = nameColumn; + this.dataType = dataType; + this.lengthColumn = lengthColumn; + this.columnText = columnText; + } } diff --git a/src/main/java/com/apside/assist/db/backend/model/InfoTable.java b/src/main/java/com/apside/assist/db/backend/model/InfoTable.java index bbfce58..2e8b6f0 100644 --- a/src/main/java/com/apside/assist/db/backend/model/InfoTable.java +++ b/src/main/java/com/apside/assist/db/backend/model/InfoTable.java @@ -30,4 +30,10 @@ public class InfoTable { this.nameSchema = nameSchema; this.tableText = tableText; } + + public InfoTable(String table, String schema, String tableText) { + this.nameTable = table; + this.nameSchema = schema; + this.tableText = tableText; + } } diff --git a/src/main/java/com/apside/assist/db/backend/model/LinkInfo.java b/src/main/java/com/apside/assist/db/backend/model/LinkInfo.java index 5dfdcdc..950698f 100644 --- a/src/main/java/com/apside/assist/db/backend/model/LinkInfo.java +++ b/src/main/java/com/apside/assist/db/backend/model/LinkInfo.java @@ -28,4 +28,10 @@ public class LinkInfo { this.nameSchema = nameSchema; this.columnId = columnId; } + + public LinkInfo(String nameTable, String nameSchema, int columnId){ + this.nameTable = nameTable; + this.nameSchema = nameSchema; + this.columnId = columnId; + } } diff --git a/src/main/java/com/apside/assist/db/backend/model/json/Column.java b/src/main/java/com/apside/assist/db/backend/model/json/Column.java new file mode 100644 index 0000000..b20ecc9 --- /dev/null +++ b/src/main/java/com/apside/assist/db/backend/model/json/Column.java @@ -0,0 +1,11 @@ +package com.apside.assist.db.backend.model.json; + +import lombok.Data; + +@Data +public class Column { + private String name; + private String data_type; + private int length; + private String column_text; +} diff --git a/src/main/java/com/apside/assist/db/backend/model/json/DataEnum.java b/src/main/java/com/apside/assist/db/backend/model/json/DataEnum.java new file mode 100644 index 0000000..4a6c30e --- /dev/null +++ b/src/main/java/com/apside/assist/db/backend/model/json/DataEnum.java @@ -0,0 +1,13 @@ +package com.apside.assist.db.backend.model.json; + + +import lombok.Data; + +import java.util.List; + +@Data +public class DataEnum { + private List columns; + private String table_text; + private String table; +} diff --git a/src/main/java/com/apside/assist/db/backend/model/json/Schema.java b/src/main/java/com/apside/assist/db/backend/model/json/Schema.java new file mode 100644 index 0000000..fc0a820 --- /dev/null +++ b/src/main/java/com/apside/assist/db/backend/model/json/Schema.java @@ -0,0 +1,11 @@ +package com.apside.assist.db.backend.model.json; + +import lombok.Data; + +import java.util.List; + +@Data +public class Schema { + private String schema; + private List data; +} diff --git a/src/main/java/com/apside/assist/db/backend/repository/InfoColumnRepository.java b/src/main/java/com/apside/assist/db/backend/repository/InfoColumnRepository.java index eee5880..41d2109 100644 --- a/src/main/java/com/apside/assist/db/backend/repository/InfoColumnRepository.java +++ b/src/main/java/com/apside/assist/db/backend/repository/InfoColumnRepository.java @@ -25,7 +25,7 @@ public interface InfoColumnRepository extends CrudRepository { "(SELECT column_id FROM link_informations WHERE name_table IN :tables AND name_schema IN :schemas GROUP BY column_id HAVING COUNT(column_id) > 1)", nativeQuery = true) Iterable getColumnsForJoinTwo(@RequestParam("tables") List tables, @RequestParam("schemas") List schemas); - @Query(value = "SELECT * FROM informations_column WHERE name_column = :namecol AND data_type = :datatype AND length_column = :lengthcol AND column_text = :columntext", nativeQuery = true) + @Query(value = "SELECT * FROM informations_column WHERE name_column = :namecol AND data_type = :datatype AND length_column = :lengthcol AND column_text = :columntext collate utf8mb4_bin", nativeQuery = true) InfoColumn getSpecColumn(@Param("namecol") String nameCol, @Param("datatype") String dataType, @Param("lengthcol") int lengthCol, @Param("columntext") String columnText); @Modifying diff --git a/src/main/java/com/apside/assist/db/backend/service/InfoColumnService.java b/src/main/java/com/apside/assist/db/backend/service/InfoColumnService.java index a0924c8..58f7a70 100644 --- a/src/main/java/com/apside/assist/db/backend/service/InfoColumnService.java +++ b/src/main/java/com/apside/assist/db/backend/service/InfoColumnService.java @@ -68,6 +68,14 @@ public interface InfoColumnService { */ InfoColumn addOrUpdateColumn(InfoColumn infoColumn); + /** + * Ajoute une liste de colonnes + * @param listOfColumns + * @return Iterable, + */ + Iterable addMultipleColumns(Iterable listOfColumns); + + /** * Récupère les informations d'une colonne spécifique * @param nameCol diff --git a/src/main/java/com/apside/assist/db/backend/service/InfoTableService.java b/src/main/java/com/apside/assist/db/backend/service/InfoTableService.java index 8ae840a..f7f2bcc 100644 --- a/src/main/java/com/apside/assist/db/backend/service/InfoTableService.java +++ b/src/main/java/com/apside/assist/db/backend/service/InfoTableService.java @@ -1,5 +1,6 @@ package com.apside.assist.db.backend.service; +import com.apside.assist.db.backend.model.InfoColumn; import com.apside.assist.db.backend.model.InfoTable; import java.util.Optional; @@ -62,6 +63,14 @@ public interface InfoTableService { */ InfoTable addOrUpdateTable(InfoTable infoTable); + /** + * Ajoute une liste de tables + * @param listOfTables + * @return Iterable, + */ + Iterable addMultipleTables(Iterable listOfTables); + + /** * Vide la table contenant toutes les tables * @param diff --git a/src/main/java/com/apside/assist/db/backend/service/LinkInfoService.java b/src/main/java/com/apside/assist/db/backend/service/LinkInfoService.java index f94d369..8a60ea8 100644 --- a/src/main/java/com/apside/assist/db/backend/service/LinkInfoService.java +++ b/src/main/java/com/apside/assist/db/backend/service/LinkInfoService.java @@ -41,6 +41,14 @@ public interface LinkInfoService { */ LinkInfo addOrUpdateLink(LinkInfo linkInfo); + /** + * Ajoute une liste de liens + * @param listOfLinkInfo + * @return Iterable, + */ + Iterable addMultipleLinkInfo(Iterable listOfLinkInfo); + + /** * Vide la table contenant tous les liens entre tables et colonnes * @param diff --git a/src/main/java/com/apside/assist/db/backend/service/ResetDataService.java b/src/main/java/com/apside/assist/db/backend/service/ResetDataService.java index 70074c9..3d82ad8 100644 --- a/src/main/java/com/apside/assist/db/backend/service/ResetDataService.java +++ b/src/main/java/com/apside/assist/db/backend/service/ResetDataService.java @@ -1,5 +1,7 @@ package com.apside.assist.db.backend.service; +import com.fasterxml.jackson.core.JsonProcessingException; + public interface ResetDataService { /** @@ -14,12 +16,13 @@ public interface ResetDataService { * @param * @return */ - void insertEverything(); + void insertEverything() throws JsonProcessingException; /** * Insert les informations des liens entre les colonnes et les tables en base de données * @param * @return */ - void checkAndInsertLinks(); + void checkAndInsertLinks() throws JsonProcessingException; + } diff --git a/src/main/java/com/apside/assist/db/backend/service/implementation/InfoColumnServiceImpl.java b/src/main/java/com/apside/assist/db/backend/service/implementation/InfoColumnServiceImpl.java index a6361e9..b9963c1 100644 --- a/src/main/java/com/apside/assist/db/backend/service/implementation/InfoColumnServiceImpl.java +++ b/src/main/java/com/apside/assist/db/backend/service/implementation/InfoColumnServiceImpl.java @@ -66,6 +66,12 @@ public class InfoColumnServiceImpl implements InfoColumnService { return infoColumnRepository.save(infoColumn); } + @Override + public Iterable addMultipleColumns(Iterable listOfColumns) { + log.debug("Start addMultipleColumns method - CRUD"); + return infoColumnRepository.saveAll(listOfColumns); + } + @Override public InfoColumn getSpecCol(String nameCol, String dataType, int lengthCol, String columnText){ log.debug("Start getSpecCol method - Custom"); diff --git a/src/main/java/com/apside/assist/db/backend/service/implementation/InfoTableServiceImpl.java b/src/main/java/com/apside/assist/db/backend/service/implementation/InfoTableServiceImpl.java index 32221bf..68ae169 100644 --- a/src/main/java/com/apside/assist/db/backend/service/implementation/InfoTableServiceImpl.java +++ b/src/main/java/com/apside/assist/db/backend/service/implementation/InfoTableServiceImpl.java @@ -1,5 +1,6 @@ package com.apside.assist.db.backend.service.implementation; +import com.apside.assist.db.backend.model.InfoColumn; import com.apside.assist.db.backend.repository.InfoTableRepository; import com.apside.assist.db.backend.model.InfoTable; import com.apside.assist.db.backend.service.InfoTableService; @@ -65,6 +66,12 @@ public class InfoTableServiceImpl implements InfoTableService { return infoTableRepository.save(infoTable); } + @Override + public Iterable addMultipleTables(Iterable listOfTables) { + log.debug("Start addMultipleTables method - CRUD"); + return infoTableRepository.saveAll(listOfTables); + } + @Override public void truncateMyTable(){ log.debug("Start truncateMyTable method - Custom"); diff --git a/src/main/java/com/apside/assist/db/backend/service/implementation/LinkInfoServiceImpl.java b/src/main/java/com/apside/assist/db/backend/service/implementation/LinkInfoServiceImpl.java index 8be0d0e..326609b 100644 --- a/src/main/java/com/apside/assist/db/backend/service/implementation/LinkInfoServiceImpl.java +++ b/src/main/java/com/apside/assist/db/backend/service/implementation/LinkInfoServiceImpl.java @@ -47,6 +47,12 @@ public class LinkInfoServiceImpl implements LinkInfoService { return linkInfoRepository.save(linkInfo); } + @Override + public Iterable addMultipleLinkInfo(Iterable listOfLinkInfo) { + log.debug("Start addMultipleLinkInfo method - CRUD"); + return linkInfoRepository.saveAll(listOfLinkInfo); + } + @Override public void truncateMyLink(){ log.debug("Start truncateMyLink method - Custom"); diff --git a/src/main/java/com/apside/assist/db/backend/service/implementation/ResetDataServiceImpl.java b/src/main/java/com/apside/assist/db/backend/service/implementation/ResetDataServiceImpl.java index 8d87ba3..8f008cc 100644 --- a/src/main/java/com/apside/assist/db/backend/service/implementation/ResetDataServiceImpl.java +++ b/src/main/java/com/apside/assist/db/backend/service/implementation/ResetDataServiceImpl.java @@ -3,23 +3,27 @@ package com.apside.assist.db.backend.service.implementation; import com.apside.assist.db.backend.model.LinkInfo; import com.apside.assist.db.backend.model.InfoColumn; import com.apside.assist.db.backend.model.InfoTable; +import com.apside.assist.db.backend.model.json.Column; +import com.apside.assist.db.backend.model.json.DataEnum; +import com.apside.assist.db.backend.model.json.Schema; import com.apside.assist.db.backend.service.InfoColumnService; import com.apside.assist.db.backend.service.InfoTableService; import com.apside.assist.db.backend.service.LinkInfoService; import com.apside.assist.db.backend.service.ResetDataService; -import lombok.Data; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; -import org.json.JSONArray; -import org.json.JSONObject; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.beans.factory.annotation.Autowired; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; @Service @Slf4j @@ -47,6 +51,8 @@ public class ResetDataServiceImpl implements ResetDataService, InitializingBean private static final String LENGTH_STRING = "length"; private static final String COLUMN_TEXT_STRING = "column_text"; + @Value("${spring.jpa.properties.hibernate.jdbc.batch_size}") + private int batchSize; @Override public void afterPropertiesSet() throws Exception { @@ -62,140 +68,52 @@ public class ResetDataServiceImpl implements ResetDataService, InitializingBean } @Override - public void insertEverything() { + public void insertEverything() throws JsonProcessingException { log.debug("Start insert every data into DB"); - JSONArray jo = new JSONArray(result); - for (int i=0; i listOfColumns = new ArrayList<>(); + List listOfTables = new ArrayList<>(); + ObjectMapper objectMapper = new ObjectMapper(); + Schema[] jsonData = objectMapper.readValue(result, Schema[].class); + for (Schema schema: jsonData) { + for (DataEnum dat: schema.getData()) { + listOfTables.add(new InfoTable(dat.getTable(), schema.getSchema(), dat.getTable_text())); + for (Column col: dat.getColumns()) { + String descCol = col.getColumn_text(); + if(descCol == null){ + descCol = ""; } + listOfColumns.add(new InfoColumn(col.getName(), col.getData_type(), col.getLength(), descCol)); } } } + List filteredListOfColumns = listOfColumns.stream().distinct().collect(Collectors.toList()); + infoTableService.addMultipleTables(listOfTables); + infoColumnService.addMultipleColumns(filteredListOfColumns); log.info("Insert all data into DB - success"); } @Override - public void checkAndInsertLinks() { + public void checkAndInsertLinks() throws JsonProcessingException { log.debug("Start check data into DB and insert links between table and columns"); - JSONArray jo = new JSONArray(result); - for (int i=0; i listOfLink = new ArrayList<>(); + ObjectMapper objectMapper = new ObjectMapper(); + Schema[] jsonData = objectMapper.readValue(result, Schema[].class); + for (Schema schema: jsonData) { + for (DataEnum dat: schema.getData()) { + for (Column col: dat.getColumns()) { + String descCol = col.getColumn_text(); + if(descCol == null){ + descCol = ""; } - if (colData.has(DATA_TYPE_STRING)){ - dataType = colData.getString(DATA_TYPE_STRING); - } else { - dataType = ""; - } - if (colData.has(LENGTH_STRING)){ - lengthCol = colData.getInt(LENGTH_STRING); - } else { - lengthCol = 0; - } - if(colData.has(COLUMN_TEXT_STRING)) { - columnText = colData.getString(COLUMN_TEXT_STRING); - } else { - columnText = ""; - } - - InfoColumn findCol = infoColumnService.getSpecCol(nameCol, dataType, lengthCol, columnText); - Long idCol = findCol.getId(); - int intId = Math.toIntExact(idCol); - + InfoColumn findCol = infoColumnService.getSpecCol(col.getName(), col.getData_type(), col.getLength(), descCol); + int idCol = Math.toIntExact(findCol.getId()); if (findCol != null){ - LinkInfo linkInfo = new LinkInfo(); - linkInfo.setNameSchema(schemaName); - linkInfo.setNameTable(tableName); - linkInfo.setColumnId(intId); - linkInfoService.addOrUpdateLink(linkInfo); + listOfLink.add(new LinkInfo(dat.getTable(), schema.getSchema(), idCol)); } } } } + linkInfoService.addMultipleLinkInfo(listOfLink); log.info("Check and insert links data into DB - success"); } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index ba679c3..26eb942 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,6 +1,6 @@ spring.application.name=AssistDBBackend -spring.datasource.url=jdbc:mysql://localhost:3306/prudencetest +spring.datasource.url=jdbc:mysql://localhost:3306/prudencetest?rewriteBatchedStatements=true&cachePrepStmts=true&useServerPrepStmts=true spring.datasource.username=root spring.datasource.password=Pompom.21 server.port=9001 @@ -16,4 +16,6 @@ PATH_TO_SCRIPT_DIR = /AssistDB_AdditionalFiles/Scripts PATH_FOR_TAG_JSON = /AssistDB_AdditionalFiles/tags.json logging.level.org.springframework.boot.web.embedded.tomcat=INFO -logging.level.org.springframework=error \ No newline at end of file +logging.level.org.springframework=error + +spring.jpa.properties.hibernate.jdbc.batch_size=30 \ No newline at end of file