add efficiency to fill database (approx 19 min)

src/main/java/com/apside/assist/db/backend/controller/ResetDataController.java

@@ -2,6 +2,7 @@ package com.apside.assist.db.backend.controller;
 
 import java.io.IOException;
 import com.apside.assist.db.backend.service.ResetDataService;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
@@ -15,7 +16,6 @@ public class ResetDataController {
     @Autowired
     private ResetDataService resetDataService;
 
-
     @PostMapping("/reset")
     public ResponseEntity<Void> resetData() {
         log.debug("ResetData called");

src/main/java/com/apside/assist/db/backend/model/InfoColumn.java

@@ -34,4 +34,11 @@ public class InfoColumn {
         this.lengthColumn = lengthColumn;
         this.columnText = columnText;
     }
+
+    public InfoColumn(String nameColumn, String dataType, int lengthColumn, String columnText){
+        this.nameColumn = nameColumn;
+        this.dataType = dataType;
+        this.lengthColumn = lengthColumn;
+        this.columnText = columnText;
+    }
 }

src/main/java/com/apside/assist/db/backend/model/InfoTable.java

@@ -30,4 +30,10 @@ public class InfoTable {
         this.nameSchema = nameSchema;
         this.tableText = tableText;
     }
+
+    public InfoTable(String table, String schema, String tableText) {
+        this.nameTable = table;
+        this.nameSchema = schema;
+        this.tableText = tableText;
+    }
 }

src/main/java/com/apside/assist/db/backend/model/LinkInfo.java

@@ -28,4 +28,10 @@ public class LinkInfo {
         this.nameSchema = nameSchema;
         this.columnId = columnId;
     }
+
+    public LinkInfo(String nameTable, String nameSchema, int columnId){
+        this.nameTable = nameTable;
+        this.nameSchema = nameSchema;
+        this.columnId = columnId;
+    }
 }

src/main/java/com/apside/assist/db/backend/model/json/Column.java

@@ -0,0 +1,11 @@
+package com.apside.assist.db.backend.model.json;
+
+import lombok.Data;
+
+@Data
+public class Column {
+    private String name;
+    private String data_type;
+    private int length;
+    private String column_text;
+}

src/main/java/com/apside/assist/db/backend/model/json/DataEnum.java

@@ -0,0 +1,13 @@
+package com.apside.assist.db.backend.model.json;
+
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class DataEnum {
+    private List<Column> columns;
+    private String table_text;
+    private String table;
+}

src/main/java/com/apside/assist/db/backend/model/json/Schema.java

@@ -0,0 +1,11 @@
+package com.apside.assist.db.backend.model.json;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class Schema {
+    private String schema;
+    private List<DataEnum> data;
+}

src/main/java/com/apside/assist/db/backend/repository/InfoColumnRepository.java

@@ -25,7 +25,7 @@ public interface InfoColumnRepository extends CrudRepository<InfoColumn, Long> {
             "(SELECT column_id FROM link_informations WHERE name_table IN :tables AND name_schema IN :schemas GROUP BY column_id  HAVING COUNT(column_id) > 1)", nativeQuery = true)
     Iterable<InfoColumn> getColumnsForJoinTwo(@RequestParam("tables") List<String> tables, @RequestParam("schemas") List<String> schemas);
 
-    @Query(value = "SELECT * FROM informations_column WHERE name_column = :namecol AND data_type = :datatype AND length_column = :lengthcol AND column_text = :columntext", nativeQuery = true)
+    @Query(value = "SELECT * FROM informations_column WHERE name_column = :namecol AND data_type = :datatype AND length_column = :lengthcol AND column_text = :columntext collate utf8mb4_bin", nativeQuery = true)
     InfoColumn getSpecColumn(@Param("namecol") String nameCol, @Param("datatype") String dataType, @Param("lengthcol") int lengthCol, @Param("columntext") String columnText);
 
     @Modifying

src/main/java/com/apside/assist/db/backend/service/InfoColumnService.java

@@ -68,6 +68,14 @@ public interface InfoColumnService {
      */
     InfoColumn addOrUpdateColumn(InfoColumn infoColumn);
 
+    /**
+     * Ajoute une liste de colonnes
+     * @param listOfColumns
+     * @return Iterable<InfoColum>,
+     */
+    Iterable<InfoColumn> addMultipleColumns(Iterable<InfoColumn> listOfColumns);
+
+
     /**
      * Récupère les informations d'une colonne spécifique
      * @param nameCol

src/main/java/com/apside/assist/db/backend/service/InfoTableService.java

@@ -1,5 +1,6 @@
 package com.apside.assist.db.backend.service;
 
+import com.apside.assist.db.backend.model.InfoColumn;
 import com.apside.assist.db.backend.model.InfoTable;
 
 import java.util.Optional;
@@ -62,6 +63,14 @@ public interface InfoTableService {
      */
     InfoTable addOrUpdateTable(InfoTable infoTable);
 
+    /**
+     * Ajoute une liste de tables
+     * @param listOfTables
+     * @return Iterable<InfoTable>,
+     */
+    Iterable<InfoTable> addMultipleTables(Iterable<InfoTable> listOfTables);
+
+
     /**
      * Vide la table contenant toutes les tables
      * @param

src/main/java/com/apside/assist/db/backend/service/LinkInfoService.java

@@ -41,6 +41,14 @@ public interface LinkInfoService {
      */
     LinkInfo addOrUpdateLink(LinkInfo linkInfo);
 
+    /**
+     * Ajoute une liste de liens
+     * @param listOfLinkInfo
+     * @return Iterable<LinkInfo>,
+     */
+    Iterable<LinkInfo> addMultipleLinkInfo(Iterable<LinkInfo> listOfLinkInfo);
+
+
     /**
      * Vide la table contenant tous les liens entre tables et colonnes
      * @param

src/main/java/com/apside/assist/db/backend/service/ResetDataService.java

@@ -1,5 +1,7 @@
 package com.apside.assist.db.backend.service;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+
 public interface ResetDataService {
 
     /**
@@ -14,12 +16,13 @@ public interface ResetDataService {
      * @param
      * @return
      */
-    void insertEverything();
+    void insertEverything() throws JsonProcessingException;
 
     /**
      * Insert les informations des liens entre les colonnes et les tables en base de données
      * @param
      * @return
      */
-    void checkAndInsertLinks();
+    void checkAndInsertLinks() throws JsonProcessingException;
+
 }

src/main/java/com/apside/assist/db/backend/service/implementation/InfoColumnServiceImpl.java

@@ -66,6 +66,12 @@ public class InfoColumnServiceImpl implements InfoColumnService {
         return infoColumnRepository.save(infoColumn);
     }
 
+    @Override
+    public Iterable<InfoColumn> addMultipleColumns(Iterable<InfoColumn> listOfColumns) {
+        log.debug("Start addMultipleColumns method - CRUD");
+        return infoColumnRepository.saveAll(listOfColumns);
+    }
+
     @Override
     public InfoColumn getSpecCol(String nameCol, String dataType, int lengthCol, String columnText){
         log.debug("Start getSpecCol method - Custom");

src/main/java/com/apside/assist/db/backend/service/implementation/InfoTableServiceImpl.java

@@ -1,5 +1,6 @@
 package com.apside.assist.db.backend.service.implementation;
 
+import com.apside.assist.db.backend.model.InfoColumn;
 import com.apside.assist.db.backend.repository.InfoTableRepository;
 import com.apside.assist.db.backend.model.InfoTable;
 import com.apside.assist.db.backend.service.InfoTableService;
@@ -65,6 +66,12 @@ public class InfoTableServiceImpl implements InfoTableService {
         return infoTableRepository.save(infoTable);
     }
 
+    @Override
+    public Iterable<InfoTable> addMultipleTables(Iterable<InfoTable> listOfTables) {
+        log.debug("Start addMultipleTables method - CRUD");
+        return infoTableRepository.saveAll(listOfTables);
+    }
+
     @Override
     public void truncateMyTable(){
         log.debug("Start truncateMyTable method - Custom");

src/main/java/com/apside/assist/db/backend/service/implementation/LinkInfoServiceImpl.java

@@ -47,6 +47,12 @@ public class LinkInfoServiceImpl implements LinkInfoService {
         return linkInfoRepository.save(linkInfo);
     }
 
+    @Override
+    public Iterable<LinkInfo> addMultipleLinkInfo(Iterable<LinkInfo> listOfLinkInfo) {
+        log.debug("Start addMultipleLinkInfo method - CRUD");
+        return linkInfoRepository.saveAll(listOfLinkInfo);
+    }
+
     @Override
     public void truncateMyLink(){
         log.debug("Start truncateMyLink method - Custom");

src/main/java/com/apside/assist/db/backend/service/implementation/ResetDataServiceImpl.java

@@ -3,23 +3,27 @@ package com.apside.assist.db.backend.service.implementation;
 import com.apside.assist.db.backend.model.LinkInfo;
 import com.apside.assist.db.backend.model.InfoColumn;
 import com.apside.assist.db.backend.model.InfoTable;
+import com.apside.assist.db.backend.model.json.Column;
+import com.apside.assist.db.backend.model.json.DataEnum;
+import com.apside.assist.db.backend.model.json.Schema;
 import com.apside.assist.db.backend.service.InfoColumnService;
 import com.apside.assist.db.backend.service.InfoTableService;
 import com.apside.assist.db.backend.service.LinkInfoService;
 import com.apside.assist.db.backend.service.ResetDataService;
-import lombok.Data;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
-import org.json.JSONArray;
-import org.json.JSONObject;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.springframework.beans.factory.annotation.Autowired;
 
-import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
 
 @Service
 @Slf4j
@@ -47,6 +51,8 @@ public class ResetDataServiceImpl implements ResetDataService, InitializingBean
     private static final String LENGTH_STRING = "length";
     private static final String COLUMN_TEXT_STRING = "column_text";
 
+    @Value("${spring.jpa.properties.hibernate.jdbc.batch_size}")
+    private int batchSize;
 
     @Override
     public void afterPropertiesSet() throws Exception {
@@ -62,140 +68,52 @@ public class ResetDataServiceImpl implements ResetDataService, InitializingBean
     }
 
     @Override
-    public void insertEverything() {
+    public void insertEverything() throws JsonProcessingException {
         log.debug("Start insert every data into DB");
-        JSONArray jo = new JSONArray(result);
+        List<InfoColumn> listOfColumns = new ArrayList<>();
-        for (int i=0; i<jo.length(); i++){
+        List<InfoTable> listOfTables = new ArrayList<>();
-            JSONObject schema = jo.getJSONObject(i);
+        ObjectMapper objectMapper = new ObjectMapper();
-            String schemaName = schema.getString(SCHEMA_STRING);
+        Schema[] jsonData = objectMapper.readValue(result, Schema[].class);
-            JSONArray data = schema.getJSONArray(DATA_STRING);
+        for (Schema schema: jsonData) {
-
+            for (DataEnum dat: schema.getData()) {
-            for (int ia = 0; ia<data.length(); ia++){
+                listOfTables.add(new InfoTable(dat.getTable(), schema.getSchema(), dat.getTable_text()));
-                JSONObject apra = data.getJSONObject(ia);
+                for (Column col: dat.getColumns()) {
-                String tableText;
+                    String descCol = col.getColumn_text();
-                String tableName;
+                    if(descCol == null){
-                if(apra.has(TABLE_TEXT_STRING)){
+                        descCol = "";
-                    tableText = apra.getString(TABLE_TEXT_STRING);
-                } else {
-                    tableText = "";
-                }
-                if(apra.has(TABLE_STRING)){
-                    tableName = apra.getString(TABLE_STRING);
-                } else {
-                    tableName = "";
-                }
-
-                JSONArray col = apra.getJSONArray(COLUMN_STRING);
-                InfoTable infoTable = new InfoTable();
-                infoTable.setNameTable(tableName);
-                infoTable.setNameSchema(schemaName);
-                infoTable.setTableText(tableText);
-                infoTableService.addOrUpdateTable(infoTable);
-
-                for (int ib =0; ib<col.length(); ib++){
-                    JSONObject colData = col.getJSONObject(ib);
-                    String nameCol;
-                    String dataType;
-                    String columnText;
-                    int lengthCol;
-                    if(colData.has(NAME_STRING)){
-                        nameCol = colData.getString(NAME_STRING);
-                    } else {
-                        nameCol = "";
-                    }
-                    if (colData.has(DATA_TYPE_STRING)){
-                        dataType = colData.getString(DATA_TYPE_STRING);
-                    } else {
-                        dataType = "";
-                    }
-                    if (colData.has(LENGTH_STRING)){
-                        lengthCol = colData.getInt(LENGTH_STRING);
-                    } else {
-                        lengthCol = 0;
-                    }
-                    if(colData.has(COLUMN_TEXT_STRING)) {
-                        columnText = colData.getString(COLUMN_TEXT_STRING);
-                    } else {
-                        columnText = "";
-                    }
-
-                    InfoColumn infoColumn = new InfoColumn();
-                    infoColumn.setNameColumn(nameCol);
-                    infoColumn.setDataType(dataType);
-                    infoColumn.setLengthColumn(lengthCol);
-                    infoColumn.setColumnText(columnText);
-
-                    InfoColumn findCol = infoColumnService.getSpecCol(nameCol, dataType, lengthCol, columnText);
-
-                    if(findCol == null){
-                        infoColumnService.addOrUpdateColumn(infoColumn);
                     }
+                    listOfColumns.add(new InfoColumn(col.getName(), col.getData_type(), col.getLength(), descCol));
                 }
             }
         }
+        List<InfoColumn> filteredListOfColumns = listOfColumns.stream().distinct().collect(Collectors.toList());
+        infoTableService.addMultipleTables(listOfTables);
+        infoColumnService.addMultipleColumns(filteredListOfColumns);
         log.info("Insert all data into DB - success");
     }
 
     @Override
-    public void checkAndInsertLinks() {
+    public void checkAndInsertLinks() throws JsonProcessingException {
         log.debug("Start check data into DB and insert links between table and columns");
-        JSONArray jo = new JSONArray(result);
+        List<LinkInfo> listOfLink = new ArrayList<>();
-        for (int i=0; i<jo.length(); i++){
+        ObjectMapper objectMapper = new ObjectMapper();
-            JSONObject schema = jo.getJSONObject(i);
+        Schema[] jsonData = objectMapper.readValue(result, Schema[].class);
-            String schemaName = schema.getString(SCHEMA_STRING);
+        for (Schema schema: jsonData) {
-            JSONArray data = schema.getJSONArray(DATA_STRING);
+            for (DataEnum dat: schema.getData()) {
-
+                for (Column col: dat.getColumns()) {
-            for (int ia = 0; ia<data.length(); ia++){
+                    String descCol = col.getColumn_text();
-                JSONObject apra = data.getJSONObject(ia);
+                    if(descCol == null){
-                String tableName;
+                        descCol = "";
-                if(apra.has(TABLE_STRING)){
+                    }
-                    tableName = apra.getString(TABLE_STRING);
+                    InfoColumn findCol = infoColumnService.getSpecCol(col.getName(), col.getData_type(), col.getLength(), descCol);
-                } else {
+                    int idCol = Math.toIntExact(findCol.getId());
-                    tableName = "";
-                }
-                JSONArray col = apra.getJSONArray(COLUMN_STRING);
-
-                for (int ib =0; ib<col.length(); ib++){
-                    JSONObject colData = col.getJSONObject(ib);
-                    String nameCol;
-                    String dataType;
-                    String columnText;
-                    int lengthCol;
-                    if(colData.has(NAME_STRING)){
-                        nameCol = colData.getString(NAME_STRING);
-                    } else {
-                        nameCol = "";
-                    }
-                    if (colData.has(DATA_TYPE_STRING)){
-                        dataType = colData.getString(DATA_TYPE_STRING);
-                    } else {
-                        dataType = "";
-                    }
-                    if (colData.has(LENGTH_STRING)){
-                        lengthCol = colData.getInt(LENGTH_STRING);
-                    } else {
-                        lengthCol = 0;
-                    }
-                    if(colData.has(COLUMN_TEXT_STRING)) {
-                        columnText = colData.getString(COLUMN_TEXT_STRING);
-                    } else {
-                        columnText = "";
-                    }
-
-                    InfoColumn findCol = infoColumnService.getSpecCol(nameCol, dataType, lengthCol, columnText);
-                    Long idCol = findCol.getId();
-                    int intId = Math.toIntExact(idCol);
-
                     if (findCol != null){
-                        LinkInfo linkInfo = new LinkInfo();
+                        listOfLink.add(new LinkInfo(dat.getTable(), schema.getSchema(), idCol));
-                        linkInfo.setNameSchema(schemaName);
-                        linkInfo.setNameTable(tableName);
-                        linkInfo.setColumnId(intId);
-                        linkInfoService.addOrUpdateLink(linkInfo);
                     }
                 }
             }
         }
+        linkInfoService.addMultipleLinkInfo(listOfLink);
         log.info("Check and insert links data into DB - success");
     }
 

src/main/resources/application.properties

@@ -1,6 +1,6 @@
 
 spring.application.name=AssistDBBackend
-spring.datasource.url=jdbc:mysql://localhost:3306/prudencetest
+spring.datasource.url=jdbc:mysql://localhost:3306/prudencetest?rewriteBatchedStatements=true&cachePrepStmts=true&useServerPrepStmts=true
 spring.datasource.username=root
 spring.datasource.password=Pompom.21
 server.port=9001
@@ -17,3 +17,5 @@ PATH_FOR_TAG_JSON = /AssistDB_AdditionalFiles/tags.json
 
 logging.level.org.springframework.boot.web.embedded.tomcat=INFO
 logging.level.org.springframework=error
+
+spring.jpa.properties.hibernate.jdbc.batch_size=30