解决导入数据量太大导致oom问题：按需读取数据

slibra-admin/pom.xml

@@ -132,6 +132,12 @@
             <version>1.62.2</version>
         </dependency>
 
+        <!-- https://mvnrepository.com/artifact/com.alibaba/easyexcel -->
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>easyexcel</artifactId>
+            <version>3.3.2</version>
+        </dependency>
 
 
 

slibra-admin/src/main/java/com/slibra/web/controller/business/ExcelController.java

@@ -1,5 +1,6 @@
 package com.slibra.web.controller.business;
 
+import com.alibaba.excel.EasyExcel;
 import com.alibaba.fastjson2.JSON;
 import com.slibra.business.domain.*;
 import com.slibra.business.mapper.*;
@@ -14,6 +15,7 @@ import com.slibra.common.core.domain.entity.SysUser;
 import com.slibra.common.exception.ServiceException;
 import com.slibra.common.utils.StringUtils;
 import com.slibra.common.utils.poi.ExcelUtil;
+import com.slibra.web.controller.listener.UserDataListener;
 import io.micrometer.core.annotation.Timed;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.ibatis.session.ExecutorType;
@@ -170,7 +172,7 @@ public class ExcelController extends BaseController {
      * 将Excel中的用户数据 导入到数据库中
      * @return
      */
-    @GetMapping("/userExcel2DB")
+//    @GetMapping("/userExcel2DB")
     @Transactional
     public String userExcel2DB(){
         long begin = System.currentTimeMillis();
@@ -184,6 +186,7 @@ public class ExcelController extends BaseController {
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
+        System.out.println("要处理的数据条数为：" + dataList.size());
         if (!CollectionUtils.isEmpty(dataList)) {
             /*SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
             TUserInfoMapper sqlSessionMapper = sqlSession.getMapper(TUserInfoMapper.class);*/
@@ -218,6 +221,23 @@ public class ExcelController extends BaseController {
 
 
 
+    /**
+     *
+     * 将Excel中的用户数据 导入到数据库中--减少内存使用
+     * @return
+     */
+    @GetMapping("/userExcel2DBFetch")
+    @Transactional
+    public String userExcel2DBFetch(){
+        String fileName = "/Users/wangmiaomiao/Documents/Excel/jmsuser.xlsx"; // Excel文件路径
+        SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
+            TUserInfoMapper sqlSessionMapper = sqlSession.getMapper(TUserInfoMapper.class);
+        // 读取Excel文件
+        EasyExcel.read(fileName, UserExcelInfo.class, new UserDataListener(sqlSession, sqlSessionMapper))
+                .sheet("全部可用个人信息") // 读取第一个工作表
+                .doRead(); // 触发实际的读取操作
+        return "操作成功";
+    }
 
 
 

slibra-admin/src/main/java/com/slibra/web/controller/listener/UserDataListener.java

@@ -0,0 +1,72 @@
+package com.slibra.web.controller.listener;
+
+import com.alibaba.excel.context.AnalysisContext;
+import com.alibaba.excel.event.AnalysisEventListener;
+import com.alibaba.fastjson.JSON;
+import com.slibra.business.domain.TUserInfo;
+import com.slibra.business.mapper.TUserInfoMapper;
+import com.slibra.business.res.UserExcelInfo;
+import org.apache.ibatis.session.ExecutorType;
+import org.apache.ibatis.session.SqlSessionFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import org.apache.ibatis.session.SqlSession;
+import org.apache.ibatis.session.SqlSessionFactory;
+
+@Component
+public class UserDataListener extends AnalysisEventListener<UserExcelInfo> {
+
+//    @Autowired
+//    private TUserInfoMapper tUserInfoMapper;
+
+//    @Autowired
+    /*private SqlSessionFactory sqlSessionFactory;
+
+    SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);*/
+    SqlSession sqlSession;
+    TUserInfoMapper sqlSessionMapper;
+
+    @Override
+    public void invoke(UserExcelInfo userExcelInfo, AnalysisContext context) {
+
+        // 处理读取到的每一行数据
+//        System.out.println("读取到数据: " + JSON.toJSONString(userExcelInfo));
+        TUserInfo tUserInfo = new TUserInfo();
+//                BeanUtils.copyProperties(userExcelInfo, tUserInfo);
+        //因为数据存在空格等 需要额外处理一下
+        tUserInfo.setUserNo(userExcelInfo.getUserNo().trim());
+        tUserInfo.setAmmeterNo(userExcelInfo.getAmmeterNo().trim());
+        tUserInfo.setPumpingStationAddress(userExcelInfo.getPumpingStationAddress().trim());
+        tUserInfo.setRemark(userExcelInfo.getRemark().trim());
+        tUserInfo.setStreet(userExcelInfo.getStreet().trim());
+        tUserInfo.setNeighbourhoodName(userExcelInfo.getNeighbourhoodName().trim());
+        tUserInfo.setBuildingNo(userExcelInfo.getBuildingNo().trim());
+        tUserInfo.setDoorNo(userExcelInfo.getDoorNo().trim());
+        tUserInfo.setMeterReader(userExcelInfo.getMeterReader().trim());
+        tUserInfo.setMeterReaderPhone(userExcelInfo.getMeterReaderPhone().trim());
+//        tUserInfoMapper.insertTUserInfo(tUserInfo);
+        sqlSessionMapper.insertTUserInfo(tUserInfo);
+    }
+
+    @Override
+    public void doAfterAllAnalysed(AnalysisContext context) {
+        // 所有数据读取完毕后的处理逻辑
+        System.out.println("所有数据读取完毕");
+        sqlSession.commit();
+        sqlSession.close();
+    }
+
+
+    public UserDataListener(TUserInfoMapper sqlSessionMapper) {
+        this.sqlSessionMapper = sqlSessionMapper;
+    }
+
+    public UserDataListener(SqlSession sqlSession, TUserInfoMapper sqlSessionMapper) {
+        this.sqlSession = sqlSession;
+        this.sqlSessionMapper = sqlSessionMapper;
+    }
+
+    public UserDataListener() {
+    }
+}

slibra-admin/src/main/resources/application-local.yml

@@ -117,3 +117,10 @@ big-model:
 
 log:
   logPath: /Users/wangmiaomiao/logs
+
+
+SIP_SUFFIX: '@pbx.fuxicarbon.com:5060'
+FILE_URL_PREFIX: /home/hongshan
+#2024年12月31日16:37:03  从docker容器获取配置
+#SIP_SUFFIX: ${SIP_SUFFIX}
+#FILE_URL_PREFIX: ${FILE_URL_PREFIX}