diff --git a/srs-admin/src/main/java/com/srs/web/controller/comprehensive/CphStuScoreMiddleController.java b/srs-admin/src/main/java/com/srs/web/controller/comprehensive/CphStuScoreMiddleController.java index d7f6625..b9b3a4c 100644 --- a/srs-admin/src/main/java/com/srs/web/controller/comprehensive/CphStuScoreMiddleController.java +++ b/srs-admin/src/main/java/com/srs/web/controller/comprehensive/CphStuScoreMiddleController.java @@ -9,6 +9,7 @@ import com.srs.comprehensive.domain.CphStuScoreMiddle; import com.srs.comprehensive.domain.Dto.CphStuScoreMiddleDto; import com.srs.comprehensive.domain.Vo.CphStuScoreMiddleDtoVo; import com.srs.comprehensive.service.ICphStuScoreMiddlesService; +import com.srs.comprehensive.config.SyncConfig; import com.sun.net.httpserver.Authenticator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; @@ -21,6 +22,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static com.srs.common.core.domain.AjaxResult.success; @@ -34,6 +38,9 @@ public class CphStuScoreMiddleController extends BaseController { @Autowired public ICphStuScoreMiddlesService cphStuScoreMiddlesService; + + @Autowired + private SyncConfig syncConfig; //同步学生成绩 @GetMapping("/selectAllw") @@ -57,28 +64,85 @@ public class CphStuScoreMiddleController extends BaseController { } //同步学生成绩，清空在添加 - @GetMapping("/selectAll")//16：03 + @GetMapping("/selectAll") public AjaxResult selectCphStuScoreMiddleEmptyAndAdd(){ - cphStuScoreMiddlesService.emptyTableDate(); - int pageNum=1; - int pageSize=3000; - int studentInfoNumber = cphStuScoreMiddlesService.serectCphStuScoreMiddleCount(null);//总数 - int sum=studentInfoNumber/ pageSize;//需要循环的次数 - int sumS=studentInfoNumber%pageSize>0? sum + 1: sum; - QueryWrapper objectQueryWrapper = new QueryWrapper<>(); - objectQueryWrapper.orderByDesc("stu_no", "kcdm"); - ExecutorService executor = Executors.newFixedThreadPool(15); - for (pageNum=1; pageNum <= sumS; pageNum++){ - final int currentPage = pageNum; - executor.execute(() -> { - List> cphStuScoreMiddles = cphStuScoreMiddlesService - .serectCphStuScoreMiddlesXh(currentPage, pageSize, null); - cphStuScoreMiddlesService.selectCphStuScoreMiddleEmptyAndAdd(cphStuScoreMiddles); - cphStuScoreMiddles.clear(); - }); + try { + logger.info("开始同步学生成绩数据..."); + long startTime = System.currentTimeMillis(); + + // 清空现有数据 + cphStuScoreMiddlesService.emptyTableDate(); + logger.info("已清空现有数据"); + + // 配置参数 + int pageSize = syncConfig.getPageSize(); + int threadPoolSize = syncConfig.getWebThreadPoolSize(); + + // 获取总数 + int studentInfoNumber = cphStuScoreMiddlesService.serectCphStuScoreMiddleCount(null); + int totalPages = (int) Math.ceil((double) studentInfoNumber / pageSize); + + logger.info("总记录数: {}, 总页数: {}, 每页大小: {}", studentInfoNumber, totalPages, pageSize); + + // 创建线程池 + ExecutorService executor = Executors.newFixedThreadPool(threadPoolSize); + CountDownLatch latch = new CountDownLatch(totalPages); + AtomicInteger successCount = new AtomicInteger(0); + AtomicInteger errorCount = new AtomicInteger(0); + + // 提交任务 + for (int pageNum = 1; pageNum <= totalPages; pageNum++) { + final int currentPage = pageNum; + executor.submit(() -> { + try { + // 分页查询数据 + List> cphStuScoreMiddles = cphStuScoreMiddlesService + .serectCphStuScoreMiddlesXh(currentPage, pageSize, null); + + if (cphStuScoreMiddles != null && !cphStuScoreMiddles.isEmpty()) { + // 处理数据 + cphStuScoreMiddlesService.selectCphStuScoreMiddleEmptyAndAdd(cphStuScoreMiddles); + successCount.incrementAndGet(); + logger.debug("第{}页处理完成，记录数: {}", currentPage, cphStuScoreMiddles.size()); + } + + } catch (Exception e) { + errorCount.incrementAndGet(); + logger.error("处理第{}页时发生错误: {}", currentPage, e.getMessage(), e); + } finally { + latch.countDown(); + } + }); + } + + // 等待所有任务完成 + boolean completed = latch.await(syncConfig.getTimeoutMinutes(), TimeUnit.MINUTES); + + if (!completed) { + logger.warn("同步任务未在指定时间内完成"); + return AjaxResult.error("同步任务超时"); + } + + // 关闭线程池 + executor.shutdown(); + + long endTime = System.currentTimeMillis(); + long duration = endTime - startTime; + + logger.info("学生成绩同步完成 - 总耗时: {}ms, 成功页数: {}, 失败页数: {}", + duration, successCount.get(), errorCount.get()); + + if (errorCount.get() > 0) { + return AjaxResult.warn("同步完成，但有部分数据失败。成功: " + successCount.get() + + "，失败: " + errorCount.get()); + } + + return AjaxResult.success("同步完成，共处理 " + totalPages + " 页数据，耗时 " + duration + "ms"); + + } catch (Exception e) { + logger.error("同步学生成绩数据时发生严重错误", e); + return AjaxResult.error("同步失败: " + e.getMessage()); } - executor.shutdown(); - return success(); } //查询sqlserver学生成绩 diff --git a/srs-admin/src/main/resources/application-sync.yml b/srs-admin/src/main/resources/application-sync.yml new file mode 100644 index 0000000..dd47cc6 --- /dev/null +++ b/srs-admin/src/main/resources/application-sync.yml @@ -0,0 +1,13 @@ +# 数据同步配置 +sync: + student-score: + # 分页大小，建议根据内存和数据库性能调整 + page-size: 3000 + # Web接口线程池大小，建议不超过CPU核心数*2 + web-thread-pool-size: 10 + # 定时任务线程池大小，建议不超过CPU核心数 + scheduled-thread-pool-size: 8 + # 同步超时时间（分钟） + timeout-minutes: 300 + # 批量插入大小，建议根据数据库性能调整（MySQL建议1000-2000） + batch-insert-size: 1000 diff --git a/srs-comprehensive/src/main/java/com/srs/comprehensive/config/SyncConfig.java b/srs-comprehensive/src/main/java/com/srs/comprehensive/config/SyncConfig.java new file mode 100644 index 0000000..0bb88a2 --- /dev/null +++ b/srs-comprehensive/src/main/java/com/srs/comprehensive/config/SyncConfig.java @@ -0,0 +1,78 @@ +package com.srs.comprehensive.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * 数据同步配置类 + * 用于管理数据同步相关的参数配置 + */ +@Component +@ConfigurationProperties(prefix = "sync.student-score") +public class SyncConfig { + + /** + * 分页大小 + */ + private int pageSize = 3000; + + /** + * Web接口线程池大小 + */ + private int webThreadPoolSize = 10; + + /** + * 定时任务线程池大小 + */ + private int scheduledThreadPoolSize = 8; + + /** + * 同步超时时间（分钟） + */ + private int timeoutMinutes = 30; + + /** + * 批量插入大小 + */ + private int batchInsertSize = 1000; + + public int getPageSize() { + return pageSize; + } + + public void setPageSize(int pageSize) { + this.pageSize = pageSize; + } + + public int getWebThreadPoolSize() { + return webThreadPoolSize; + } + + public void setWebThreadPoolSize(int webThreadPoolSize) { + this.webThreadPoolSize = webThreadPoolSize; + } + + public int getScheduledThreadPoolSize() { + return scheduledThreadPoolSize; + } + + public void setScheduledThreadPoolSize(int scheduledThreadPoolSize) { + this.scheduledThreadPoolSize = scheduledThreadPoolSize; + } + + public int getTimeoutMinutes() { + return timeoutMinutes; + } + + public void setTimeoutMinutes(int timeoutMinutes) { + this.timeoutMinutes = timeoutMinutes; + } + + public int getBatchInsertSize() { + return batchInsertSize; + } + + public void setBatchInsertSize(int batchInsertSize) { + this.batchInsertSize = batchInsertSize; + } +} diff --git a/srs-comprehensive/src/main/java/com/srs/comprehensive/service/impl/CphStuScoreMiddlesServiceImpl.java b/srs-comprehensive/src/main/java/com/srs/comprehensive/service/impl/CphStuScoreMiddlesServiceImpl.java index d1f0461..5f71a1c 100644 --- a/srs-comprehensive/src/main/java/com/srs/comprehensive/service/impl/CphStuScoreMiddlesServiceImpl.java +++ b/srs-comprehensive/src/main/java/com/srs/comprehensive/service/impl/CphStuScoreMiddlesServiceImpl.java @@ -12,6 +12,7 @@ import com.srs.comprehensive.mapper.CphStuScoreMiddleDtoMapper; import com.srs.comprehensive.mapper.CphStuScoreMiddlesMapper; import com.srs.comprehensive.service.ICphStuScoreMiddlesService; import com.srs.comprehensive.util.ListSliceUtil; +import com.srs.comprehensive.config.SyncConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -27,6 +28,9 @@ public class CphStuScoreMiddlesServiceImpl implements ICphStuScoreMiddlesService @Autowired private CphStuScoreMiddleDtoMapper cphStuScoreMiddleDtoMapper; + + @Autowired + private SyncConfig syncConfig; @Override @DataSource(DataSourceType.DATABASE)//sqlserver @@ -183,68 +187,115 @@ public class CphStuScoreMiddlesServiceImpl implements ICphStuScoreMiddlesService //转换 private CphStuScoreMiddleDto convertToB(Map a) { - CphStuScoreMiddleDto b = new CphStuScoreMiddleDto(); - // 进行相应的属性赋值 - if (a.get("CJ")!=null) { - if (Objects.equals(a.get("CJ").toString(), "不合格")) { - b.setCj("45.00"); - } else if (Objects.equals(a.get("CJ").toString(), "合格")) { - b.setCj("65.00"); - } else if (Objects.equals(a.get("CJ").toString(), "中等")) { - b.setCj("75.00"); - } else if (Objects.equals(a.get("CJ").toString(), "良好")) { - b.setCj("85.00"); - } else if (Objects.equals(a.get("CJ").toString(), "优秀")) { - b.setCj("95.00"); - } else if (Objects.equals(a.get("CJ").toString(), "0.00")|| - Objects.equals(a.get("CJ").toString(), "")|| - Objects.equals(a.get("CJ").toString(), null)|| - Objects.equals(a.get("CJ").toString(), "NULL")|| - Objects.equals(a.get("CJ").toString(), "null")) { - System.out.println(a.get("CJ")); - if (a.get("BkScore")!=null){ - if (!Objects.equals(a.get("BkScore").toString(), "0.00") || - !Objects.equals(a.get("BkScore").toString(), "")|| - !Objects.equals(a.get("BkScore").toString(), "null")|| - !Objects.equals(a.get("BkScore").toString(), "NULL")|| - !Objects.equals(a.get("BkScore").toString(), null)){ - b.setCj(a.get("BkScore").toString()); - } else { - b.setCj(a.get("CJ").toString()); - } - }else { - b.setCj(a.get("CJ").toString()); - } - }else { - b.setCj(a.get("CJ").toString()); + if (a == null) { + return null; + } + + try { + CphStuScoreMiddleDto b = new CphStuScoreMiddleDto(); + + // 设置学号和课程代码（必要字段） + b.setStuNo(getStringValue(a, "XH")); + b.setKcdm(getStringValue(a, "KCDM")); + b.setXqid(1L); + + // 处理成绩字段 + String score = processScore(a); + b.setCj(score); + + // 设置其他字段 + b.setXndm(getStringValue(a, "XND")); + b.setXqdm(getStringValue(a, "XQMC")); + + // 处理数值类型字段 + setBigDecimalValue(b::setXf, a, "XSHDXF"); + setBigDecimalValue(b::setJd, a, "XSHDJD"); + setLongValue(b::setFzlx, a, "FZLX"); + + // 设置是否通过 + b.setIsPass(getStringValue(a, "Passed")); + + return b; + + } catch (Exception e) { + System.err.println("转换数据时发生错误: " + e.getMessage()); + return null; + } + } + + /** + * 安全获取字符串值 + */ + private String getStringValue(Map map, String key) { + Object value = map.get(key); + if (value == null) { + return null; + } + String strValue = value.toString().trim(); + return strValue.isEmpty() || "null".equalsIgnoreCase(strValue) || "NULL".equals(strValue) ? null : strValue; + } + + /** + * 安全设置BigDecimal值 + */ + private void setBigDecimalValue(java.util.function.Consumer setter, Map map, String key) { + try { + String value = getStringValue(map, key); + if (value != null) { + setter.accept(new BigDecimal(value)); } + } catch (NumberFormatException e) { + // 忽略格式错误，保持null值 } - b.setXqid(1L); - if (a.get("XND") != null) { - b.setXndm(a.get("XND").toString()); + } + + /** + * 安全设置Long值 + */ + private void setLongValue(java.util.function.Consumer setter, Map map, String key) { + try { + String value = getStringValue(map, key); + if (value != null) { + setter.accept(Long.parseLong(value)); + } + } catch (NumberFormatException e) { + // 忽略格式错误，保持null值 } - if (a.get("XQMC")!=null) { - b.setXqdm(a.get("XQMC").toString()); + } + + /** + * 处理成绩字段，支持等级制和百分制 + */ + private String processScore(Map data) { + String score = getStringValue(data, "CJ"); + + if (score == null || score.isEmpty()) { + // 如果成绩为空，尝试使用补考成绩 + return getStringValue(data, "BkScore"); } - if (a.get("XH")!=null) { - b.setStuNo(a.get("XH").toString()); + + // 处理等级制成绩 + switch (score) { + case "不合格": + return "45.00"; + case "合格": + return "65.00"; + case "中等": + return "75.00"; + case "良好": + return "85.00"; + case "优秀": + return "95.00"; + case "0.00": + case "": + case "null": + case "NULL": + // 空成绩时尝试使用补考成绩 + String bkScore = getStringValue(data, "BkScore"); + return bkScore != null ? bkScore : score; + default: + return score; } - if (a.get("KCDM")!=null) { - b.setKcdm(a.get("KCDM").toString()); - } - if (a.get("XSHDXF")!=null) { - b.setXf(new BigDecimal(a.get("XSHDXF").toString())); - } - if (a.get("XSHDJD")!=null) { - b.setJd(new BigDecimal(a.get("XSHDJD").toString())); - } - if (a.get("FZLX")!=null) { - b.setFzlx(Long.parseLong(a.get("FZLX").toString())); - } - if (a.get("Passed")!=null) { - b.setIsPass(a.get("Passed").toString()); - } - return b; } //同步学生成绩，先清空在添加 @@ -277,11 +328,58 @@ public class CphStuScoreMiddlesServiceImpl implements ICphStuScoreMiddlesService @Override @DataSource(DataSourceType.MASTER)//mysql public void selectCphStuScoreMiddleEmptyAndAdd(List> cphStuScoreMiddles) { - List cphStuScoreMiddleDto = cphStuScoreMiddles.stream() - .map(m -> convertToB(m)).collect(Collectors.toList()); - if (cphStuScoreMiddleDto.size() != 0) { - cphStuScoreMiddleDtoMapper.insertBatchSomeColumn(cphStuScoreMiddleDto);//添加 + if (cphStuScoreMiddles == null || cphStuScoreMiddles.isEmpty()) { + return; } - + + try { + // 过滤无效数据 + List> validData = cphStuScoreMiddles.stream() + .filter(this::isValidData) + .collect(Collectors.toList()); + + if (validData.isEmpty()) { + return; + } + + // 批量转换数据 + List cphStuScoreMiddleDto = validData.parallelStream() + .map(this::convertToB) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + if (!cphStuScoreMiddleDto.isEmpty()) { + // 分批插入，避免单次插入数据量过大 + List> batches = ListSliceUtil.insertSlice(cphStuScoreMiddleDto, syncConfig.getBatchInsertSize()); + if (batches != null) { + for (List batch : batches) { + if (batch != null && !batch.isEmpty()) { + cphStuScoreMiddleDtoMapper.insertBatchSomeColumn(batch); + } + } + } + } + + } catch (Exception e) { + // 记录错误但不抛出异常，避免影响其他批次 + System.err.println("处理学生成绩数据时发生错误: " + e.getMessage()); + e.printStackTrace(); + } + } + + /** + * 验证数据有效性 + */ + private boolean isValidData(Map data) { + if (data == null) { + return false; + } + + // 检查必要字段 + Object stuNo = data.get("XH"); + Object kcdm = data.get("KCDM"); + + return stuNo != null && !stuNo.toString().trim().isEmpty() && + kcdm != null && !kcdm.toString().trim().isEmpty(); } } diff --git a/srs-comprehensive/src/main/java/com/srs/comprehensive/util/ListSliceUtil.java b/srs-comprehensive/src/main/java/com/srs/comprehensive/util/ListSliceUtil.java index b9a1794..6aff5db 100644 --- a/srs-comprehensive/src/main/java/com/srs/comprehensive/util/ListSliceUtil.java +++ b/srs-comprehensive/src/main/java/com/srs/comprehensive/util/ListSliceUtil.java @@ -10,29 +10,64 @@ import java.util.List; */ public class ListSliceUtil { public static List> updateSlice(List list){ - if (list.size()==0){ + return sliceList(list, 1000); + } + + /** + * 插入数据时切割列表，使用较大的批次大小 + */ + public static List> insertSlice(List list){ + return sliceList(list, 1000); + } + + /** + * 插入数据时切割列表，支持自定义批次大小 + * @param list 要切割的列表 + * @param batchSize 每批的大小 + * @return 切割后的列表集合 + */ + public static List> insertSlice(List list, int batchSize){ + return sliceList(list, batchSize); + } + + /** + * 通用列表切割方法 + * @param list 要切割的列表 + * @param batchSize 每批的大小 + * @return 切割后的列表集合 + */ + private static List> sliceList(List list, int batchSize){ + if (list == null || list.isEmpty()){ return null; } - int userListSize=list.size(); - int sum=userListSize/1000; - int sums=userListSize%1000; - if (sums!=0){ - sum++; + + int listSize = list.size(); + int batchCount = listSize / batchSize; + int remainder = listSize % batchSize; + + if (remainder != 0){ + batchCount++; } - List> listList=new ArrayList<>(); - int sumIm=0; - for (int i=0;i objects = new ArrayList<>(); - int listSum=1000; - if (sums!=0&&i==sum-1){ - listSum=sums; + + List> result = new ArrayList<>(); + int currentIndex = 0; + + for (int i = 0; i < batchCount; i++){ + List batch = new ArrayList<>(); + int currentBatchSize = batchSize; + + // 最后一批可能不满 + if (remainder != 0 && i == batchCount - 1){ + currentBatchSize = remainder; } - for (int j=0;j0? sum + 1: sum; - QueryWrapper objectQueryWrapper = new QueryWrapper<>(); - objectQueryWrapper.orderByDesc("stu_no", "kcdm"); - ExecutorService executor = Executors.newFixedThreadPool(15); - for (pageNum=1; pageNum <= sumS; pageNum++){ - final int currentPage = pageNum; - executor.execute(() -> { - List> cphStuScoreMiddles = cphStuScoreMiddlesService - .serectCphStuScoreMiddlesXh(currentPage, pageSize, null); - cphStuScoreMiddlesService.selectCphStuScoreMiddleEmptyAndAdd(cphStuScoreMiddles); - cphStuScoreMiddles.clear(); - }); + try { + System.out.println("开始同步学生成绩数据..."); + long startTime = System.currentTimeMillis(); + + // 清空现有数据 + cphStuScoreMiddlesService.emptyTableDate(); + System.out.println("已清空现有数据"); + + // 配置参数 + int pageSize = 3000; + int threadPoolSize = 8; // 定时任务使用较少线程，避免影响其他任务 + + // 获取总数 + int studentInfoNumber = cphStuScoreMiddlesService.serectCphStuScoreMiddleCount(null); + int totalPages = (int) Math.ceil((double) studentInfoNumber / pageSize); + + System.out.println("总记录数: " + studentInfoNumber + ", 总页数: " + totalPages); + + // 创建线程池 + ExecutorService executor = Executors.newFixedThreadPool(threadPoolSize); + CountDownLatch latch = new CountDownLatch(totalPages); + AtomicInteger successCount = new AtomicInteger(0); + AtomicInteger errorCount = new AtomicInteger(0); + + // 提交任务 + for (int pageNum = 1; pageNum <= totalPages; pageNum++) { + final int currentPage = pageNum; + executor.submit(() -> { + try { + List> cphStuScoreMiddles = cphStuScoreMiddlesService + .serectCphStuScoreMiddlesXh(currentPage, pageSize, null); + + if (cphStuScoreMiddles != null && !cphStuScoreMiddles.isEmpty()) { + cphStuScoreMiddlesService.selectCphStuScoreMiddleEmptyAndAdd(cphStuScoreMiddles); + successCount.incrementAndGet(); + } + + } catch (Exception e) { + errorCount.incrementAndGet(); + System.err.println("处理第" + currentPage + "页时发生错误: " + e.getMessage()); + } finally { + latch.countDown(); + } + }); + } + + // 等待所有任务完成 + boolean completed = latch.await(30, TimeUnit.MINUTES); + + if (!completed) { + System.err.println("同步任务未在指定时间内完成"); + return; + } + + // 关闭线程池 + executor.shutdown(); + + long endTime = System.currentTimeMillis(); + long duration = endTime - startTime; + + System.out.println("学生成绩同步完成 - 总耗时: " + duration + "ms, 成功页数: " + + successCount.get() + ", 失败页数: " + errorCount.get()); + + } catch (Exception e) { + System.err.println("同步学生成绩数据时发生严重错误: " + e.getMessage()); + e.printStackTrace(); } - executor.shutdown(); - //cphStuScoreMiddlesService.syncCphStuScoreMiddles(); } /**