xgxt_sd/zhxg_java
15
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化同步成绩

Browse Source
This commit is contained in:
zhy
2025-10-11 10:56:23 +08:00
parent c911be9fbc
commit 818a6f9911
6 changed files with 457 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
srs-admin/src/main/java/com/srs/web/controller/comprehensive/CphStuScoreMiddleController.java
View File

@@ -9,6 +9,7 @@ import com.srs.comprehensive.domain.CphStuScoreMiddle;
import com.srs.comprehensive.domain.Dto.CphStuScoreMiddleDto;
import com.srs.comprehensive.domain.Vo.CphStuScoreMiddleDtoVo;
import com.srs.comprehensive.service.ICphStuScoreMiddlesService;
import com.srs.comprehensive.config.SyncConfig;
import com.sun.net.httpserver.Authenticator;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
@@ -21,6 +22,9 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static com.srs.common.core.domain.AjaxResult.success;
@@ -34,6 +38,9 @@ public class CphStuScoreMiddleController extends BaseController {
@Autowired
public ICphStuScoreMiddlesService cphStuScoreMiddlesService;
@Autowired
private SyncConfig syncConfig;
//同步学生成绩
@GetMapping("/selectAllw")
@@ -57,28 +64,85 @@ public class CphStuScoreMiddleController extends BaseController {
}
//同步学生成绩，清空在添加
@GetMapping("/selectAll")//16：03
@GetMapping("/selectAll")
public AjaxResult selectCphStuScoreMiddleEmptyAndAdd(){
cphStuScoreMiddlesService.emptyTableDate();
int pageNum=1;
int pageSize=3000;
int studentInfoNumber = cphStuScoreMiddlesService.serectCphStuScoreMiddleCount(null);//总数
int sum=studentInfoNumber/ pageSize;//需要循环的次数
int sumS=studentInfoNumber%pageSize>0? sum + 1: sum;
QueryWrapper<CphStuScoreMiddleDto> objectQueryWrapper = new QueryWrapper<>();
objectQueryWrapper.orderByDesc("stu_no", "kcdm");
ExecutorService executor = Executors.newFixedThreadPool(15);
for (pageNum=1; pageNum <= sumS; pageNum++){
final int currentPage = pageNum;
executor.execute(() -> {
List<Map<String, Object>> cphStuScoreMiddles = cphStuScoreMiddlesService
.serectCphStuScoreMiddlesXh(currentPage, pageSize, null);
cphStuScoreMiddlesService.selectCphStuScoreMiddleEmptyAndAdd(cphStuScoreMiddles);
cphStuScoreMiddles.clear();
});
try {
logger.info("开始同步学生成绩数据...");
long startTime = System.currentTimeMillis();
// 清空现有数据
cphStuScoreMiddlesService.emptyTableDate();
logger.info("已清空现有数据");
// 配置参数
int pageSize = syncConfig.getPageSize();
int threadPoolSize = syncConfig.getWebThreadPoolSize();
// 获取总数
int studentInfoNumber = cphStuScoreMiddlesService.serectCphStuScoreMiddleCount(null);
int totalPages = (int) Math.ceil((double) studentInfoNumber / pageSize);
logger.info("总记录数: {}, 总页数: {}, 每页大小: {}", studentInfoNumber, totalPages, pageSize);
// 创建线程池
ExecutorService executor = Executors.newFixedThreadPool(threadPoolSize);
CountDownLatch latch = new CountDownLatch(totalPages);
AtomicInteger successCount = new AtomicInteger(0);
AtomicInteger errorCount = new AtomicInteger(0);
// 提交任务
for (int pageNum = 1; pageNum <= totalPages; pageNum++) {
final int currentPage = pageNum;
executor.submit(() -> {
try {
// 分页查询数据
List<Map<String, Object>> cphStuScoreMiddles = cphStuScoreMiddlesService
.serectCphStuScoreMiddlesXh(currentPage, pageSize, null);
if (cphStuScoreMiddles != null && !cphStuScoreMiddles.isEmpty()) {
// 处理数据
cphStuScoreMiddlesService.selectCphStuScoreMiddleEmptyAndAdd(cphStuScoreMiddles);
successCount.incrementAndGet();
logger.debug("第{}页处理完成，记录数: {}", currentPage, cphStuScoreMiddles.size());
}
} catch (Exception e) {
errorCount.incrementAndGet();
logger.error("处理第{}页时发生错误: {}", currentPage, e.getMessage(), e);
} finally {
latch.countDown();
}
});
}
// 等待所有任务完成
boolean completed = latch.await(syncConfig.getTimeoutMinutes(), TimeUnit.MINUTES);
if (!completed) {
logger.warn("同步任务未在指定时间内完成");
return AjaxResult.error("同步任务超时");
}
// 关闭线程池
executor.shutdown();
long endTime = System.currentTimeMillis();
long duration = endTime - startTime;
logger.info("学生成绩同步完成 - 总耗时: {}ms, 成功页数: {}, 失败页数: {}",
duration, successCount.get(), errorCount.get());
if (errorCount.get() > 0) {
return AjaxResult.warn("同步完成，但有部分数据失败。成功: " + successCount.get() +
"，失败: " + errorCount.get());
}
return AjaxResult.success("同步完成，共处理 " + totalPages + " 页数据，耗时 " + duration + "ms");
} catch (Exception e) {
logger.error("同步学生成绩数据时发生严重错误", e);
return AjaxResult.error("同步失败: " + e.getMessage());
}
executor.shutdown();
return success();
}
//查询sqlserver学生成绩

13
srs-admin/src/main/resources/application-sync.yml Normal file
View File

@@ -0,0 +1,13 @@
# 数据同步配置
sync:
student-score:
# 分页大小，建议根据内存和数据库性能调整
page-size: 3000
# Web接口线程池大小，建议不超过CPU核心数*2
web-thread-pool-size: 10
# 定时任务线程池大小，建议不超过CPU核心数
scheduled-thread-pool-size: 8
# 同步超时时间（分钟）
timeout-minutes: 300
# 批量插入大小，建议根据数据库性能调整（MySQL建议1000-2000）
batch-insert-size: 1000

78
srs-comprehensive/src/main/java/com/srs/comprehensive/config/SyncConfig.java Normal file
View File

@@ -0,0 +1,78 @@
package com.srs.comprehensive.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* 数据同步配置类
* 用于管理数据同步相关的参数配置
*/
@Component
@ConfigurationProperties(prefix = "sync.student-score")
public class SyncConfig {
/**
* 分页大小
*/
private int pageSize = 3000;
/**
* Web接口线程池大小
*/
private int webThreadPoolSize = 10;
/**
* 定时任务线程池大小
*/
private int scheduledThreadPoolSize = 8;
/**
* 同步超时时间（分钟）
*/
private int timeoutMinutes = 30;
/**
* 批量插入大小
*/
private int batchInsertSize = 1000;
public int getPageSize() {
return pageSize;
}
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
public int getWebThreadPoolSize() {
return webThreadPoolSize;
}
public void setWebThreadPoolSize(int webThreadPoolSize) {
this.webThreadPoolSize = webThreadPoolSize;
}
public int getScheduledThreadPoolSize() {
return scheduledThreadPoolSize;
}
public void setScheduledThreadPoolSize(int scheduledThreadPoolSize) {
this.scheduledThreadPoolSize = scheduledThreadPoolSize;
}
public int getTimeoutMinutes() {
return timeoutMinutes;
}
public void setTimeoutMinutes(int timeoutMinutes) {
this.timeoutMinutes = timeoutMinutes;
}
public int getBatchInsertSize() {
return batchInsertSize;
}
public void setBatchInsertSize(int batchInsertSize) {
this.batchInsertSize = batchInsertSize;
}
}

222
srs-comprehensive/src/main/java/com/srs/comprehensive/service/impl/CphStuScoreMiddlesServiceImpl.java
View File

@@ -12,6 +12,7 @@ import com.srs.comprehensive.mapper.CphStuScoreMiddleDtoMapper;
import com.srs.comprehensive.mapper.CphStuScoreMiddlesMapper;
import com.srs.comprehensive.service.ICphStuScoreMiddlesService;
import com.srs.comprehensive.util.ListSliceUtil;
import com.srs.comprehensive.config.SyncConfig;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -27,6 +28,9 @@ public class CphStuScoreMiddlesServiceImpl implements ICphStuScoreMiddlesService
@Autowired
private CphStuScoreMiddleDtoMapper cphStuScoreMiddleDtoMapper;
@Autowired
private SyncConfig syncConfig;
@Override
@DataSource(DataSourceType.DATABASE)//sqlserver
@@ -183,68 +187,115 @@ public class CphStuScoreMiddlesServiceImpl implements ICphStuScoreMiddlesService
//转换
private CphStuScoreMiddleDto convertToB(Map<String,Object> a) {
CphStuScoreMiddleDto b = new CphStuScoreMiddleDto();
// 进行相应的属性赋值
if (a.get("CJ")!=null) {
if (Objects.equals(a.get("CJ").toString(), "不合格")) {
b.setCj("45.00");
} else if (Objects.equals(a.get("CJ").toString(), "合格")) {
b.setCj("65.00");
} else if (Objects.equals(a.get("CJ").toString(), "中等")) {
b.setCj("75.00");
} else if (Objects.equals(a.get("CJ").toString(), "良好")) {
b.setCj("85.00");
} else if (Objects.equals(a.get("CJ").toString(), "优秀")) {
b.setCj("95.00");
} else if (Objects.equals(a.get("CJ").toString(), "0.00")||
Objects.equals(a.get("CJ").toString(), "")||
Objects.equals(a.get("CJ").toString(), null)||
Objects.equals(a.get("CJ").toString(), "NULL")||
Objects.equals(a.get("CJ").toString(), "null")) {
System.out.println(a.get("CJ"));
if (a.get("BkScore")!=null){
if (!Objects.equals(a.get("BkScore").toString(), "0.00") ||
!Objects.equals(a.get("BkScore").toString(), "")||
!Objects.equals(a.get("BkScore").toString(), "null")||
!Objects.equals(a.get("BkScore").toString(), "NULL")||
!Objects.equals(a.get("BkScore").toString(), null)){
b.setCj(a.get("BkScore").toString());
} else {
b.setCj(a.get("CJ").toString());
}
}else {
b.setCj(a.get("CJ").toString());
}
}else {
b.setCj(a.get("CJ").toString());
if (a == null) {
return null;
}
try {
CphStuScoreMiddleDto b = new CphStuScoreMiddleDto();
// 设置学号和课程代码（必要字段）
b.setStuNo(getStringValue(a, "XH"));
b.setKcdm(getStringValue(a, "KCDM"));
b.setXqid(1L);
// 处理成绩字段
String score = processScore(a);
b.setCj(score);
// 设置其他字段
b.setXndm(getStringValue(a, "XND"));
b.setXqdm(getStringValue(a, "XQMC"));
// 处理数值类型字段
setBigDecimalValue(b::setXf, a, "XSHDXF");
setBigDecimalValue(b::setJd, a, "XSHDJD");
setLongValue(b::setFzlx, a, "FZLX");
// 设置是否通过
b.setIsPass(getStringValue(a, "Passed"));
return b;
} catch (Exception e) {
System.err.println("转换数据时发生错误: " + e.getMessage());
return null;
}
}
/**
* 安全获取字符串值
*/
private String getStringValue(Map<String, Object> map, String key) {
Object value = map.get(key);
if (value == null) {
return null;
}
String strValue = value.toString().trim();
return strValue.isEmpty() || "null".equalsIgnoreCase(strValue) || "NULL".equals(strValue) ? null : strValue;
}
/**
* 安全设置BigDecimal值
*/
private void setBigDecimalValue(java.util.function.Consumer<BigDecimal> setter, Map<String, Object> map, String key) {
try {
String value = getStringValue(map, key);
if (value != null) {
setter.accept(new BigDecimal(value));
}
} catch (NumberFormatException e) {
// 忽略格式错误，保持null值
}
b.setXqid(1L);
if (a.get("XND") != null) {
b.setXndm(a.get("XND").toString());
}
/**
* 安全设置Long值
*/
private void setLongValue(java.util.function.Consumer<Long> setter, Map<String, Object> map, String key) {
try {
String value = getStringValue(map, key);
if (value != null) {
setter.accept(Long.parseLong(value));
}
} catch (NumberFormatException e) {
// 忽略格式错误，保持null值
}
if (a.get("XQMC")!=null) {
b.setXqdm(a.get("XQMC").toString());
}
/**
* 处理成绩字段，支持等级制和百分制
*/
private String processScore(Map<String, Object> data) {
String score = getStringValue(data, "CJ");
if (score == null || score.isEmpty()) {
// 如果成绩为空，尝试使用补考成绩
return getStringValue(data, "BkScore");
}
if (a.get("XH")!=null) {
b.setStuNo(a.get("XH").toString());
// 处理等级制成绩
switch (score) {
case "不合格":
return "45.00";
case "合格":
return "65.00";
case "中等":
return "75.00";
case "良好":
return "85.00";
case "优秀":
return "95.00";
case "0.00":
case "":
case "null":
case "NULL":
// 空成绩时尝试使用补考成绩
String bkScore = getStringValue(data, "BkScore");
return bkScore != null ? bkScore : score;
default:
return score;
}
if (a.get("KCDM")!=null) {
b.setKcdm(a.get("KCDM").toString());
}
if (a.get("XSHDXF")!=null) {
b.setXf(new BigDecimal(a.get("XSHDXF").toString()));
}
if (a.get("XSHDJD")!=null) {
b.setJd(new BigDecimal(a.get("XSHDJD").toString()));
}
if (a.get("FZLX")!=null) {
b.setFzlx(Long.parseLong(a.get("FZLX").toString()));
}
if (a.get("Passed")!=null) {
b.setIsPass(a.get("Passed").toString());
}
return b;
}
//同步学生成绩，先清空在添加
@@ -277,11 +328,58 @@ public class CphStuScoreMiddlesServiceImpl implements ICphStuScoreMiddlesService
@Override
@DataSource(DataSourceType.MASTER)//mysql
public void selectCphStuScoreMiddleEmptyAndAdd(List<Map<String,Object>> cphStuScoreMiddles) {
List<CphStuScoreMiddleDto> cphStuScoreMiddleDto = cphStuScoreMiddles.stream()
.map(m -> convertToB(m)).collect(Collectors.toList());
if (cphStuScoreMiddleDto.size() != 0) {
cphStuScoreMiddleDtoMapper.insertBatchSomeColumn(cphStuScoreMiddleDto);//添加
if (cphStuScoreMiddles == null || cphStuScoreMiddles.isEmpty()) {
return;
}
try {
// 过滤无效数据
List<Map<String, Object>> validData = cphStuScoreMiddles.stream()
.filter(this::isValidData)
.collect(Collectors.toList());
if (validData.isEmpty()) {
return;
}
// 批量转换数据
List<CphStuScoreMiddleDto> cphStuScoreMiddleDto = validData.parallelStream()
.map(this::convertToB)
.filter(Objects::nonNull)
.collect(Collectors.toList());
if (!cphStuScoreMiddleDto.isEmpty()) {
// 分批插入，避免单次插入数据量过大
List<List<CphStuScoreMiddleDto>> batches = ListSliceUtil.insertSlice(cphStuScoreMiddleDto, syncConfig.getBatchInsertSize());
if (batches != null) {
for (List<CphStuScoreMiddleDto> batch : batches) {
if (batch != null && !batch.isEmpty()) {
cphStuScoreMiddleDtoMapper.insertBatchSomeColumn(batch);
}
}
}
}
} catch (Exception e) {
// 记录错误但不抛出异常，避免影响其他批次
System.err.println("处理学生成绩数据时发生错误: " + e.getMessage());
e.printStackTrace();
}
}
/**
* 验证数据有效性
*/
private boolean isValidData(Map<String, Object> data) {
if (data == null) {
return false;
}
// 检查必要字段
Object stuNo = data.get("XH");
Object kcdm = data.get("KCDM");
return stuNo != null && !stuNo.toString().trim().isEmpty() &&
kcdm != null && !kcdm.toString().trim().isEmpty();
}
}

71
srs-comprehensive/src/main/java/com/srs/comprehensive/util/ListSliceUtil.java
View File

@@ -10,29 +10,64 @@ import java.util.List;
*/
public class ListSliceUtil {
public static <T> List<List<T>> updateSlice(List<T> list){
if (list.size()==0){
return sliceList(list, 1000);
}
/**
* 插入数据时切割列表，使用较大的批次大小
*/
public static <T> List<List<T>> insertSlice(List<T> list){
return sliceList(list, 1000);
}
/**
* 插入数据时切割列表，支持自定义批次大小
* @param list 要切割的列表
* @param batchSize 每批的大小
* @return 切割后的列表集合
*/
public static <T> List<List<T>> insertSlice(List<T> list, int batchSize){
return sliceList(list, batchSize);
}
/**
* 通用列表切割方法
* @param list 要切割的列表
* @param batchSize 每批的大小
* @return 切割后的列表集合
*/
private static <T> List<List<T>> sliceList(List<T> list, int batchSize){
if (list == null || list.isEmpty()){
return null;
}
int userListSize=list.size();
int sum=userListSize/1000;
int sums=userListSize%1000;
if (sums!=0){
sum++;
int listSize = list.size();
int batchCount = listSize / batchSize;
int remainder = listSize % batchSize;
if (remainder != 0){
batchCount++;
}
List<List<T>> listList=new ArrayList<>();
int sumIm=0;
for (int i=0;i<sum;i++){
List<T> objects = new ArrayList<>();
int listSum=1000;
if (sums!=0&&i==sum-1){
listSum=sums;
List<List<T>> result = new ArrayList<>();
int currentIndex = 0;
for (int i = 0; i < batchCount; i++){
List<T> batch = new ArrayList<>();
int currentBatchSize = batchSize;
// 最后一批可能不满
if (remainder != 0 && i == batchCount - 1){
currentBatchSize = remainder;
}
for (int j=0;j<listSum;j++){
objects.add(list.get(sumIm));
sumIm++;
for (int j = 0; j < currentBatchSize; j++){
batch.add(list.get(currentIndex));
currentIndex++;
}
listList.add(objects);
result.add(batch);
}
return listList;
return result;
}
}

88
srs-quartz/src/main/java/com/srs/quartz/task/CphTask.java
View File

@@ -12,6 +12,9 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@Component("CphTask")
public class CphTask {
@@ -56,26 +59,73 @@ public class CphTask {
* 同步学生成绩
*/
public void stuScoreMiddleSync(){
cphStuScoreMiddlesService.emptyTableDate();
int pageNum=1;
int pageSize=3000;
int studentInfoNumber = cphStuScoreMiddlesService.serectCphStuScoreMiddleCount(null);//总数
int sum=studentInfoNumber/ pageSize;//需要循环的次数
int sumS=studentInfoNumber%pageSize>0? sum + 1: sum;
QueryWrapper<CphStuScoreMiddleDto> objectQueryWrapper = new QueryWrapper<>();
objectQueryWrapper.orderByDesc("stu_no", "kcdm");
ExecutorService executor = Executors.newFixedThreadPool(15);
for (pageNum=1; pageNum <= sumS; pageNum++){
final int currentPage = pageNum;
executor.execute(() -> {
List<Map<String, Object>> cphStuScoreMiddles = cphStuScoreMiddlesService
.serectCphStuScoreMiddlesXh(currentPage, pageSize, null);
cphStuScoreMiddlesService.selectCphStuScoreMiddleEmptyAndAdd(cphStuScoreMiddles);
cphStuScoreMiddles.clear();
});
try {
System.out.println("开始同步学生成绩数据...");
long startTime = System.currentTimeMillis();
// 清空现有数据
cphStuScoreMiddlesService.emptyTableDate();
System.out.println("已清空现有数据");
// 配置参数
int pageSize = 3000;
int threadPoolSize = 8; // 定时任务使用较少线程，避免影响其他任务
// 获取总数
int studentInfoNumber = cphStuScoreMiddlesService.serectCphStuScoreMiddleCount(null);
int totalPages = (int) Math.ceil((double) studentInfoNumber / pageSize);
System.out.println("总记录数: " + studentInfoNumber + ", 总页数: " + totalPages);
// 创建线程池
ExecutorService executor = Executors.newFixedThreadPool(threadPoolSize);
CountDownLatch latch = new CountDownLatch(totalPages);
AtomicInteger successCount = new AtomicInteger(0);
AtomicInteger errorCount = new AtomicInteger(0);
// 提交任务
for (int pageNum = 1; pageNum <= totalPages; pageNum++) {
final int currentPage = pageNum;
executor.submit(() -> {
try {
List<Map<String, Object>> cphStuScoreMiddles = cphStuScoreMiddlesService
.serectCphStuScoreMiddlesXh(currentPage, pageSize, null);
if (cphStuScoreMiddles != null && !cphStuScoreMiddles.isEmpty()) {
cphStuScoreMiddlesService.selectCphStuScoreMiddleEmptyAndAdd(cphStuScoreMiddles);
successCount.incrementAndGet();
}
} catch (Exception e) {
errorCount.incrementAndGet();
System.err.println("处理第" + currentPage + "页时发生错误: " + e.getMessage());
} finally {
latch.countDown();
}
});
}
// 等待所有任务完成
boolean completed = latch.await(30, TimeUnit.MINUTES);
if (!completed) {
System.err.println("同步任务未在指定时间内完成");
return;
}
// 关闭线程池
executor.shutdown();
long endTime = System.currentTimeMillis();
long duration = endTime - startTime;
System.out.println("学生成绩同步完成 - 总耗时: " + duration + "ms, 成功页数: " +
successCount.get() + ", 失败页数: " + errorCount.get());
} catch (Exception e) {
System.err.println("同步学生成绩数据时发生严重错误: " + e.getMessage());
e.printStackTrace();
}
executor.shutdown();
//cphStuScoreMiddlesService.syncCphStuScoreMiddles();
}
/**