Merge branch 'main' of http://47.112.118.149:10082/xgxt_sd/zhxg_java
This commit is contained in:
@@ -9,6 +9,7 @@ import com.srs.comprehensive.domain.CphStuScoreMiddle;
|
|||||||
import com.srs.comprehensive.domain.Dto.CphStuScoreMiddleDto;
|
import com.srs.comprehensive.domain.Dto.CphStuScoreMiddleDto;
|
||||||
import com.srs.comprehensive.domain.Vo.CphStuScoreMiddleDtoVo;
|
import com.srs.comprehensive.domain.Vo.CphStuScoreMiddleDtoVo;
|
||||||
import com.srs.comprehensive.service.ICphStuScoreMiddlesService;
|
import com.srs.comprehensive.service.ICphStuScoreMiddlesService;
|
||||||
|
import com.srs.comprehensive.config.SyncConfig;
|
||||||
import com.sun.net.httpserver.Authenticator;
|
import com.sun.net.httpserver.Authenticator;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.scheduling.annotation.Scheduled;
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
@@ -21,6 +22,9 @@ import java.util.List;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.ExecutorService;
|
||||||
import java.util.concurrent.Executors;
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
import static com.srs.common.core.domain.AjaxResult.success;
|
import static com.srs.common.core.domain.AjaxResult.success;
|
||||||
|
|
||||||
@@ -34,6 +38,9 @@ public class CphStuScoreMiddleController extends BaseController {
|
|||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
public ICphStuScoreMiddlesService cphStuScoreMiddlesService;
|
public ICphStuScoreMiddlesService cphStuScoreMiddlesService;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private SyncConfig syncConfig;
|
||||||
|
|
||||||
//同步学生成绩
|
//同步学生成绩
|
||||||
@GetMapping("/selectAllw")
|
@GetMapping("/selectAllw")
|
||||||
@@ -57,28 +64,85 @@ public class CphStuScoreMiddleController extends BaseController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
//同步学生成绩,清空在添加
|
//同步学生成绩,清空在添加
|
||||||
@GetMapping("/selectAll")//16:03
|
@GetMapping("/selectAll")
|
||||||
public AjaxResult selectCphStuScoreMiddleEmptyAndAdd(){
|
public AjaxResult selectCphStuScoreMiddleEmptyAndAdd(){
|
||||||
cphStuScoreMiddlesService.emptyTableDate();
|
try {
|
||||||
int pageNum=1;
|
logger.info("开始同步学生成绩数据...");
|
||||||
int pageSize=3000;
|
long startTime = System.currentTimeMillis();
|
||||||
int studentInfoNumber = cphStuScoreMiddlesService.serectCphStuScoreMiddleCount(null);//总数
|
|
||||||
int sum=studentInfoNumber/ pageSize;//需要循环的次数
|
// 清空现有数据
|
||||||
int sumS=studentInfoNumber%pageSize>0? sum + 1: sum;
|
cphStuScoreMiddlesService.emptyTableDate();
|
||||||
QueryWrapper<CphStuScoreMiddleDto> objectQueryWrapper = new QueryWrapper<>();
|
logger.info("已清空现有数据");
|
||||||
objectQueryWrapper.orderByDesc("stu_no", "kcdm");
|
|
||||||
ExecutorService executor = Executors.newFixedThreadPool(15);
|
// 配置参数
|
||||||
for (pageNum=1; pageNum <= sumS; pageNum++){
|
int pageSize = syncConfig.getPageSize();
|
||||||
final int currentPage = pageNum;
|
int threadPoolSize = syncConfig.getWebThreadPoolSize();
|
||||||
executor.execute(() -> {
|
|
||||||
List<Map<String, Object>> cphStuScoreMiddles = cphStuScoreMiddlesService
|
// 获取总数
|
||||||
.serectCphStuScoreMiddlesXh(currentPage, pageSize, null);
|
int studentInfoNumber = cphStuScoreMiddlesService.serectCphStuScoreMiddleCount(null);
|
||||||
cphStuScoreMiddlesService.selectCphStuScoreMiddleEmptyAndAdd(cphStuScoreMiddles);
|
int totalPages = (int) Math.ceil((double) studentInfoNumber / pageSize);
|
||||||
cphStuScoreMiddles.clear();
|
|
||||||
});
|
logger.info("总记录数: {}, 总页数: {}, 每页大小: {}", studentInfoNumber, totalPages, pageSize);
|
||||||
|
|
||||||
|
// 创建线程池
|
||||||
|
ExecutorService executor = Executors.newFixedThreadPool(threadPoolSize);
|
||||||
|
CountDownLatch latch = new CountDownLatch(totalPages);
|
||||||
|
AtomicInteger successCount = new AtomicInteger(0);
|
||||||
|
AtomicInteger errorCount = new AtomicInteger(0);
|
||||||
|
|
||||||
|
// 提交任务
|
||||||
|
for (int pageNum = 1; pageNum <= totalPages; pageNum++) {
|
||||||
|
final int currentPage = pageNum;
|
||||||
|
executor.submit(() -> {
|
||||||
|
try {
|
||||||
|
// 分页查询数据
|
||||||
|
List<Map<String, Object>> cphStuScoreMiddles = cphStuScoreMiddlesService
|
||||||
|
.serectCphStuScoreMiddlesXh(currentPage, pageSize, null);
|
||||||
|
|
||||||
|
if (cphStuScoreMiddles != null && !cphStuScoreMiddles.isEmpty()) {
|
||||||
|
// 处理数据
|
||||||
|
cphStuScoreMiddlesService.selectCphStuScoreMiddleEmptyAndAdd(cphStuScoreMiddles);
|
||||||
|
successCount.incrementAndGet();
|
||||||
|
logger.debug("第{}页处理完成,记录数: {}", currentPage, cphStuScoreMiddles.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
errorCount.incrementAndGet();
|
||||||
|
logger.error("处理第{}页时发生错误: {}", currentPage, e.getMessage(), e);
|
||||||
|
} finally {
|
||||||
|
latch.countDown();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// 等待所有任务完成
|
||||||
|
boolean completed = latch.await(syncConfig.getTimeoutMinutes(), TimeUnit.MINUTES);
|
||||||
|
|
||||||
|
if (!completed) {
|
||||||
|
logger.warn("同步任务未在指定时间内完成");
|
||||||
|
return AjaxResult.error("同步任务超时");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 关闭线程池
|
||||||
|
executor.shutdown();
|
||||||
|
|
||||||
|
long endTime = System.currentTimeMillis();
|
||||||
|
long duration = endTime - startTime;
|
||||||
|
|
||||||
|
logger.info("学生成绩同步完成 - 总耗时: {}ms, 成功页数: {}, 失败页数: {}",
|
||||||
|
duration, successCount.get(), errorCount.get());
|
||||||
|
|
||||||
|
if (errorCount.get() > 0) {
|
||||||
|
return AjaxResult.warn("同步完成,但有部分数据失败。成功: " + successCount.get() +
|
||||||
|
",失败: " + errorCount.get());
|
||||||
|
}
|
||||||
|
|
||||||
|
return AjaxResult.success("同步完成,共处理 " + totalPages + " 页数据,耗时 " + duration + "ms");
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("同步学生成绩数据时发生严重错误", e);
|
||||||
|
return AjaxResult.error("同步失败: " + e.getMessage());
|
||||||
}
|
}
|
||||||
executor.shutdown();
|
|
||||||
return success();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//查询sqlserver学生成绩
|
//查询sqlserver学生成绩
|
||||||
|
|||||||
13
srs-admin/src/main/resources/application-sync.yml
Normal file
13
srs-admin/src/main/resources/application-sync.yml
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
# 数据同步配置
|
||||||
|
sync:
|
||||||
|
student-score:
|
||||||
|
# 分页大小,建议根据内存和数据库性能调整
|
||||||
|
page-size: 3000
|
||||||
|
# Web接口线程池大小,建议不超过CPU核心数*2
|
||||||
|
web-thread-pool-size: 10
|
||||||
|
# 定时任务线程池大小,建议不超过CPU核心数
|
||||||
|
scheduled-thread-pool-size: 8
|
||||||
|
# 同步超时时间(分钟)
|
||||||
|
timeout-minutes: 300
|
||||||
|
# 批量插入大小,建议根据数据库性能调整(MySQL建议1000-2000)
|
||||||
|
batch-insert-size: 1000
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package com.srs.comprehensive.config;
|
||||||
|
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 数据同步配置类
|
||||||
|
* 用于管理数据同步相关的参数配置
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
@ConfigurationProperties(prefix = "sync.student-score")
|
||||||
|
public class SyncConfig {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分页大小
|
||||||
|
*/
|
||||||
|
private int pageSize = 3000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Web接口线程池大小
|
||||||
|
*/
|
||||||
|
private int webThreadPoolSize = 10;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 定时任务线程池大小
|
||||||
|
*/
|
||||||
|
private int scheduledThreadPoolSize = 8;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 同步超时时间(分钟)
|
||||||
|
*/
|
||||||
|
private int timeoutMinutes = 30;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 批量插入大小
|
||||||
|
*/
|
||||||
|
private int batchInsertSize = 1000;
|
||||||
|
|
||||||
|
public int getPageSize() {
|
||||||
|
return pageSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPageSize(int pageSize) {
|
||||||
|
this.pageSize = pageSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getWebThreadPoolSize() {
|
||||||
|
return webThreadPoolSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setWebThreadPoolSize(int webThreadPoolSize) {
|
||||||
|
this.webThreadPoolSize = webThreadPoolSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getScheduledThreadPoolSize() {
|
||||||
|
return scheduledThreadPoolSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setScheduledThreadPoolSize(int scheduledThreadPoolSize) {
|
||||||
|
this.scheduledThreadPoolSize = scheduledThreadPoolSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getTimeoutMinutes() {
|
||||||
|
return timeoutMinutes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTimeoutMinutes(int timeoutMinutes) {
|
||||||
|
this.timeoutMinutes = timeoutMinutes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getBatchInsertSize() {
|
||||||
|
return batchInsertSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setBatchInsertSize(int batchInsertSize) {
|
||||||
|
this.batchInsertSize = batchInsertSize;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,6 +12,7 @@ import com.srs.comprehensive.mapper.CphStuScoreMiddleDtoMapper;
|
|||||||
import com.srs.comprehensive.mapper.CphStuScoreMiddlesMapper;
|
import com.srs.comprehensive.mapper.CphStuScoreMiddlesMapper;
|
||||||
import com.srs.comprehensive.service.ICphStuScoreMiddlesService;
|
import com.srs.comprehensive.service.ICphStuScoreMiddlesService;
|
||||||
import com.srs.comprehensive.util.ListSliceUtil;
|
import com.srs.comprehensive.util.ListSliceUtil;
|
||||||
|
import com.srs.comprehensive.config.SyncConfig;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -27,6 +28,9 @@ public class CphStuScoreMiddlesServiceImpl implements ICphStuScoreMiddlesService
|
|||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private CphStuScoreMiddleDtoMapper cphStuScoreMiddleDtoMapper;
|
private CphStuScoreMiddleDtoMapper cphStuScoreMiddleDtoMapper;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private SyncConfig syncConfig;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@DataSource(DataSourceType.DATABASE)//sqlserver
|
@DataSource(DataSourceType.DATABASE)//sqlserver
|
||||||
@@ -183,68 +187,115 @@ public class CphStuScoreMiddlesServiceImpl implements ICphStuScoreMiddlesService
|
|||||||
|
|
||||||
//转换
|
//转换
|
||||||
private CphStuScoreMiddleDto convertToB(Map<String,Object> a) {
|
private CphStuScoreMiddleDto convertToB(Map<String,Object> a) {
|
||||||
CphStuScoreMiddleDto b = new CphStuScoreMiddleDto();
|
if (a == null) {
|
||||||
// 进行相应的属性赋值
|
return null;
|
||||||
if (a.get("CJ")!=null) {
|
}
|
||||||
if (Objects.equals(a.get("CJ").toString(), "不合格")) {
|
|
||||||
b.setCj("45.00");
|
try {
|
||||||
} else if (Objects.equals(a.get("CJ").toString(), "合格")) {
|
CphStuScoreMiddleDto b = new CphStuScoreMiddleDto();
|
||||||
b.setCj("65.00");
|
|
||||||
} else if (Objects.equals(a.get("CJ").toString(), "中等")) {
|
// 设置学号和课程代码(必要字段)
|
||||||
b.setCj("75.00");
|
b.setStuNo(getStringValue(a, "XH"));
|
||||||
} else if (Objects.equals(a.get("CJ").toString(), "良好")) {
|
b.setKcdm(getStringValue(a, "KCDM"));
|
||||||
b.setCj("85.00");
|
b.setXqid(1L);
|
||||||
} else if (Objects.equals(a.get("CJ").toString(), "优秀")) {
|
|
||||||
b.setCj("95.00");
|
// 处理成绩字段
|
||||||
} else if (Objects.equals(a.get("CJ").toString(), "0.00")||
|
String score = processScore(a);
|
||||||
Objects.equals(a.get("CJ").toString(), "")||
|
b.setCj(score);
|
||||||
Objects.equals(a.get("CJ").toString(), null)||
|
|
||||||
Objects.equals(a.get("CJ").toString(), "NULL")||
|
// 设置其他字段
|
||||||
Objects.equals(a.get("CJ").toString(), "null")) {
|
b.setXndm(getStringValue(a, "XND"));
|
||||||
System.out.println(a.get("CJ"));
|
b.setXqdm(getStringValue(a, "XQMC"));
|
||||||
if (a.get("BkScore")!=null){
|
|
||||||
if (!Objects.equals(a.get("BkScore").toString(), "0.00") ||
|
// 处理数值类型字段
|
||||||
!Objects.equals(a.get("BkScore").toString(), "")||
|
setBigDecimalValue(b::setXf, a, "XSHDXF");
|
||||||
!Objects.equals(a.get("BkScore").toString(), "null")||
|
setBigDecimalValue(b::setJd, a, "XSHDJD");
|
||||||
!Objects.equals(a.get("BkScore").toString(), "NULL")||
|
setLongValue(b::setFzlx, a, "FZLX");
|
||||||
!Objects.equals(a.get("BkScore").toString(), null)){
|
|
||||||
b.setCj(a.get("BkScore").toString());
|
// 设置是否通过
|
||||||
} else {
|
b.setIsPass(getStringValue(a, "Passed"));
|
||||||
b.setCj(a.get("CJ").toString());
|
|
||||||
}
|
return b;
|
||||||
}else {
|
|
||||||
b.setCj(a.get("CJ").toString());
|
} catch (Exception e) {
|
||||||
}
|
System.err.println("转换数据时发生错误: " + e.getMessage());
|
||||||
}else {
|
return null;
|
||||||
b.setCj(a.get("CJ").toString());
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 安全获取字符串值
|
||||||
|
*/
|
||||||
|
private String getStringValue(Map<String, Object> map, String key) {
|
||||||
|
Object value = map.get(key);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String strValue = value.toString().trim();
|
||||||
|
return strValue.isEmpty() || "null".equalsIgnoreCase(strValue) || "NULL".equals(strValue) ? null : strValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 安全设置BigDecimal值
|
||||||
|
*/
|
||||||
|
private void setBigDecimalValue(java.util.function.Consumer<BigDecimal> setter, Map<String, Object> map, String key) {
|
||||||
|
try {
|
||||||
|
String value = getStringValue(map, key);
|
||||||
|
if (value != null) {
|
||||||
|
setter.accept(new BigDecimal(value));
|
||||||
}
|
}
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
// 忽略格式错误,保持null值
|
||||||
}
|
}
|
||||||
b.setXqid(1L);
|
}
|
||||||
if (a.get("XND") != null) {
|
|
||||||
b.setXndm(a.get("XND").toString());
|
/**
|
||||||
|
* 安全设置Long值
|
||||||
|
*/
|
||||||
|
private void setLongValue(java.util.function.Consumer<Long> setter, Map<String, Object> map, String key) {
|
||||||
|
try {
|
||||||
|
String value = getStringValue(map, key);
|
||||||
|
if (value != null) {
|
||||||
|
setter.accept(Long.parseLong(value));
|
||||||
|
}
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
// 忽略格式错误,保持null值
|
||||||
}
|
}
|
||||||
if (a.get("XQMC")!=null) {
|
}
|
||||||
b.setXqdm(a.get("XQMC").toString());
|
|
||||||
|
/**
|
||||||
|
* 处理成绩字段,支持等级制和百分制
|
||||||
|
*/
|
||||||
|
private String processScore(Map<String, Object> data) {
|
||||||
|
String score = getStringValue(data, "CJ");
|
||||||
|
|
||||||
|
if (score == null || score.isEmpty()) {
|
||||||
|
// 如果成绩为空,尝试使用补考成绩
|
||||||
|
return getStringValue(data, "BkScore");
|
||||||
}
|
}
|
||||||
if (a.get("XH")!=null) {
|
|
||||||
b.setStuNo(a.get("XH").toString());
|
// 处理等级制成绩
|
||||||
|
switch (score) {
|
||||||
|
case "不合格":
|
||||||
|
return "45.00";
|
||||||
|
case "合格":
|
||||||
|
return "65.00";
|
||||||
|
case "中等":
|
||||||
|
return "75.00";
|
||||||
|
case "良好":
|
||||||
|
return "85.00";
|
||||||
|
case "优秀":
|
||||||
|
return "95.00";
|
||||||
|
case "0.00":
|
||||||
|
case "":
|
||||||
|
case "null":
|
||||||
|
case "NULL":
|
||||||
|
// 空成绩时尝试使用补考成绩
|
||||||
|
String bkScore = getStringValue(data, "BkScore");
|
||||||
|
return bkScore != null ? bkScore : score;
|
||||||
|
default:
|
||||||
|
return score;
|
||||||
}
|
}
|
||||||
if (a.get("KCDM")!=null) {
|
|
||||||
b.setKcdm(a.get("KCDM").toString());
|
|
||||||
}
|
|
||||||
if (a.get("XSHDXF")!=null) {
|
|
||||||
b.setXf(new BigDecimal(a.get("XSHDXF").toString()));
|
|
||||||
}
|
|
||||||
if (a.get("XSHDJD")!=null) {
|
|
||||||
b.setJd(new BigDecimal(a.get("XSHDJD").toString()));
|
|
||||||
}
|
|
||||||
if (a.get("FZLX")!=null) {
|
|
||||||
b.setFzlx(Long.parseLong(a.get("FZLX").toString()));
|
|
||||||
}
|
|
||||||
if (a.get("Passed")!=null) {
|
|
||||||
b.setIsPass(a.get("Passed").toString());
|
|
||||||
}
|
|
||||||
return b;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//同步学生成绩,先清空在添加
|
//同步学生成绩,先清空在添加
|
||||||
@@ -277,11 +328,58 @@ public class CphStuScoreMiddlesServiceImpl implements ICphStuScoreMiddlesService
|
|||||||
@Override
|
@Override
|
||||||
@DataSource(DataSourceType.MASTER)//mysql
|
@DataSource(DataSourceType.MASTER)//mysql
|
||||||
public void selectCphStuScoreMiddleEmptyAndAdd(List<Map<String,Object>> cphStuScoreMiddles) {
|
public void selectCphStuScoreMiddleEmptyAndAdd(List<Map<String,Object>> cphStuScoreMiddles) {
|
||||||
List<CphStuScoreMiddleDto> cphStuScoreMiddleDto = cphStuScoreMiddles.stream()
|
if (cphStuScoreMiddles == null || cphStuScoreMiddles.isEmpty()) {
|
||||||
.map(m -> convertToB(m)).collect(Collectors.toList());
|
return;
|
||||||
if (cphStuScoreMiddleDto.size() != 0) {
|
|
||||||
cphStuScoreMiddleDtoMapper.insertBatchSomeColumn(cphStuScoreMiddleDto);//添加
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// 过滤无效数据
|
||||||
|
List<Map<String, Object>> validData = cphStuScoreMiddles.stream()
|
||||||
|
.filter(this::isValidData)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
if (validData.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 批量转换数据
|
||||||
|
List<CphStuScoreMiddleDto> cphStuScoreMiddleDto = validData.parallelStream()
|
||||||
|
.map(this::convertToB)
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
if (!cphStuScoreMiddleDto.isEmpty()) {
|
||||||
|
// 分批插入,避免单次插入数据量过大
|
||||||
|
List<List<CphStuScoreMiddleDto>> batches = ListSliceUtil.insertSlice(cphStuScoreMiddleDto, syncConfig.getBatchInsertSize());
|
||||||
|
if (batches != null) {
|
||||||
|
for (List<CphStuScoreMiddleDto> batch : batches) {
|
||||||
|
if (batch != null && !batch.isEmpty()) {
|
||||||
|
cphStuScoreMiddleDtoMapper.insertBatchSomeColumn(batch);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
// 记录错误但不抛出异常,避免影响其他批次
|
||||||
|
System.err.println("处理学生成绩数据时发生错误: " + e.getMessage());
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证数据有效性
|
||||||
|
*/
|
||||||
|
private boolean isValidData(Map<String, Object> data) {
|
||||||
|
if (data == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 检查必要字段
|
||||||
|
Object stuNo = data.get("XH");
|
||||||
|
Object kcdm = data.get("KCDM");
|
||||||
|
|
||||||
|
return stuNo != null && !stuNo.toString().trim().isEmpty() &&
|
||||||
|
kcdm != null && !kcdm.toString().trim().isEmpty();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,29 +10,64 @@ import java.util.List;
|
|||||||
*/
|
*/
|
||||||
public class ListSliceUtil {
|
public class ListSliceUtil {
|
||||||
public static <T> List<List<T>> updateSlice(List<T> list){
|
public static <T> List<List<T>> updateSlice(List<T> list){
|
||||||
if (list.size()==0){
|
return sliceList(list, 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 插入数据时切割列表,使用较大的批次大小
|
||||||
|
*/
|
||||||
|
public static <T> List<List<T>> insertSlice(List<T> list){
|
||||||
|
return sliceList(list, 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 插入数据时切割列表,支持自定义批次大小
|
||||||
|
* @param list 要切割的列表
|
||||||
|
* @param batchSize 每批的大小
|
||||||
|
* @return 切割后的列表集合
|
||||||
|
*/
|
||||||
|
public static <T> List<List<T>> insertSlice(List<T> list, int batchSize){
|
||||||
|
return sliceList(list, batchSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 通用列表切割方法
|
||||||
|
* @param list 要切割的列表
|
||||||
|
* @param batchSize 每批的大小
|
||||||
|
* @return 切割后的列表集合
|
||||||
|
*/
|
||||||
|
private static <T> List<List<T>> sliceList(List<T> list, int batchSize){
|
||||||
|
if (list == null || list.isEmpty()){
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
int userListSize=list.size();
|
|
||||||
int sum=userListSize/1000;
|
int listSize = list.size();
|
||||||
int sums=userListSize%1000;
|
int batchCount = listSize / batchSize;
|
||||||
if (sums!=0){
|
int remainder = listSize % batchSize;
|
||||||
sum++;
|
|
||||||
|
if (remainder != 0){
|
||||||
|
batchCount++;
|
||||||
}
|
}
|
||||||
List<List<T>> listList=new ArrayList<>();
|
|
||||||
int sumIm=0;
|
List<List<T>> result = new ArrayList<>();
|
||||||
for (int i=0;i<sum;i++){
|
int currentIndex = 0;
|
||||||
List<T> objects = new ArrayList<>();
|
|
||||||
int listSum=1000;
|
for (int i = 0; i < batchCount; i++){
|
||||||
if (sums!=0&&i==sum-1){
|
List<T> batch = new ArrayList<>();
|
||||||
listSum=sums;
|
int currentBatchSize = batchSize;
|
||||||
|
|
||||||
|
// 最后一批可能不满
|
||||||
|
if (remainder != 0 && i == batchCount - 1){
|
||||||
|
currentBatchSize = remainder;
|
||||||
}
|
}
|
||||||
for (int j=0;j<listSum;j++){
|
|
||||||
objects.add(list.get(sumIm));
|
for (int j = 0; j < currentBatchSize; j++){
|
||||||
sumIm++;
|
batch.add(list.get(currentIndex));
|
||||||
|
currentIndex++;
|
||||||
}
|
}
|
||||||
listList.add(objects);
|
result.add(batch);
|
||||||
}
|
}
|
||||||
return listList;
|
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,9 @@ import java.util.List;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.ExecutorService;
|
||||||
import java.util.concurrent.Executors;
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
@Component("CphTask")
|
@Component("CphTask")
|
||||||
public class CphTask {
|
public class CphTask {
|
||||||
@@ -56,26 +59,73 @@ public class CphTask {
|
|||||||
* 同步学生成绩
|
* 同步学生成绩
|
||||||
*/
|
*/
|
||||||
public void stuScoreMiddleSync(){
|
public void stuScoreMiddleSync(){
|
||||||
cphStuScoreMiddlesService.emptyTableDate();
|
try {
|
||||||
int pageNum=1;
|
System.out.println("开始同步学生成绩数据...");
|
||||||
int pageSize=3000;
|
long startTime = System.currentTimeMillis();
|
||||||
int studentInfoNumber = cphStuScoreMiddlesService.serectCphStuScoreMiddleCount(null);//总数
|
|
||||||
int sum=studentInfoNumber/ pageSize;//需要循环的次数
|
// 清空现有数据
|
||||||
int sumS=studentInfoNumber%pageSize>0? sum + 1: sum;
|
cphStuScoreMiddlesService.emptyTableDate();
|
||||||
QueryWrapper<CphStuScoreMiddleDto> objectQueryWrapper = new QueryWrapper<>();
|
System.out.println("已清空现有数据");
|
||||||
objectQueryWrapper.orderByDesc("stu_no", "kcdm");
|
|
||||||
ExecutorService executor = Executors.newFixedThreadPool(15);
|
// 配置参数
|
||||||
for (pageNum=1; pageNum <= sumS; pageNum++){
|
int pageSize = 3000;
|
||||||
final int currentPage = pageNum;
|
int threadPoolSize = 8; // 定时任务使用较少线程,避免影响其他任务
|
||||||
executor.execute(() -> {
|
|
||||||
List<Map<String, Object>> cphStuScoreMiddles = cphStuScoreMiddlesService
|
// 获取总数
|
||||||
.serectCphStuScoreMiddlesXh(currentPage, pageSize, null);
|
int studentInfoNumber = cphStuScoreMiddlesService.serectCphStuScoreMiddleCount(null);
|
||||||
cphStuScoreMiddlesService.selectCphStuScoreMiddleEmptyAndAdd(cphStuScoreMiddles);
|
int totalPages = (int) Math.ceil((double) studentInfoNumber / pageSize);
|
||||||
cphStuScoreMiddles.clear();
|
|
||||||
});
|
System.out.println("总记录数: " + studentInfoNumber + ", 总页数: " + totalPages);
|
||||||
|
|
||||||
|
// 创建线程池
|
||||||
|
ExecutorService executor = Executors.newFixedThreadPool(threadPoolSize);
|
||||||
|
CountDownLatch latch = new CountDownLatch(totalPages);
|
||||||
|
AtomicInteger successCount = new AtomicInteger(0);
|
||||||
|
AtomicInteger errorCount = new AtomicInteger(0);
|
||||||
|
|
||||||
|
// 提交任务
|
||||||
|
for (int pageNum = 1; pageNum <= totalPages; pageNum++) {
|
||||||
|
final int currentPage = pageNum;
|
||||||
|
executor.submit(() -> {
|
||||||
|
try {
|
||||||
|
List<Map<String, Object>> cphStuScoreMiddles = cphStuScoreMiddlesService
|
||||||
|
.serectCphStuScoreMiddlesXh(currentPage, pageSize, null);
|
||||||
|
|
||||||
|
if (cphStuScoreMiddles != null && !cphStuScoreMiddles.isEmpty()) {
|
||||||
|
cphStuScoreMiddlesService.selectCphStuScoreMiddleEmptyAndAdd(cphStuScoreMiddles);
|
||||||
|
successCount.incrementAndGet();
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
errorCount.incrementAndGet();
|
||||||
|
System.err.println("处理第" + currentPage + "页时发生错误: " + e.getMessage());
|
||||||
|
} finally {
|
||||||
|
latch.countDown();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// 等待所有任务完成
|
||||||
|
boolean completed = latch.await(30, TimeUnit.MINUTES);
|
||||||
|
|
||||||
|
if (!completed) {
|
||||||
|
System.err.println("同步任务未在指定时间内完成");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 关闭线程池
|
||||||
|
executor.shutdown();
|
||||||
|
|
||||||
|
long endTime = System.currentTimeMillis();
|
||||||
|
long duration = endTime - startTime;
|
||||||
|
|
||||||
|
System.out.println("学生成绩同步完成 - 总耗时: " + duration + "ms, 成功页数: " +
|
||||||
|
successCount.get() + ", 失败页数: " + errorCount.get());
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.err.println("同步学生成绩数据时发生严重错误: " + e.getMessage());
|
||||||
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
executor.shutdown();
|
|
||||||
//cphStuScoreMiddlesService.syncCphStuScoreMiddles();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user