xgxt_sd/zhxg_java
15
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化同步成绩

Browse Source
This commit is contained in:
zhy
2025-10-11 10:56:23 +08:00
parent c911be9fbc
commit 818a6f9911
6 changed files with 457 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
srs-comprehensive/src/main/java/com/srs/comprehensive/config/SyncConfig.java Normal file
View File

@@ -0,0 +1,78 @@
package com.srs.comprehensive.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* 数据同步配置类
* 用于管理数据同步相关的参数配置
*/
@Component
@ConfigurationProperties(prefix = "sync.student-score")
public class SyncConfig {
/**
* 分页大小
*/
private int pageSize = 3000;
/**
* Web接口线程池大小
*/
private int webThreadPoolSize = 10;
/**
* 定时任务线程池大小
*/
private int scheduledThreadPoolSize = 8;
/**
* 同步超时时间（分钟）
*/
private int timeoutMinutes = 30;
/**
* 批量插入大小
*/
private int batchInsertSize = 1000;
public int getPageSize() {
return pageSize;
}
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
public int getWebThreadPoolSize() {
return webThreadPoolSize;
}
public void setWebThreadPoolSize(int webThreadPoolSize) {
this.webThreadPoolSize = webThreadPoolSize;
}
public int getScheduledThreadPoolSize() {
return scheduledThreadPoolSize;
}
public void setScheduledThreadPoolSize(int scheduledThreadPoolSize) {
this.scheduledThreadPoolSize = scheduledThreadPoolSize;
}
public int getTimeoutMinutes() {
return timeoutMinutes;
}
public void setTimeoutMinutes(int timeoutMinutes) {
this.timeoutMinutes = timeoutMinutes;
}
public int getBatchInsertSize() {
return batchInsertSize;
}
public void setBatchInsertSize(int batchInsertSize) {
this.batchInsertSize = batchInsertSize;
}
}

222
srs-comprehensive/src/main/java/com/srs/comprehensive/service/impl/CphStuScoreMiddlesServiceImpl.java
View File

@@ -12,6 +12,7 @@ import com.srs.comprehensive.mapper.CphStuScoreMiddleDtoMapper;
import com.srs.comprehensive.mapper.CphStuScoreMiddlesMapper;
import com.srs.comprehensive.service.ICphStuScoreMiddlesService;
import com.srs.comprehensive.util.ListSliceUtil;
import com.srs.comprehensive.config.SyncConfig;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -27,6 +28,9 @@ public class CphStuScoreMiddlesServiceImpl implements ICphStuScoreMiddlesService
@Autowired
private CphStuScoreMiddleDtoMapper cphStuScoreMiddleDtoMapper;
@Autowired
private SyncConfig syncConfig;
@Override
@DataSource(DataSourceType.DATABASE)//sqlserver
@@ -183,68 +187,115 @@ public class CphStuScoreMiddlesServiceImpl implements ICphStuScoreMiddlesService
//转换
private CphStuScoreMiddleDto convertToB(Map<String,Object> a) {
CphStuScoreMiddleDto b = new CphStuScoreMiddleDto();
// 进行相应的属性赋值
if (a.get("CJ")!=null) {
if (Objects.equals(a.get("CJ").toString(), "不合格")) {
b.setCj("45.00");
} else if (Objects.equals(a.get("CJ").toString(), "合格")) {
b.setCj("65.00");
} else if (Objects.equals(a.get("CJ").toString(), "中等")) {
b.setCj("75.00");
} else if (Objects.equals(a.get("CJ").toString(), "良好")) {
b.setCj("85.00");
} else if (Objects.equals(a.get("CJ").toString(), "优秀")) {
b.setCj("95.00");
} else if (Objects.equals(a.get("CJ").toString(), "0.00")||
Objects.equals(a.get("CJ").toString(), "")||
Objects.equals(a.get("CJ").toString(), null)||
Objects.equals(a.get("CJ").toString(), "NULL")||
Objects.equals(a.get("CJ").toString(), "null")) {
System.out.println(a.get("CJ"));
if (a.get("BkScore")!=null){
if (!Objects.equals(a.get("BkScore").toString(), "0.00") ||
!Objects.equals(a.get("BkScore").toString(), "")||
!Objects.equals(a.get("BkScore").toString(), "null")||
!Objects.equals(a.get("BkScore").toString(), "NULL")||
!Objects.equals(a.get("BkScore").toString(), null)){
b.setCj(a.get("BkScore").toString());
} else {
b.setCj(a.get("CJ").toString());
}
}else {
b.setCj(a.get("CJ").toString());
}
}else {
b.setCj(a.get("CJ").toString());
if (a == null) {
return null;
}
try {
CphStuScoreMiddleDto b = new CphStuScoreMiddleDto();
// 设置学号和课程代码（必要字段）
b.setStuNo(getStringValue(a, "XH"));
b.setKcdm(getStringValue(a, "KCDM"));
b.setXqid(1L);
// 处理成绩字段
String score = processScore(a);
b.setCj(score);
// 设置其他字段
b.setXndm(getStringValue(a, "XND"));
b.setXqdm(getStringValue(a, "XQMC"));
// 处理数值类型字段
setBigDecimalValue(b::setXf, a, "XSHDXF");
setBigDecimalValue(b::setJd, a, "XSHDJD");
setLongValue(b::setFzlx, a, "FZLX");
// 设置是否通过
b.setIsPass(getStringValue(a, "Passed"));
return b;
} catch (Exception e) {
System.err.println("转换数据时发生错误: " + e.getMessage());
return null;
}
}
/**
* 安全获取字符串值
*/
private String getStringValue(Map<String, Object> map, String key) {
Object value = map.get(key);
if (value == null) {
return null;
}
String strValue = value.toString().trim();
return strValue.isEmpty() || "null".equalsIgnoreCase(strValue) || "NULL".equals(strValue) ? null : strValue;
}
/**
* 安全设置BigDecimal值
*/
private void setBigDecimalValue(java.util.function.Consumer<BigDecimal> setter, Map<String, Object> map, String key) {
try {
String value = getStringValue(map, key);
if (value != null) {
setter.accept(new BigDecimal(value));
}
} catch (NumberFormatException e) {
// 忽略格式错误，保持null值
}
b.setXqid(1L);
if (a.get("XND") != null) {
b.setXndm(a.get("XND").toString());
}
/**
* 安全设置Long值
*/
private void setLongValue(java.util.function.Consumer<Long> setter, Map<String, Object> map, String key) {
try {
String value = getStringValue(map, key);
if (value != null) {
setter.accept(Long.parseLong(value));
}
} catch (NumberFormatException e) {
// 忽略格式错误，保持null值
}
if (a.get("XQMC")!=null) {
b.setXqdm(a.get("XQMC").toString());
}
/**
* 处理成绩字段，支持等级制和百分制
*/
private String processScore(Map<String, Object> data) {
String score = getStringValue(data, "CJ");
if (score == null || score.isEmpty()) {
// 如果成绩为空，尝试使用补考成绩
return getStringValue(data, "BkScore");
}
if (a.get("XH")!=null) {
b.setStuNo(a.get("XH").toString());
// 处理等级制成绩
switch (score) {
case "不合格":
return "45.00";
case "合格":
return "65.00";
case "中等":
return "75.00";
case "良好":
return "85.00";
case "优秀":
return "95.00";
case "0.00":
case "":
case "null":
case "NULL":
// 空成绩时尝试使用补考成绩
String bkScore = getStringValue(data, "BkScore");
return bkScore != null ? bkScore : score;
default:
return score;
}
if (a.get("KCDM")!=null) {
b.setKcdm(a.get("KCDM").toString());
}
if (a.get("XSHDXF")!=null) {
b.setXf(new BigDecimal(a.get("XSHDXF").toString()));
}
if (a.get("XSHDJD")!=null) {
b.setJd(new BigDecimal(a.get("XSHDJD").toString()));
}
if (a.get("FZLX")!=null) {
b.setFzlx(Long.parseLong(a.get("FZLX").toString()));
}
if (a.get("Passed")!=null) {
b.setIsPass(a.get("Passed").toString());
}
return b;
}
//同步学生成绩，先清空在添加
@@ -277,11 +328,58 @@ public class CphStuScoreMiddlesServiceImpl implements ICphStuScoreMiddlesService
@Override
@DataSource(DataSourceType.MASTER)//mysql
public void selectCphStuScoreMiddleEmptyAndAdd(List<Map<String,Object>> cphStuScoreMiddles) {
List<CphStuScoreMiddleDto> cphStuScoreMiddleDto = cphStuScoreMiddles.stream()
.map(m -> convertToB(m)).collect(Collectors.toList());
if (cphStuScoreMiddleDto.size() != 0) {
cphStuScoreMiddleDtoMapper.insertBatchSomeColumn(cphStuScoreMiddleDto);//添加
if (cphStuScoreMiddles == null || cphStuScoreMiddles.isEmpty()) {
return;
}
try {
// 过滤无效数据
List<Map<String, Object>> validData = cphStuScoreMiddles.stream()
.filter(this::isValidData)
.collect(Collectors.toList());
if (validData.isEmpty()) {
return;
}
// 批量转换数据
List<CphStuScoreMiddleDto> cphStuScoreMiddleDto = validData.parallelStream()
.map(this::convertToB)
.filter(Objects::nonNull)
.collect(Collectors.toList());
if (!cphStuScoreMiddleDto.isEmpty()) {
// 分批插入，避免单次插入数据量过大
List<List<CphStuScoreMiddleDto>> batches = ListSliceUtil.insertSlice(cphStuScoreMiddleDto, syncConfig.getBatchInsertSize());
if (batches != null) {
for (List<CphStuScoreMiddleDto> batch : batches) {
if (batch != null && !batch.isEmpty()) {
cphStuScoreMiddleDtoMapper.insertBatchSomeColumn(batch);
}
}
}
}
} catch (Exception e) {
// 记录错误但不抛出异常，避免影响其他批次
System.err.println("处理学生成绩数据时发生错误: " + e.getMessage());
e.printStackTrace();
}
}
/**
* 验证数据有效性
*/
private boolean isValidData(Map<String, Object> data) {
if (data == null) {
return false;
}
// 检查必要字段
Object stuNo = data.get("XH");
Object kcdm = data.get("KCDM");
return stuNo != null && !stuNo.toString().trim().isEmpty() &&
kcdm != null && !kcdm.toString().trim().isEmpty();
}
}

71
srs-comprehensive/src/main/java/com/srs/comprehensive/util/ListSliceUtil.java
View File

@@ -10,29 +10,64 @@ import java.util.List;
*/
public class ListSliceUtil {
public static <T> List<List<T>> updateSlice(List<T> list){
if (list.size()==0){
return sliceList(list, 1000);
}
/**
* 插入数据时切割列表，使用较大的批次大小
*/
public static <T> List<List<T>> insertSlice(List<T> list){
return sliceList(list, 1000);
}
/**
* 插入数据时切割列表，支持自定义批次大小
* @param list 要切割的列表
* @param batchSize 每批的大小
* @return 切割后的列表集合
*/
public static <T> List<List<T>> insertSlice(List<T> list, int batchSize){
return sliceList(list, batchSize);
}
/**
* 通用列表切割方法
* @param list 要切割的列表
* @param batchSize 每批的大小
* @return 切割后的列表集合
*/
private static <T> List<List<T>> sliceList(List<T> list, int batchSize){
if (list == null || list.isEmpty()){
return null;
}
int userListSize=list.size();
int sum=userListSize/1000;
int sums=userListSize%1000;
if (sums!=0){
sum++;
int listSize = list.size();
int batchCount = listSize / batchSize;
int remainder = listSize % batchSize;
if (remainder != 0){
batchCount++;
}
List<List<T>> listList=new ArrayList<>();
int sumIm=0;
for (int i=0;i<sum;i++){
List<T> objects = new ArrayList<>();
int listSum=1000;
if (sums!=0&&i==sum-1){
listSum=sums;
List<List<T>> result = new ArrayList<>();
int currentIndex = 0;
for (int i = 0; i < batchCount; i++){
List<T> batch = new ArrayList<>();
int currentBatchSize = batchSize;
// 最后一批可能不满
if (remainder != 0 && i == batchCount - 1){
currentBatchSize = remainder;
}
for (int j=0;j<listSum;j++){
objects.add(list.get(sumIm));
sumIm++;
for (int j = 0; j < currentBatchSize; j++){
batch.add(list.get(currentIndex));
currentIndex++;
}
listList.add(objects);
result.add(batch);
}
return listList;
return result;
}
}