hzya
/
kangarooDataCenterV3
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(buildpackage): 优化数据处理逻辑和性能 

- 调整了数据过滤和保存的顺序，先过滤再保存，提高效率
- 使用流式处理和并发插入数据，提升性能
-修复了一些潜在的空指针异常问题
- 优化了日志输出，增加了耗时记录
This commit is contained in:
liuy 2024-10-26 15:27:56 +08:00
parent 4c89f1f88e
commit ba115da9bb
2 changed files with 79 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
buildpackage/src/main/java/com/hzya/frame/plugin/lets/plugin/sales/SoSaleOutPluginInitializerToC.java
View File

@ -405,10 +405,10 @@ public class SoSaleOutPluginInitializerToC extends PluginBaseEntity {
* @author liuyang
*/
private void getSetStock(List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.HeaderDetailsDto> headerDetailsDtoList) throws Exception {
//保存到mysql底表
batchInsert(headerDetailsDtoList);
//过滤成功的数据
List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.HeaderDetailsDto> headerDetailsDtos = filterDataStock(headerDetailsDtoList);
//保存到mysql底表
batchInsert(headerDetailsDtos);
//执行推送主逻辑
implementStock(headerDetailsDtos);
}
@ -419,11 +419,11 @@ public class SoSaleOutPluginInitializerToC extends PluginBaseEntity {
* @author liuyang
*/
private void getSetTran(List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.HeaderDetailsDto> headerDetailsDtoList) throws Exception {
//保存到mysql底表
batchInsert(headerDetailsDtoList);
//过滤成功的数据
//先过滤成功的数据，再保存到低表
//headerDetailsDtos携带红、蓝单据推送成功与否的明细行
List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.HeaderDetailsDto> headerDetailsDtos = filterDataTran(headerDetailsDtoList);
//保存到mysql底表
batchInsert(headerDetailsDtos);
//执行推送采购主逻辑
implementTran(headerDetailsDtos);
}
@ -587,6 +587,18 @@ public class SoSaleOutPluginInitializerToC extends PluginBaseEntity {
private List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.HeaderDetailsDto> filterDataRowsAsPushOrFailedTran(List<TocofsSaleoutDetailedEntity> allTocofsSaleoutDetailedEntityList, List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.HeaderDetailsDto> headerDetailsDtoList) throws Exception {
List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.HeaderDetailsDto> targetHeaderDetailsDtoList = new ArrayList<>();
if (headerDetailsDtoList != null && headerDetailsDtoList.size() > 0) {
List<TocofsSaleoutDetailedEntity> resultList2 = allTocofsSaleoutDetailedEntityList.stream().map(obj -> {
if (obj.getNewstate2() == null) {
obj.setNewstate2("");
}
if (obj.getNewstate3() == null) {
obj.setNewstate3("");
}
return obj;
}).collect(Collectors.toList());
Map<String, String> no2NameMap2 = resultList2.stream().collect(Collectors.toMap(TocofsSaleoutDetailedEntity::getId, TocofsSaleoutDetailedEntity::getNewstate2));
Map<String, String> no2NameMap3 = resultList2.stream().collect(Collectors.toMap(TocofsSaleoutDetailedEntity::getId, TocofsSaleoutDetailedEntity::getNewstate3));
//字段Y(成功)或者为H(待处理)可以视为成功，完成了小段业务闭环的数据行
String succeseeY = "Y";
String succeseeH = "H";
@ -603,21 +615,28 @@ public class SoSaleOutPluginInitializerToC extends PluginBaseEntity {
String newstate2 = null;
String newstate3 = null;
// TocofsSaleoutDetailedEntity targetTocofsSaleoutDetailedEntity = null;
if (allTocofsSaleoutDetailedEntityList != null && allTocofsSaleoutDetailedEntityList.size() > 0) {
for (int k = 0; k < allTocofsSaleoutDetailedEntityList.size(); k++) {
TocofsSaleoutDetailedEntity tocofsSaleoutDetailedEntity = allTocofsSaleoutDetailedEntityList.get(k);
if (tocofsSaleoutDetailedEntity.getId().equals(detailsDto.getId())) {
newstate2 = tocofsSaleoutDetailedEntity.getNewstate2();
newstate3 = tocofsSaleoutDetailedEntity.getNewstate3();
// targetTocofsSaleoutDetailedEntity = tocofsSaleoutDetailedEntity;
//验证交易成功(红)是否成功、(销售)交易成功(蓝)是否成功 两者如果都成功，则过滤掉，如果其中一个不能成功，则还是需要进行补推！
//验证红或蓝单据是否推送成功的代码，在implementTran中实现
if ((succeseeY.equals(newstate2) || succeseeH.equals(newstate2)) && (succeseeY.equals(newstate3) || succeseeH.equals(newstate3))) {
isSuccess = true;
}
}
}
// if (allTocofsSaleoutDetailedEntityList != null && allTocofsSaleoutDetailedEntityList.size() > 0) {
// for (int k = 0; k < allTocofsSaleoutDetailedEntityList.size(); k++) {
// TocofsSaleoutDetailedEntity tocofsSaleoutDetailedEntity = allTocofsSaleoutDetailedEntityList.get(k);
// if (tocofsSaleoutDetailedEntity.getId().equals(detailsDto.getId())) {
// newstate2 = tocofsSaleoutDetailedEntity.getNewstate2();
// newstate3 = tocofsSaleoutDetailedEntity.getNewstate3();
//// targetTocofsSaleoutDetailedEntity = tocofsSaleoutDetailedEntity;
// //验证交易成功(红)是否成功、(销售)交易成功(蓝)是否成功 两者如果都成功，则过滤掉，如果其中一个不能成功，则还是需要进行补推！
// //验证红或蓝单据是否推送成功的代码，在implementTran中实现
// if ((succeseeY.equals(newstate2) || succeseeH.equals(newstate2)) && (succeseeY.equals(newstate3) || succeseeH.equals(newstate3))) {
// isSuccess = true;
// }
// }
// }
// }
//交易成功红
newstate2 = no2NameMap2.get(detailsDto.getId());
newstate3 = no2NameMap3.get(detailsDto.getId());
if ((newstate2 != null && newstate3 != null) && (succeseeY.equals(newstate2) || succeseeH.equals(newstate2)) && (succeseeY.equals(newstate3) || succeseeH.equals(newstate3))) {
isSuccess = true;
}
if (!isSuccess) {
//拷贝推送状态参数，方便推送是进行判断
detailsDto.setNewState2(newstate2);
@ -681,8 +700,9 @@ public class SoSaleOutPluginInitializerToC extends PluginBaseEntity {
}
//每50作为一个批次插入主表，根据主键（id）判断是否重复，如果重复的，则不进行插入
List<List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.HeaderDto>> ofssaleorderoutsearchList = SplitListByCountUtil.splitListByCount(headerDetailsDtoList1, 500);
List<List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.HeaderDto>> ofssaleorderoutsearchList = SplitListByCountUtil.splitListByCount(headerDetailsDtoList1, 200);
for (int i = 0; i < ofssaleorderoutsearchList.size(); i++) {
long startTime = System.currentTimeMillis();
List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.HeaderDto> headerDtoList = ofssaleorderoutsearchList.get(i);
Thread thread = new Thread(new Runnable() {
@Override
@ -706,12 +726,15 @@ public class SoSaleOutPluginInitializerToC extends PluginBaseEntity {
} catch (Exception e) {
logger.error("堵塞线程抛出异常", e);
}
long endTime = System.currentTimeMillis();
logger.info("插入或更新TOC销售出库表头-耗时：{}", (endTime - startTime));
}
//插入明细表
List<List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.DetailsDto>> detailsDtoList = SplitListByCountUtil.splitListByCount(headerDetailsDtoList2, 500);
List<List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.DetailsDto>> detailsDtoList = SplitListByCountUtil.splitListByCount(headerDetailsDtoList2, 200);
for (int i = 0; i < detailsDtoList.size(); i++) {
List<com.hzya.frame.ttxofs.dto.ofssaleorderoutsearch.DetailsDto> detailsDtos = detailsDtoList.get(i);
long startTime = System.currentTimeMillis();
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
@ -734,6 +757,8 @@ public class SoSaleOutPluginInitializerToC extends PluginBaseEntity {
} catch (Exception e) {
logger.error("堵塞线程抛出异常", e);
}
long endTime = System.currentTimeMillis();
logger.info("插入或更新TOC销售出库表体-耗时：{}", (endTime - startTime));
}
} catch (Exception e) {
logger.error("TOC业务保存到Mysql底表失败！TocofsSaleoutEntity、TocofsSaleoutDetailedEntity保存失败", e);

48
buildpackage/src/main/java/com/hzya/frame/plugin/lets/plugin/sales/SoSaleReturnPluginInitializerToC.java
View File

@ -332,10 +332,10 @@ public class SoSaleReturnPluginInitializerToC extends PluginBaseEntity {
* @author liuyang
*/
private void getSetStock(List<StockinOrderSearchResponse.StockinOrder> returnGoodHeaderDetailsDataDtoArrayList) throws Exception {
//保存到mysql
batchInsert(returnGoodHeaderDetailsDataDtoArrayList);
//过滤成功的数据
List<StockinOrderSearchResponse.StockinOrder> stockinOrderList = filterDataStock(returnGoodHeaderDetailsDataDtoArrayList);
//保存到mysql
batchInsert(stockinOrderList);
//执行推送主逻辑
implementStock(stockinOrderList);
}
@ -346,10 +346,10 @@ public class SoSaleReturnPluginInitializerToC extends PluginBaseEntity {
* @author liuyang
*/
private void getSetTran(List<StockinOrderSearchResponse.StockinOrder> returnGoodHeaderDetailsDataDtoArrayList) throws Exception {
//保存到mysql
batchInsert(returnGoodHeaderDetailsDataDtoArrayList);
//过滤成功的数据
List<StockinOrderSearchResponse.StockinOrder> stockinOrderList = filterDataTran(returnGoodHeaderDetailsDataDtoArrayList);
//保存到mysql
batchInsert(stockinOrderList);
//执行推送主逻辑
implementTran(stockinOrderList);
}
@ -462,9 +462,10 @@ public class SoSaleReturnPluginInitializerToC extends PluginBaseEntity {
}
//每250作为一个批次插入主表，根据主键（id）判断是否重复，如果重复的，则不进行插入
List<List<StockinOrderSearchResponse.StockinOrder.StockinH>> lists = SplitListByCountUtil.splitListByCount(headerDetailsDtoList1, 500);
List<List<StockinOrderSearchResponse.StockinOrder.StockinH>> lists = SplitListByCountUtil.splitListByCount(headerDetailsDtoList1, 100);
for (int i = 0; i < lists.size(); i++) {
List<StockinOrderSearchResponse.StockinOrder.StockinH> stockinHS = lists.get(i);
long startTime = System.currentTimeMillis();
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
@ -487,13 +488,16 @@ public class SoSaleReturnPluginInitializerToC extends PluginBaseEntity {
} catch (Exception e) {
logger.error("线程保存TOC退货主表抛出异常", e);
}
long endTime = System.currentTimeMillis();
logger.info("插入或更新TOC退货入库表头-耗时：{}", (endTime - startTime));
}
//插入明细表
List<List<StockinOrderSearchResponse.StockinOrder.StockinB>> lists1 = SplitListByCountUtil.splitListByCount(headerDetailsDtoList2, 500);
List<List<StockinOrderSearchResponse.StockinOrder.StockinB>> lists1 = SplitListByCountUtil.splitListByCount(headerDetailsDtoList2, 100);
for (int i = 0; i < lists1.size(); i++) {
List<StockinOrderSearchResponse.StockinOrder.StockinB> stockinBS = lists1.get(i);
List<TocofsReturngoodsDetailedEntity> tocofsReturngoodsDetailedEntities = copyDetailsDto(stockinBS);
long startTime = System.currentTimeMillis();
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
@ -515,6 +519,8 @@ public class SoSaleReturnPluginInitializerToC extends PluginBaseEntity {
} catch (Exception e) {
logger.error("线程保存TOC退货明细抛出异常", e);
}
long endTime = System.currentTimeMillis();
logger.info("插入或更新TOC退货入库表体-耗时：{}", (endTime - startTime));
}
}
}
@ -2182,8 +2188,15 @@ public class SoSaleReturnPluginInitializerToC extends PluginBaseEntity {
Assert.notNull(returnGoodHeaderDetailsDataDtoArrayList, "returnGoodHeaderDetailsDataDtoArrayList不能为空");
List<StockinOrderSearchResponse.StockinOrder> targetHeaderDetailsDtoList = new ArrayList<>();
if (tocofsReturngoodsDetailedEntityList != null && tocofsReturngoodsDetailedEntityList.size() > 0 && returnGoodHeaderDetailsDataDtoArrayList != null && returnGoodHeaderDetailsDataDtoArrayList.size() > 0) {
List<TocofsReturngoodsDetailedEntity> collect = tocofsReturngoodsDetailedEntityList.stream().map(obj -> {
if (obj.getNewstate4() == null) {
obj.setNewstate4("");
}
return obj;
}).collect(Collectors.toList());
Map<String, String> no2NameMap2 = collect.stream().collect(Collectors.toMap(TocofsReturngoodsDetailedEntity::getId, TocofsReturngoodsDetailedEntity::getNewstate4));
//字段Y(成功)或者为H(待处理)可以视为成功，完成了小段业务闭环的数据行
String succeseeY = "Y";
String succeseeH = "H";
@ -2197,15 +2210,20 @@ public class SoSaleReturnPluginInitializerToC extends PluginBaseEntity {
StockinOrderSearchResponse.StockinOrder.StockinB stockinB = details.get(j);
Boolean isSuccess = false;
String newstate = null;
for (int k = 0; k < tocofsReturngoodsDetailedEntityList.size(); k++) {
TocofsReturngoodsDetailedEntity tocofsReturngoodsDetailedEntity = tocofsReturngoodsDetailedEntityList.get(k);
if (tocofsReturngoodsDetailedEntity.getId().equals(stockinB.getId())) {
newstate = tocofsReturngoodsDetailedEntity.getNewstate4();
if (succeseeY.equals(newstate) || succeseeH.equals(newstate)) {
isSuccess = true;
}
}
// for (int k = 0; k < tocofsReturngoodsDetailedEntityList.size(); k++) {
// TocofsReturngoodsDetailedEntity tocofsReturngoodsDetailedEntity = tocofsReturngoodsDetailedEntityList.get(k);
// if (tocofsReturngoodsDetailedEntity.getId().equals(stockinB.getId())) {
// newstate = tocofsReturngoodsDetailedEntity.getNewstate4();
// if (succeseeY.equals(newstate) || succeseeH.equals(newstate)) {
// isSuccess = true;
// }
// }
// }
newstate = no2NameMap2.get(stockinB.getId());
if (newstate != null && !"".equals(newstate) && (succeseeY.equals(newstate) || succeseeH.equals(newstate))) {
isSuccess = true;
}
if (!isSuccess) {
stockinB.setNewstate4(newstate);
targetDetails.add(stockinB);