Skip to content

Commit 605d20e

Browse files
moocstudentmoocstudent
committed
demo by implementsteam @zq 
1 parent 98de1fc commit 605d20e

File tree

16 files changed

+15179
-116
lines changed

16 files changed

+15179
-116
lines changed

‎README.MD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
最终,经过测试,不管是websocket的实时数据返回,以及另一个接口的调用查询,都跟excel的Upload操作是有阻塞的关系,不过通过eventBus的调用可以实时地记录本次的上传具体信息,虽然没办法做到实时返回进度,但是记录失败条数,以及记录每次导入的是按哪些条件/每次批量大小还是可用的.

‎pom.xml‎

Lines changed: 48 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,44 @@
1919
</properties>
2020

2121
<dependencies>
22+
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-webflux -->
23+
<dependency>
24+
<groupId>org.springframework.boot</groupId>
25+
<artifactId>spring-boot-starter-webflux</artifactId>
26+
<version>2.4.1</version>
27+
</dependency>
28+
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-data-r2dbc -->
29+
<dependency>
30+
<groupId>org.springframework.boot</groupId>
31+
<artifactId>spring-boot-starter-data-r2dbc</artifactId>
32+
<version>2.4.1</version>
33+
</dependency>
34+
<!-- https://mvnrepository.com/artifact/dev.miku/r2dbc-mysql -->
35+
<dependency>
36+
<groupId>dev.miku</groupId>
37+
<artifactId>r2dbc-mysql</artifactId>
38+
<scope>runtime</scope>
39+
<version>0.8.2.RELEASE</version>
40+
</dependency>
41+
<!-- https://mvnrepository.com/artifact/io.projectreactor.kotlin/reactor-kotlin-extensions -->
42+
<dependency>
43+
<groupId>io.projectreactor.kotlin</groupId>
44+
<artifactId>reactor-kotlin-extensions</artifactId>
45+
<version>1.1.1</version>
46+
</dependency>
47+
<!-- https://mvnrepository.com/artifact/io.projectreactor.addons/reactor-adapter -->
48+
<dependency>
49+
<groupId>io.projectreactor.addons</groupId>
50+
<artifactId>reactor-adapter</artifactId>
51+
<version>3.4.1</version>
52+
</dependency>
53+
<!-- https://mvnrepository.com/artifact/org.junit/junit5-engine -->
54+
<dependency>
55+
<groupId>org.junit</groupId>
56+
<artifactId>junit5-engine</artifactId>
57+
<version>5.0.0-ALPHA</version>
58+
</dependency>
59+
2260
<dependency>
2361
<groupId>org.springframework.boot</groupId>
2462
<artifactId>spring-boot-starter-actuator</artifactId>
@@ -34,24 +72,16 @@
3472
<!-- </dependency>-->
3573

3674

37-
<!-- <dependency>-->
38-
<!-- <groupId>org.springframework.boot</groupId>-->
39-
<!-- <artifactId>spring-boot-starter-data-jdbc</artifactId>-->
40-
<!-- </dependency>-->
41-
<!-- <dependency>-->
42-
<!-- <groupId>org.springframework.boot</groupId>-->
43-
<!-- <artifactId>spring-boot-starter-data-r2dbc</artifactId>-->
44-
<!-- </dependency>-->
45-
<!-- <dependency>-->
46-
<!-- <groupId>dev.miku</groupId>-->
47-
<!-- <artifactId>r2dbc-mysql</artifactId>-->
48-
<!-- <scope>runtime</scope>-->
49-
<!-- </dependency>-->
50-
<!-- <dependency>-->
51-
<!-- <groupId>mysql</groupId>-->
52-
<!-- <artifactId>mysql-connector-java</artifactId>-->
53-
<!-- <scope>runtime</scope>-->
54-
<!-- </dependency>-->
75+
<dependency>
76+
<groupId>org.springframework.boot</groupId>
77+
<artifactId>spring-boot-starter-data-jdbc</artifactId>
78+
</dependency>
79+
80+
<dependency>
81+
<groupId>mysql</groupId>
82+
<artifactId>mysql-connector-java</artifactId>
83+
<scope>runtime</scope>
84+
</dependency>
5585

5686
<!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
5787
<dependency>
@@ -74,16 +104,11 @@
74104
<version>30.0-jre</version>
75105
</dependency>
76106

77-
<dependency>
78-
<groupId>org.springframework.boot</groupId>
79-
<artifactId>spring-boot-starter-webflux</artifactId>
80-
</dependency>
81107
<dependency>
82108
<groupId>org.springframework.boot</groupId>
83109
<artifactId>spring-boot-starter-websocket</artifactId>
84110
</dependency>
85111

86-
87112
<dependency>
88113
<groupId>org.projectlombok</groupId>
89114
<artifactId>lombok</artifactId>

‎sql/zq-excelinfo.sql‎

Lines changed: 14712 additions & 0 deletions
Large diffs are not rendered by default.

‎src/main/java/moocstudent/github/io/excelprocessdemo/ExcelProcessDemoApplication.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22

33
import org.springframework.boot.SpringApplication;
44
import org.springframework.boot.autoconfigure.SpringBootApplication;
5+
import org.springframework.data.r2dbc.repository.config.EnableR2dbcRepositories;
56

67
@SpringBootApplication
8+
@EnableR2dbcRepositories
79
public class ExcelProcessDemoApplication {
810

911
public static void main(String[] args) {

‎src/main/java/moocstudent/github/io/excelprocessdemo/config/CorsConfig.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package moocstudent.github.io.excelprocessdemo.config;
2+
3+
import org.springframework.context.annotation.Bean;
4+
import org.springframework.context.annotation.Configuration;
5+
import org.springframework.web.cors.CorsConfiguration;
6+
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
7+
import org.springframework.web.filter.CorsFilter;
8+
9+
/**
10+
* 解决接口访问跨域问题
11+
*/
12+
@Configuration
13+
public class CorsConfig {
14+
@Bean
15+
public CorsFilter corsFilter(){
16+
UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
17+
source.registerCorsConfiguration("/**",buildConfig());
18+
return new CorsFilter(source);
19+
}
20+
21+
private CorsConfiguration buildConfig(){
22+
CorsConfiguration corsConfiguration = new CorsConfiguration();
23+
corsConfiguration.addAllowedOrigin("*");
24+
corsConfiguration.addAllowedHeader("*");
25+
corsConfiguration.addAllowedMethod("*");
26+
return corsConfiguration;
27+
}
28+
}

‎src/main/java/moocstudent/github/io/excelprocessdemo/controller/ExcelImportController.java‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,17 @@
22

33
import com.alibaba.excel.EasyExcel;
44
import com.google.common.eventbus.EventBus;
5+
import moocstudent.github.io.excelprocessdemo.event.ExcelProcessEvent;
56
import moocstudent.github.io.excelprocessdemo.handler.ExcelEventListener;
67
import moocstudent.github.io.excelprocessdemo.listener.UploadExcelInfoDataListener;
78
import moocstudent.github.io.excelprocessdemo.model.ExcelInfo;
89
import moocstudent.github.io.excelprocessdemo.service.ExcelInfoServiceImpl;
10+
import moocstudent.github.io.excelprocessdemo.service.RulesServiceImpl;
911
import org.springframework.beans.factory.annotation.Autowired;
12+
import org.springframework.beans.factory.annotation.Qualifier;
1013
import org.springframework.web.bind.annotation.*;
1114
import org.springframework.web.multipart.MultipartFile;
15+
import reactor.core.publisher.Mono;
1216

1317
import java.io.IOException;
1418
import java.util.HashMap;
@@ -25,31 +29,44 @@ public class ExcelImportController {
2529
@Autowired
2630
private ExcelInfoServiceImpl excelInfoService;
2731
@Autowired
32+
private RulesServiceImpl rulesService;
33+
@Autowired
2834
private EventBus eventBus;
35+
@Qualifier("excelEventListener")
2936
@Autowired
3037
private ExcelEventListener listener;
3138

3239
/**
3340
* @param theExcel excel文件
3441
* @param num 商家编号
35-
* @param length excel内容条目数量
3642
* @return
3743
*/
38-
@PostMapping("/importExcel/{num}/{length}")
39-
public Integer importExcel(@RequestParam(value = "theExcel") MultipartFile theExcel,
40-
@PathVariable String num,
41-
@PathVariable Long length) throws IOException {
42-
44+
@PostMapping("/importExcel/{num}")
45+
public Mono<ExcelProcessEvent> importExcel(@RequestParam(value = "theExcel") MultipartFile theExcel,
46+
@PathVariable String num) throws IOException {
47+
//这里输出前端传入的num了? 传入就开始操作下一个步骤吧
48+
System.out.println("获取了num:" + num);
4349
Map<String, Object> paramsMap = new HashMap<>();
4450
paramsMap.put("num", num);
45-
paramsMap.put("length", length);
46-
51+
//开始读数据
4752
EasyExcel.read(theExcel.getInputStream(),
4853
ExcelInfo.class,
49-
new UploadExcelInfoDataListener(excelInfoService, eventBus,listener, paramsMap))
54+
new UploadExcelInfoDataListener(excelInfoService, rulesService, eventBus, listener, paramsMap))
5055
.sheet().doRead();
5156

5257
eventBus.unregister(listener);
53-
return 1;
58+
/*
59+
websocket轮询获取进度, 这里是进行了补充
60+
因为经过测试 发现websocket和excel的导入其中总会有一个进行了阻塞
61+
并不能在excel导入时将数据发到websocket回传给前端
62+
而只能通过在结束时, 将最终数据发给前端(不过这也是有了一个不小的提升不是吗 ?)
63+
*/
64+
ExcelProcessEvent event = listener.getProcessEventInListener(num);
65+
//当一切上传完毕,归零
66+
if (event.getAllOk()) {
67+
System.out.println("isAllOk!!");
68+
listener.resetEventHandled();
69+
}
70+
return Mono.just(event);
5471
}
5572
}

‎src/main/java/moocstudent/github/io/excelprocessdemo/event/ExcelProcessEvent.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
import lombok.Builder;
55
import lombok.Data;
66
import lombok.NoArgsConstructor;
7+
import moocstudent.github.io.excelprocessdemo.model.ExcelInfo;
8+
9+
import java.util.List;
710

811
/**
912
* @Author: zhangQi
@@ -13,12 +16,17 @@
1316
@AllArgsConstructor
1417
@NoArgsConstructor
1518
@Builder
16-
public class ExcelProcessEvent {
19+
public class ExcelProcessEvent{
20+
21+
private String num;
1722

1823
private Integer alreadyInSize;
1924

2025
private Boolean allOk;
2126

27+
private List<ExcelInfo> listNotValid;
28+
2229
// private Integer totalSize;
2330

31+
2432
}

‎src/main/java/moocstudent/github/io/excelprocessdemo/handler/ExcelEventListener.java‎

Lines changed: 82 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,70 +3,133 @@
33
import com.alibaba.fastjson.JSONObject;
44
import com.google.common.eventbus.Subscribe;
55
import moocstudent.github.io.excelprocessdemo.event.ExcelProcessEvent;
6+
import moocstudent.github.io.excelprocessdemo.model.ExcelInfo;
67
import org.slf4j.Logger;
78
import org.slf4j.LoggerFactory;
9+
import org.springframework.stereotype.Component;
810
import org.springframework.web.socket.CloseStatus;
911
import org.springframework.web.socket.TextMessage;
1012
import org.springframework.web.socket.WebSocketSession;
1113
import org.springframework.web.socket.handler.AbstractWebSocketHandler;
12-
import reactor.core.publisher.Mono;
1314

14-
import java.time.Duration;
15+
import java.util.ArrayList;
16+
import java.util.List;
1517

1618
/**
1719
* @Author: zhangQi
1820
* @Date: 2020-12-13 14:40
1921
*/
22+
@Component
2023
public class ExcelEventListener extends AbstractWebSocketHandler {
2124

2225
private static final Logger log = LoggerFactory.getLogger(ExcelEventListener.class);
2326

27+
/**
28+
* 模拟了这个excel的num编号,用于区分其它的excel,但其实这并没有什么卵用
29+
* 估计用它用于判定吧,当并行和多线程产生时,当作一个区分的值也好
30+
*/
31+
private String num;
32+
2433
private int eventsHandled;
2534

2635
private boolean isAllOk;
2736

28-
@Subscribe
29-
public void excelProcessEvent(ExcelProcessEvent event){
30-
eventsHandled+=event.getAlreadyInSize();
31-
isAllOk = event.getAllOk();
37+
private List<ExcelInfo> listNotValid = null;
38+
39+
public String getNum() {
40+
return num;
3241
}
3342

34-
int getEventsHandled() {
43+
public int getEventsHandled() {
3544
return eventsHandled;
3645
}
3746

38-
boolean getAllOk(){
47+
/**
48+
* 如果人生一切都是allOk,那么这个人生也许并不是那么精彩
49+
*
50+
* @return
51+
*/
52+
public boolean getAllOk() {
3953
return isAllOk;
4054
}
4155

42-
void resetEventHandled(){
56+
public List<ExcelInfo> getListNotValid() {
57+
return listNotValid;
58+
}
59+
60+
/**
61+
* 前端获取当前进度,通过轮询方式,这里已经通过@Subscribe拿到了进度数据
62+
* 但是最终会不会产生效果呢?这个还得问问老天爷--靠玄学吧:)
63+
* @param num
64+
* @return
65+
*/
66+
public ExcelProcessEvent getProcessEventInListener(String num) {
67+
if (this.num.equals(num)) {
68+
return ExcelProcessEvent.builder()
69+
.num(this.num)
70+
.allOk(this.isAllOk)
71+
.alreadyInSize(this.eventsHandled)
72+
.listNotValid(this.listNotValid)
73+
.build();
74+
} else {
75+
return null;
76+
}
77+
}
78+
79+
/**
80+
* 通过在UploadExcelInfoDataListener中的事件event post过来数据
81+
* @param event
82+
*/
83+
@Subscribe
84+
public void excelProcessEvent(ExcelProcessEvent event) {
85+
if (num == null || num == "") {
86+
num = event.getNum();
87+
}
88+
if(event.getListNotValid()!=null){
89+
this.listNotValid = new ArrayList<>();
90+
}
91+
System.out.println("current num:" + num);
92+
//每次将非法的excel条目传入
93+
this.listNotValid = event.getListNotValid();
94+
System.out.println("current listNotValid:" + listNotValid);
95+
eventsHandled += event.getAlreadyInSize();
96+
System.out.println("current eventsHandled:" + eventsHandled);
97+
isAllOk = event.getAllOk();
98+
System.out.println("current isAllOk:" + isAllOk);
99+
}
100+
101+
public void resetEventHandled() {
102+
num = null;
43103
eventsHandled = 0;
44104
isAllOk = false;
105+
listNotValid = null;
45106
}
46107

47108
@Override
48-
public void afterConnectionEstablished(WebSocketSession session){
109+
public void afterConnectionEstablished(WebSocketSession session) {
49110
log.info("WebSocket 建立连接...");
50111
}
51112

52-
53113
@Override
54-
protected void handleTextMessage(WebSocketSession session, TextMessage message)throws Exception{
55-
log.info("接收到消息: "+message.getPayload());
56-
Mono.delay(Duration.ofMillis(2000));
114+
protected void handleTextMessage(WebSocketSession session, TextMessage message) throws Exception {
115+
log.info("接收到消息: " + message.getPayload());
57116
//发送文本消息
58117
JSONObject jsonObject = new JSONObject();
59-
118+
jsonObject.put("num", num);
60119
// jsonObject.put("total",1100); //total可由前端判定excel文件内容rows大小获得也可以
61-
jsonObject.put("currentSize",eventsHandled);
62-
jsonObject.put("isAllOk",isAllOk);
63-
120+
jsonObject.put("currentSize", eventsHandled);
121+
System.out.println("send currentSize:" + eventsHandled);
122+
jsonObject.put("isAllOk", isAllOk);
123+
jsonObject.put("listNotValid", listNotValid);
124+
System.out.println("send isAllOk:" + isAllOk);
125+
System.out.println("send listNotValid:" + listNotValid);
64126
session.sendMessage(new TextMessage(jsonObject.toString()));
65127
}
66128

67129
@Override
68-
public void afterConnectionClosed(WebSocketSession session, CloseStatus closeStatus){
130+
public void afterConnectionClosed(WebSocketSession session, CloseStatus closeStatus) {
69131
log.info("WebSocket 连接关闭...");
70132
}
71133

134+
72135
}

0 commit comments

Comments
 (0)