diff --git a/docs/demo.sql b/docs/demo.sql index 1498ad0..2a38e66 100644 --- a/docs/demo.sql +++ b/docs/demo.sql @@ -12,4 +12,12 @@ CREATE TABLE fail_order `create_dt` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `update_dt` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', PRIMARY KEY (`id`) -); \ No newline at end of file +); + +-- 手机号表 +CREATE TABLE `phone_number` ( + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键', + `phone_number` varchar(20) NOT NULL DEFAULT '' COMMENT '手机号', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='手机号表' diff --git a/pom.xml b/pom.xml index 716b2d1..51efe03 100644 --- a/pom.xml +++ b/pom.xml @@ -111,6 +111,16 @@ okhttp 4.2.0 + + com.alibaba + easyexcel + 3.1.3 + + + com.alibaba + fastjson + 1.2.83 + diff --git a/src/main/java/com/example/snailjob/bo/PhoneNumberBo.java b/src/main/java/com/example/snailjob/bo/PhoneNumberBo.java new file mode 100644 index 0000000..2b79e6d --- /dev/null +++ b/src/main/java/com/example/snailjob/bo/PhoneNumberBo.java @@ -0,0 +1,18 @@ +package com.example.snailjob.bo; + +import com.alibaba.excel.annotation.ExcelProperty; +import lombok.Data; + +/** + * excel表格手机号BO + * + * @author JiChenWang + * @since 2024/6/27 20:28 + */ +@Data +public class PhoneNumberBo { + + @ExcelProperty(value = "手机号码", index = 0) + private String phoneNumber; + +} diff --git a/src/main/java/com/example/snailjob/bo/PhoneNumberCheckBo.java b/src/main/java/com/example/snailjob/bo/PhoneNumberCheckBo.java new file mode 100644 index 0000000..caa8adb --- /dev/null +++ b/src/main/java/com/example/snailjob/bo/PhoneNumberCheckBo.java @@ -0,0 +1,40 @@ +package com.example.snailjob.bo; + +import com.example.snailjob.po.PhoneNumberPo; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.List; + +/** + * 手机号检测BO + * + * @author JiChenWang + * @since 2024/6/27 20:50 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class PhoneNumberCheckBo { + + @Schema(description = "检测总条数", accessMode = Schema.AccessMode.READ_WRITE) + private Long checkTotalNum = 0L; + + @Schema(description = "检测失败条数", accessMode = Schema.AccessMode.READ_WRITE) + private Long checkErrorNum = 0L; + + @Schema(description = "检测成功条数", accessMode = Schema.AccessMode.READ_WRITE) + private Long checkSuccessNum = 0L; + + @Schema(description = "检测失败临时的数据", accessMode = Schema.AccessMode.READ_WRITE) + private List checkErrorPhoneNumberList = new ArrayList<>(); + + @Schema(description = "检测成功临时的数据", accessMode = Schema.AccessMode.READ_WRITE) + private List checkSuccessPhoneNumberList = new ArrayList<>(); + +} diff --git a/src/main/java/com/example/snailjob/dao/PhoneNumberBaseMapper.java b/src/main/java/com/example/snailjob/dao/PhoneNumberBaseMapper.java new file mode 100644 index 0000000..fe36b1d --- /dev/null +++ b/src/main/java/com/example/snailjob/dao/PhoneNumberBaseMapper.java @@ -0,0 +1,15 @@ +package com.example.snailjob.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.example.snailjob.po.PhoneNumberPo; +import org.springframework.stereotype.Repository; + +/** + * 手机号mapper + * + * @author JiChenWang + * @since 2024/6/30 11:55 + */ +@Repository +public interface PhoneNumberBaseMapper extends BaseMapper { +} diff --git a/src/main/java/com/example/snailjob/dao/PhoneNumberDao.java b/src/main/java/com/example/snailjob/dao/PhoneNumberDao.java new file mode 100644 index 0000000..18d71dd --- /dev/null +++ b/src/main/java/com/example/snailjob/dao/PhoneNumberDao.java @@ -0,0 +1,34 @@ +package com.example.snailjob.dao; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.example.snailjob.po.PhoneNumberPo; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * TODO + * + * @author JiChenWang + * @since 2024/6/30 11:58 + */ +@Service +public class PhoneNumberDao extends ServiceImpl { + + @Autowired + private PhoneNumberBaseMapper phoneNumberBaseMapper; + + /** + * 批量保存手机号信息 + * + * @param phoneNumberPoList 手机号po列表 + * @return Boolean 保存成功标识：true-成功、false-失败 + * @author JichenWang + * @since 2024/6/30 12:03 + */ + public Boolean insertBatch (List phoneNumberPoList) { + return this.saveBatch(phoneNumberPoList); + } + +} diff --git a/src/main/java/com/example/snailjob/job/TestExcelAnalyseMapJobExecutor.java b/src/main/java/com/example/snailjob/job/TestExcelAnalyseMapJobExecutor.java new file mode 100644 index 0000000..3b577dd --- /dev/null +++ b/src/main/java/com/example/snailjob/job/TestExcelAnalyseMapJobExecutor.java @@ -0,0 +1,97 @@ +package com.example.snailjob.job; + +import cn.hutool.core.util.ObjectUtil; +import com.aizuda.snailjob.client.job.core.MapHandler; +import com.aizuda.snailjob.client.job.core.annotation.JobExecutor; +import com.aizuda.snailjob.client.job.core.annotation.MapExecutor; +import com.aizuda.snailjob.client.job.core.dto.MapArgs; +import com.aizuda.snailjob.client.model.ExecuteResult; +import com.alibaba.excel.EasyExcel; +import com.example.snailjob.bo.PhoneNumberBo; +import com.example.snailjob.bo.PhoneNumberCheckBo; +import com.example.snailjob.dao.PhoneNumberDao; +import com.example.snailjob.listener.PhoneNumberExcelListener; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.InputStream; +import java.util.List; + +/** + * 解析手机号excel文件，并将正确的手机号分片入库 + * + * @author JiChenWang + * @since 2024/6/30 10:37 + */ +@Slf4j +@Component +@JobExecutor(name = "testExcelAnalyseMapJobExecutor") +public class TestExcelAnalyseMapJobExecutor { + + private final Integer BATCH_SIZE = 100; + + @Autowired + private PhoneNumberDao phoneNumberDao; + + /** + * 读取手机号文件总行数，并进行分组 + * 比如文档中的手机号总量为307条，每100条一个分组，分组结果为[{0,99}, {100, 199}, {200,299}, {300, 307}] + * + * @return ExecuteResult + * @author JichenWang + * @since 2024/6/30 10:48 + */ + @MapExecutor + public ExecuteResult rootMapExecute(MapArgs mapArgs, MapHandler mapHandler) { + List> ranges = null; + // 先获取文件总行数，便于分组 + try { + @Cleanup InputStream numberInputStream = getClass().getClassLoader().getResourceAsStream("doc/number.xlsx"); + final PhoneNumberCheckBo phoneNumberCheckBo = new PhoneNumberCheckBo(); + PhoneNumberExcelListener phoneNumberExcelListener = new PhoneNumberExcelListener(phoneNumberCheckBo, true, BATCH_SIZE); + EasyExcel.read(numberInputStream, PhoneNumberBo.class, phoneNumberExcelListener).sheet().headRowNumber(1).doReadSync(); + + // 设置区间范围 + ranges = TestMapReduceJobExecutor.doSharding(0L, phoneNumberCheckBo.getCheckTotalNum(), BATCH_SIZE); + } catch (Exception e) { + log.error("文件读取异常", e.getMessage()); + } + return mapHandler.doMap(ranges, "MONTH_MAP"); + + } + + /** + * + * + * @param mapArgs + * @return ExecuteResult + * @author JichenWang + * @since 2024/6/30 11:05 + */ + @MapExecutor(taskName = "MONTH_MAP") + public ExecuteResult monthMapExecute(MapArgs mapArgs) { + // 获取本次要处理的区间 + final List mapResult = (List) mapArgs.getMapResult(); + log.info("本次要处理的区间为：{}", mapResult); + + // 按照处理区间，去读取数据 + final PhoneNumberCheckBo phoneNumberCheckBo = new PhoneNumberCheckBo(); + try { + @Cleanup InputStream numberInputStream = getClass().getClassLoader().getResourceAsStream("doc/number.xlsx"); + PhoneNumberExcelListener phoneNumberExcelListener = new PhoneNumberExcelListener(phoneNumberCheckBo, false, BATCH_SIZE); + EasyExcel.read(numberInputStream, PhoneNumberBo.class, phoneNumberExcelListener).sheet().headRowNumber(mapResult.get(0) + 1).doReadSync(); + } catch (Exception e) { + log.error("文件读取异常：", e.getMessage()); + } + + // 如果正确手机号不为空，则入库 + if (ObjectUtil.isNotEmpty(phoneNumberCheckBo.getCheckSuccessPhoneNumberList())) { + phoneNumberDao.insertBatch(phoneNumberCheckBo.getCheckSuccessPhoneNumberList()); + } + + return ExecuteResult.success(phoneNumberCheckBo.getCheckSuccessNum()); + } + +} diff --git a/src/main/java/com/example/snailjob/job/TestExcelAnalyseMapReduceJobExecutor.java b/src/main/java/com/example/snailjob/job/TestExcelAnalyseMapReduceJobExecutor.java new file mode 100644 index 0000000..aabf428 --- /dev/null +++ b/src/main/java/com/example/snailjob/job/TestExcelAnalyseMapReduceJobExecutor.java @@ -0,0 +1,132 @@ +package com.example.snailjob.job; + +import com.aizuda.snailjob.client.job.core.MapHandler; +import com.aizuda.snailjob.client.job.core.annotation.JobExecutor; +import com.aizuda.snailjob.client.job.core.annotation.MapExecutor; +import com.aizuda.snailjob.client.job.core.annotation.MergeReduceExecutor; +import com.aizuda.snailjob.client.job.core.annotation.ReduceExecutor; +import com.aizuda.snailjob.client.job.core.dto.MapArgs; +import com.aizuda.snailjob.client.job.core.dto.MergeReduceArgs; +import com.aizuda.snailjob.client.job.core.dto.ReduceArgs; +import com.aizuda.snailjob.client.model.ExecuteResult; +import com.alibaba.excel.EasyExcel; +import com.alibaba.fastjson.JSONArray; +import com.example.snailjob.bo.PhoneNumberBo; +import com.example.snailjob.bo.PhoneNumberCheckBo; +import com.example.snailjob.listener.PhoneNumberExcelListener; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * 解析校验Excel中的手机号，统计出错手机号数量，并返回错误手机号详情 + * + * @author JichenWang + * @since 2024/6/27 19:52 + */ +@Slf4j +@Component +@JobExecutor(name = "TestExcelAnalyseMapReduceJobExecutor") +public class TestExcelAnalyseMapReduceJobExecutor { + + private final Integer BATCH_SIZE = 100; + + /** + * 处理手机号文件信息，将文档中的手机号进行分组 + * 比如文档中的手机号总量为307条，每100条一个分组，分组结果为[{0,99}, {100, 199}, {200,299}, {300, 307}] + * + * @return ExecuteResult + * @author JichenWang + * @since 2024/6/29 14:03 + */ + @MapExecutor + public ExecuteResult rootMapExecute(MapArgs mapArgs, MapHandler mapHandler) { + List> ranges = null; + // 先获取文件总行数，便于分组 + try { + @Cleanup InputStream numberInputStream = getClass().getClassLoader().getResourceAsStream("doc/number.xlsx"); + final PhoneNumberCheckBo phoneNumberCheckBo = new PhoneNumberCheckBo(); + PhoneNumberExcelListener phoneNumberExcelListener = new PhoneNumberExcelListener(phoneNumberCheckBo, true, BATCH_SIZE); + EasyExcel.read(numberInputStream, PhoneNumberBo.class, phoneNumberExcelListener).sheet().headRowNumber(1).doReadSync(); + + // 设置区间范围 + ranges = TestMapReduceJobExecutor.doSharding(0L, phoneNumberCheckBo.getCheckTotalNum(), BATCH_SIZE); + } catch (Exception e) { + log.error("文件读取异常", e.getMessage()); + } + return mapHandler.doMap(ranges, "MONTH_MAP"); + } + + /** + * 处理每个分组内容，如读取{0,99}区间的手机号，并解析 + * + * @return ExecuteResult + * @author JichenWang + * @since 2024/6/29 14:04 + */ + @MapExecutor(taskName = "MONTH_MAP") + public ExecuteResult monthMapExecute(MapArgs mapArgs) { + // 获取本次要处理的区间 + final List mapResult = (List) mapArgs.getMapResult(); + log.info("本次要处理的区间为：{}", mapResult); + + // 按照处理区间，去读取数据 + final PhoneNumberCheckBo phoneNumberCheckBo = new PhoneNumberCheckBo(); + try { + @Cleanup InputStream numberInputStream = getClass().getClassLoader().getResourceAsStream("doc/number.xlsx"); + PhoneNumberExcelListener phoneNumberExcelListener = new PhoneNumberExcelListener(phoneNumberCheckBo, false, BATCH_SIZE); + EasyExcel.read(numberInputStream, PhoneNumberBo.class, phoneNumberExcelListener).sheet().headRowNumber(mapResult.get(0) + 1).doReadSync(); + } catch (Exception e) { + log.error("文件读取异常：", e.getMessage()); + } + + return ExecuteResult.success(phoneNumberCheckBo); + } + + + @ReduceExecutor + public ExecuteResult reduceExecute(ReduceArgs mapReduceArgs) { + log.info("WJC Test reduceExecute, 参数为：{}", mapReduceArgs.getMapResult()); + final PhoneNumberCheckBo phoneNumberCheckBo = this.buildGatherPhoneNumberCheckBo(mapReduceArgs.getMapResult().toString()); + return ExecuteResult.success(phoneNumberCheckBo); + } + + /** + * 当只有一个reduce任务时无此执行器 + */ + @MergeReduceExecutor + public ExecuteResult mergeReduceExecute(MergeReduceArgs mergeReduceArgs) { + final PhoneNumberCheckBo phoneNumberCheckBo = this.buildGatherPhoneNumberCheckBo(mergeReduceArgs.getReduces().toString()); + log.info("WJC 最终检测结果为：{}", phoneNumberCheckBo); + return ExecuteResult.success(phoneNumberCheckBo); + } + + /** + * 构造汇总手机号校验结果BO + * + * @param phoneNumberCheckBoStr 手机号校验BO字符串 + * @return PhoneNumberCheckBo 汇总手机号校验结果BO + * @author JichenWang + * @since 2024/6/29 14:24 + */ + private PhoneNumberCheckBo buildGatherPhoneNumberCheckBo(String phoneNumberCheckBoStr) { + final List phoneNumberCheckBoList = JSONArray.parseArray(phoneNumberCheckBoStr, PhoneNumberCheckBo.class); + // 获取校验总数 + final long checkTotalNum = phoneNumberCheckBoList.get(0).getCheckTotalNum(); + // 汇总校验失败数量 + final long checkErrorNum = phoneNumberCheckBoList.stream().mapToLong(PhoneNumberCheckBo::getCheckErrorNum).sum(); + // 汇总校验成功数量 + final long checkSuccessNum = phoneNumberCheckBoList.stream().mapToLong(PhoneNumberCheckBo::getCheckSuccessNum).sum(); + // 汇总错误手机号 + final List errorPhoneNumberList = new ArrayList<>(); + phoneNumberCheckBoList.forEach(item -> errorPhoneNumberList.addAll(item.getCheckErrorPhoneNumberList())); + + // 汇总手机号校验结果 + return PhoneNumberCheckBo.builder().checkTotalNum(checkTotalNum).checkErrorNum(checkErrorNum).checkSuccessNum(checkSuccessNum).checkErrorPhoneNumberList(errorPhoneNumberList).build(); + } + +} diff --git a/src/main/java/com/example/snailjob/listener/PhoneNumberExcelListener.java b/src/main/java/com/example/snailjob/listener/PhoneNumberExcelListener.java new file mode 100644 index 0000000..62bf08f --- /dev/null +++ b/src/main/java/com/example/snailjob/listener/PhoneNumberExcelListener.java @@ -0,0 +1,78 @@ +package com.example.snailjob.listener; + +import cn.hutool.core.lang.Validator; +import cn.hutool.core.util.ObjectUtil; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.example.snailjob.bo.PhoneNumberBo; +import com.example.snailjob.bo.PhoneNumberCheckBo; +import com.example.snailjob.po.PhoneNumberPo; +import lombok.extern.slf4j.Slf4j; + +import java.time.LocalDateTime; +import java.util.Map; + +/** + * 手机号excel解析 Listener + * + * @author JiChenWang + * @since 2024/6/27 20:38 + */ +@Slf4j +public class PhoneNumberExcelListener extends AnalysisEventListener { + + /** 手机号校验BO **/ + private PhoneNumberCheckBo phoneNumberCheckBo; + + /** 是否第一次读取Excel **/ + private Boolean firstReadStatus = false; + + /** 读取批次大小 **/ + private Integer batchSize = 100; + + /** 已读取的数据数量 **/ + private Integer cacheSize = 0; + + @Override + public void invokeHeadMap(Map headMap, AnalysisContext context) { + this.phoneNumberCheckBo.setCheckTotalNum(Long.parseLong(String.valueOf(context.readSheetHolder().getApproximateTotalRowNumber() - 1))); + } + + @Override + public void invoke(PhoneNumberBo phoneNumberBo, AnalysisContext context) { + // 如果是第一次读该文件，已读取的数量已超过读取批次大小，直接返回 + if (firstReadStatus || cacheSize >= batchSize) { + return; + } + + cacheSize++; + + if (ObjectUtil.isEmpty(phoneNumberBo.getPhoneNumber())) { + return; + } + + // 校验手机号 + log.info("本次校验的手机号为: {}", phoneNumberBo.getPhoneNumber()); + Boolean validateStatus = Validator.isMobile(phoneNumberBo.getPhoneNumber()); + if (validateStatus) { + this.phoneNumberCheckBo.setCheckSuccessNum(this.phoneNumberCheckBo.getCheckSuccessNum() + 1); + final PhoneNumberPo phoneNumberPo = PhoneNumberPo.builder().phoneNumber(phoneNumberBo.getPhoneNumber()).createTime(LocalDateTime.now()).build(); + this.phoneNumberCheckBo.getCheckSuccessPhoneNumberList().add(phoneNumberPo); + } else { + this.phoneNumberCheckBo.setCheckErrorNum(this.phoneNumberCheckBo.getCheckErrorNum() + 1); + this.phoneNumberCheckBo.getCheckErrorPhoneNumberList().add(phoneNumberBo.getPhoneNumber()); + } + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + + } + + public PhoneNumberExcelListener(PhoneNumberCheckBo phoneNumberCheckBo, Boolean firstReadStatus, Integer batchSize) { + this.phoneNumberCheckBo = phoneNumberCheckBo; + this.firstReadStatus = firstReadStatus; + this.batchSize = batchSize; + } + +} diff --git a/src/main/java/com/example/snailjob/po/PhoneNumberPo.java b/src/main/java/com/example/snailjob/po/PhoneNumberPo.java new file mode 100644 index 0000000..6372220 --- /dev/null +++ b/src/main/java/com/example/snailjob/po/PhoneNumberPo.java @@ -0,0 +1,39 @@ +package com.example.snailjob.po; + +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +/** + * phone_number + * + * @author JiChenWang + * @since 2024/6/30 11:48 + */ +@TableName("phone_number") +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class PhoneNumberPo { + + /** + * 主键 + */ + private Long id; + + /** + * 手机号 + */ + private String phoneNumber; + + /** + * 创建时间 + */ + private LocalDateTime createTime; + +} diff --git a/src/main/resources/doc/number.xlsx b/src/main/resources/doc/number.xlsx new file mode 100644 index 0000000..4653ed0 Binary files /dev/null and b/src/main/resources/doc/number.xlsx differ