Spring Batch mybatis 예제 - Spring Batch mybatis yeje

프로젝트의 목적은 많은 수의 데이터를 insert, update 할 때 얼마나 빨리 할 수 있는가 이다.

대상은 Spring SqlSesssion, Mybatis foreach이다.

배치 데이터 수는 10만개, 루프당 데이터 수는 1000개 이다.

- Spring boot (2.2.2.RELEASE)

- Mybatis

- H2 Database

- Spring AOP

- SpringSession 배치

- Mybatis foreach 배치

- SpringSession + AOP 배치

- Mybatis foreach + AOP 배치

​

추가적으로 Custom Annotation을 만들어 AOP에 적용하는 것도 테스트 하였다.

1. SqlSession이 속도가 제일 빠르다.

2. Mybatis foreach가 속도는 제일 빠를거라 생각했지만 생각보다 많~~~이 느렸다.

(foreach로 쿼리 생산하는 속도가 느린것으로 생각됨)

3. AOP가 전체적으로 속도를 느리게 하는 건가(?), 잘모르겠다...이건

(아마도 포인트컷 조건을 체크하면서 더 느려지는 것으로 생각됨)

package com.mybatis.batch.aspect; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import org.springframework.util.StopWatch; import java.util.*; @Aspect @Component public class BatchAspect { // Logger private final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); /** * 데이터 생성 advice * * @param pjp the pjp * @return the object * @throws Throwable the throwable */ @Around("@annotation(com.mybatis.batch.annotation.CreateData)") public Object CreateData(ProceedingJoinPoint pjp) throws Throwable { String name = pjp.getSignature().getName(); StopWatch stopWatch = new StopWatch(); stopWatch.start(); LOGGER.info("@@@@@@@@@@@@@@@@@@@@"); LOGGER.info(name + " 배치 데이터 등록 시작")ㅣ Object retVal = null; try { int totalCount = 100000; int perPageNum = 1000; LOGGER.info("전체 데이터 개수 : " + totalCount); LOGGER.info("루프당 데이터 개수 : " + perPageNum); int maxPageNum = (int) Math.ceil(totalCount / perPageNUm); maxPageNum = maxPageNum == 0 ? 1: maxPageNum; if (totalCount < perPageNum) { perPageNum = totalCount; } List<Map<String, Object>> paramList = new ArrayList<>(); String[] nameArr = {Aadan","Aaliah","Aalleeyaa","Aaric","Ab-Owen","Abadine","Abagael","Abagale","Abbagail","Abbegail","Abbeygail","Abbi","Abbigail","Abbigayle","Abbot","Abbott","Aberfa","Abergavenny","Abgail","Abhorson","Abigale","Abrielle","Acca","Acennan","Acey","Acker","Ackerley","Ackerman","Ackley","Acton","Acwel","Acwellen","Adain","Adalbeorht","Adalbrechta","Adalson","Adam","Adamm","Adamson","Addeline","Addis","Addison","Adelaido","Adelyn","Adia","Adken","Adkins","Adler","Adney","Adny","Adonna","Adram","Adrean","Adrena","Adrenalijna","Adriann","Adrieanna","Adrin","Adrion","Adriona","Adron","Adyn","Adyna","Adyson","Aeccestane","Aedd","Aedre","Aefentid","Aefre","Aeker","Aekerman","Aelfraed","Aelfthryth","Aelfwine","Aelita","Aelle","Aelwen","Aenedleah","Aenescumb","Aeoelric","Aeric","Aerica","Aerlene","Aerona","Aesclin","Aesoburne","Aethelberht","Aethelbert","Aethelind","Aethelred","Aethelweard","Aethelwine","Aethelwulf","Aetheston","Agness","Ahebban","Ahreddan","Aiken","Aikin","Ailen"}; Random random = new Random(); for (int i = 0; i < maxPageNum; i++) { for (int j = 0; j < perPageNum; j++) { Map<String, Object> param = new HashMap<>(); param.put("name", nameArr[(int) (Math.random() * 100)]); param.put("sex", random.nextInt(3)); paramList.add(param); if (j == 999 || (i == maxPageNum - 1 && j == totalCount - maxPageNum * perPageNum)) { retVal = pjp.proceed(new Object[]{ paramList }); } } } } finally { stopWatch.stop(); LOGGER.info(name + " 배치 데이터 등록 종료 : " + (int) Math.ceil(stopWatch.getTotalTimeSeconds()) + "초"); LOGGER.info("@@@@@@@@@@@@@@@@@@@@@@@@@@"); } return retVal; } }

package com.mybatis.batch.service; import com.mybatis.batch.annotation.CreateData; import org.apache.ibatis.session.ExecutorType; import org.apache.ibatis.session.SqlSession; import org.apache.ibatis.session.SqlSessionFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import java.util.*; @Service public class BatchService { private final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); // Sql SessionFactory ( 배치 처리 ) private final SqlSessionFactory sqlSessionFactory; /** * Instantiates a new Batch service. * * @param sqlSessionFactory the sql session factory */ public BatchService(SqlSessionFactory sqlSessionFactory){ this.sqlSessionFactory = sqlSessionFactory; } /** * sqlSession 배치 처리 ( 데이터 생성은 AOP로 구현 ) * * @param paramList the param list */ @CreateData public void sqlSessionBatchWithAop(List<Map<String, Object>> paramList) { SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH); try { for (Map<String, Object> param : paramList) { sqlSession.insert("com.mybatis.batch.mapper.BatchMapper.insertSqlSession", param); } } finally { sqlSession.flushStatements(); sqlSession.close(); sqlSession.clearCache(); } } /** * forEach 배치 처리 ( 데이터 생성은 AOP로 구현 ) * * @param paramList the param list */ @CreateData public void forEachBatchWithAop(List<Map<String, Object>> paramList) { SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH); try { Map<String, Object> param = new HashMap<>(); param.put("memberList", paramList); sqlSession.insert("com.mybatis.batch.mapper.BatchMapper.insertForEach", param); } finally { sqlSession.flushStatements(); sqlSession.close(); sqlSession.clearCache(); } } /** * sqlSession 배치 처리 */ public void sqlSessionBatch() { LOGGER.info("트랜젝션 sqlSession 배치 테스트 시작"); int totalCount = 100000; int perPageNum = 1000; int maxPageNum = (int) Math.ceil(totalCount / perPageNum); maxPageNum = maxPageNum == 0 ? 1 : maxPageNum; String[] nameArr = {"Aadan","Aaliah","Aalleeyaa","Aaric","Ab-Owen","Abadine","Abagael","Abagale","Abbagail","Abbegail","Abbeygail","Abbi","Abbigail","Abbigayle","Abbot","Abbott","Aberfa","Abergavenny","Abgail","Abhorson","Abigale","Abrielle","Acca","Acennan","Acey","Acker","Ackerley","Ackerman","Ackley","Acton","Acwel","Acwellen","Adain","Adalbeorht","Adalbrechta","Adalson","Adam","Adamm","Adamson","Addeline","Addis","Addison","Adelaido","Adelyn","Adia","Adken","Adkins","Adler","Adney","Adny","Adonna","Adram","Adrean","Adrena","Adrenalijna","Adriann","Adrieanna","Adrin","Adrion","Adriona","Adron","Adyn","Adyna","Adyson","Aeccestane","Aedd","Aedre","Aefentid","Aefre","Aeker","Aekerman","Aelfraed","Aelfthryth","Aelfwine","Aelita","Aelle","Aelwen","Aenedleah","Aenescumb","Aeoelric","Aeric","Aerica","Aerlene","Aerona","Aesclin","Aesoburne","Aethelberht","Aethelbert","Aethelind","Aethelred","Aethelweard","Aethelwine","Aethelwulf","Aetheston","Agness","Ahebban","Ahreddan","Aiken","Aikin","Ailen"}; List<Map<String, Object>> paramList = new ArrayList<>(); LOGGER.info("트랜젝션 배치 전체 데이터 수 : " + totalCount); LOGGER.info("트랜젝션 배치 루프 당 데이터 수 : " + perPageNum); long startTime = System.currentTimeMillis(); Random random = new Random(); for (int i = 0; i < maxPageNum; i++) { for (int j = 0; j < perPageNum; j++) { Map<String, Object> param = new HashMap<>(); param.put("name", nameArr[(int) (Math.random() * 100)]); param.put("sex", random.nextInt(3)); paramList.add(param); if (j == 999 || (i == maxPageNum - 1 && j == totalCount - maxPageNum * perPageNum)) { SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH); try { Map<String, Object> params = new HashMap<>(); param.put("memberList", paramList); sqlSession.insert("com.mybatis.batch.mapper.BatchMapper.insertSqlSession", param); } finally { sqlSession.flushStatements(); sqlSession.close(); sqlSession.clearCache(); } paramList.clear(); } } } long endTime = System.currentTimeMillis(); long resultTime = endTime - startTime; LOGGER.info("트랜젝션 sqlSession 배치 테스트 종료"); LOGGER.info("트랜젝션 sqlSession 배치" + " 소요시간 : " + resultTime/1000 + "초("+resultTime+")"); }

/** * forEach 배치 처리 */ public void forEachBatch() { LOGGER.info("트랜젝션 forEach 배치 테스트 시작"); int totalCount = 100000; int perPageNum = 1000; int maxPageNum = (int) Math.ceil(totalCount / perPageNum); maxPageNum = maxPageNum == 0 ? 1 : maxPageNum; String[] nameArr = {"Aadan","Aaliah","Aalleeyaa","Aaric","Ab-Owen","Abadine","Abagael","Abagale","Abbagail","Abbegail","Abbeygail","Abbi","Abbigail","Abbigayle","Abbot","Abbott","Aberfa","Abergavenny","Abgail","Abhorson","Abigale","Abrielle","Acca","Acennan","Acey","Acker","Ackerley","Ackerman","Ackley","Acton","Acwel","Acwellen","Adain","Adalbeorht","Adalbrechta","Adalson","Adam","Adamm","Adamson","Addeline","Addis","Addison","Adelaido","Adelyn","Adia","Adken","Adkins","Adler","Adney","Adny","Adonna","Adram","Adrean","Adrena","Adrenalijna","Adriann","Adrieanna","Adrin","Adrion","Adriona","Adron","Adyn","Adyna","Adyson","Aeccestane","Aedd","Aedre","Aefentid","Aefre","Aeker","Aekerman","Aelfraed","Aelfthryth","Aelfwine","Aelita","Aelle","Aelwen","Aenedleah","Aenescumb","Aeoelric","Aeric","Aerica","Aerlene","Aerona","Aesclin","Aesoburne","Aethelberht","Aethelbert","Aethelind","Aethelred","Aethelweard","Aethelwine","Aethelwulf","Aetheston","Agness","Ahebban","Ahreddan","Aiken","Aikin","Ailen"}; List<Map<String, Object>> paramList = new ArrayList<>(); LOGGER.info("트랜젝션 배치 전체 데이터 수 : " + totalCount); LOGGER.info("트랜젝션 배치 루프 당 데이터 수 : " + perPageNum); long startTime = System.currentTimeMillis(); Random random = new Random(); for (int i = 0; i < maxPageNum; i++) { for (int j = 0; j < perPageNum; j++) { Map<String, Object> param = new HashMap<>(); param.put("name", nameArr[(int) (Math.random() * 100)]); param.put("sex", random.nextInt(3)); paramList.add(param); if (j == 999 || (i == maxPageNum - 1 && j == totalCount - maxPageNum * perPageNum)) { SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH); try { Map<String, Object> params = new HashMap<>(); param.put("memberList", paramList); sqlSession.insert("com.mybatis.batch.mapper.BatchMapper.insertForEach", param); } finally { sqlSession.flushStatements(); sqlSession.close(); sqlSession.clearCache(); } paramList.clear(); } } } long endTime = System.currentTimeMillis(); long resultTime = endTime - startTime; LOGGER.info("트랜젝션 forEach 배치 테스트 종료"); LOGGER.info("트랜젝션 forEach 배치" + " 소요시간 : " + resultTime/1000 + "초("+resultTime+")"); } }

foreach AOP 테스트는 1분이상이 걸려 테스트에서 주석처리 후 진행했다.

​