From c495c248bbfd4965b7a61d6210feb09a2f3ad3d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EB=AA=85=EC=A4=80?= <86913355+mjj111@users.noreply.github.com> Date: Fri, 25 Apr 2025 15:47:20 +0900 Subject: [PATCH 1/3] =?UTF-8?q?[IDLE-567]=20=EC=BF=BC=EB=A6=AC=20=EC=98=A4?= =?UTF-8?q?=ED=83=80=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/swm/idle/domain/chat/repository/ChatRoomRepository.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/idle-domain/src/main/kotlin/com/swm/idle/domain/chat/repository/ChatRoomRepository.kt b/idle-domain/src/main/kotlin/com/swm/idle/domain/chat/repository/ChatRoomRepository.kt index 7141d716..e85661be 100644 --- a/idle-domain/src/main/kotlin/com/swm/idle/domain/chat/repository/ChatRoomRepository.kt +++ b/idle-domain/src/main/kotlin/com/swm/idle/domain/chat/repository/ChatRoomRepository.kt @@ -30,7 +30,7 @@ interface ChatRoomRepository : JpaRepository { FROM chat_message cm WHERE cm.chat_room_id IN (SELECT chat_room_id FROM FilteredChatRooms) AND cm.is_read = false - AND cm.receiverId = :userId + AND cm.receiver_id = :userId GROUP BY cm.chat_room_id ) @@ -70,7 +70,7 @@ interface ChatRoomRepository : JpaRepository { FROM chat_message cm WHERE cm.chat_room_id IN (SELECT chat_room_id FROM FilteredChatRooms) AND cm.is_read = false - AND cm.receiverId = :userId + AND cm.receiver_id = :userId GROUP BY cm.chat_room_id ) From 3b4cf466f5cab00d1175d7bb35a2bd432f5a9796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EB=AA=85=EC=A4=80?= <86913355+mjj111@users.noreply.github.com> Date: Fri, 25 Apr 2025 15:48:04 +0900 Subject: [PATCH 2/3] =?UTF-8?q?[IDLE-568]=20pageIndex=20=EC=B4=88=EA=B8=B0?= =?UTF-8?q?=ED=99=94=20=EC=98=A4=EB=A5=98=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/swm/idle/batch/step/PostingReader.kt | 44 ++++++------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/idle-batch/src/main/kotlin/com/swm/idle/batch/step/PostingReader.kt b/idle-batch/src/main/kotlin/com/swm/idle/batch/step/PostingReader.kt index c5468579..54f8fed2 100644 --- a/idle-batch/src/main/kotlin/com/swm/idle/batch/step/PostingReader.kt +++ b/idle-batch/src/main/kotlin/com/swm/idle/batch/step/PostingReader.kt @@ -1,46 +1,30 @@ package com.swm.idle.batch.step import com.swm.idle.batch.common.dto.CrawledJobPostingDto -import com.swm.idle.batch.crawler.CrawlerConsts -import com.swm.idle.batch.crawler.WorknetPageCrawler -import com.swm.idle.batch.crawler.WorknetPostCrawler +import com.swm.idle.batch.crawler.WorknetPageParser +import com.swm.idle.batch.crawler.WorknetPostParser import org.springframework.batch.item.ItemStreamReader import java.util.concurrent.atomic.AtomicInteger -class PostingReader : ItemStreamReader> { - var crawlingUrl: String = "" - var postingCount: Int = 1 - var lastPageJobPostingCount: Int = 1 - var pageCount: Int = 1 - - init { - WorknetPageCrawler().initCounts(this) - } +class PostingReader: ItemStreamReader> { + val pageParser: WorknetPageParser = WorknetPageParser() companion object { - var nextPage = AtomicInteger(1) + var pageIndex = AtomicInteger(1) } override fun read(): List? { - val currentPage = nextPage.getAndIncrement() - if (currentPage > pageCount) return null + val accessIndex = pageIndex.getAndIncrement() - val amount = getAmount(currentPage) - val nowUrl = getPageUrl(currentPage) - - return WorknetPostCrawler().crawlPosts(amount, nowUrl) - } - - private fun getAmount(currentPage: Int): Int { - if (currentPage == pageCount && lastPageJobPostingCount > 0) { - return lastPageJobPostingCount + if (pageParser.isOverPage(accessIndex)) { + pageIndex = AtomicInteger(1) + return null } - return CrawlerConsts.JOB_POSTING_COUNT_PER_PAGE.getIntValue() - } - private fun getPageUrl(currentPage: Int): String { - return crawlingUrl.replace( - Regex("pageIndex=\\d+"), - "pageIndex=$currentPage") + val fetchCount = pageParser.getFetchCount(accessIndex) + val accessURL = pageParser.getAccessURL(accessIndex) + val posts = WorknetPostParser().crawlPosts(fetchCount, accessURL) + + return posts } } \ No newline at end of file From 423ccb6e79cc843279b5f82064444e5e4ec5f0bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EB=AA=85=EC=A4=80?= <86913355+mjj111@users.noreply.github.com> Date: Fri, 25 Apr 2025 15:48:58 +0900 Subject: [PATCH 3/3] =?UTF-8?q?[IDLE-568]=20=EB=B0=B0=EC=B9=98=20=EA=B4=80?= =?UTF-8?q?=EB=A0=A8=20=EC=BD=94=EB=93=9C=20=EC=8A=A4=ED=83=80=EC=9D=BC=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../swm/idle/batch/crawler/ErrorRecorder.kt | 19 +++ .../idle/batch/crawler/WorknetPageCrawler.kt | 46 ------ .../idle/batch/crawler/WorknetPageParser.kt | 96 ++++++++++++ .../idle/batch/crawler/WorknetPostCrawler.kt | 142 ----------------- .../idle/batch/crawler/WorknetPostParser.kt | 145 ++++++++++++++++++ .../com/swm/idle/batch/job/JobConfig.kt | 3 - 6 files changed, 260 insertions(+), 191 deletions(-) create mode 100644 idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/ErrorRecorder.kt delete mode 100644 idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPageCrawler.kt create mode 100644 idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPageParser.kt delete mode 100644 idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPostCrawler.kt create mode 100644 idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPostParser.kt diff --git a/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/ErrorRecorder.kt b/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/ErrorRecorder.kt new file mode 100644 index 00000000..9fda5fef --- /dev/null +++ b/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/ErrorRecorder.kt @@ -0,0 +1,19 @@ +package com.swm.idle.batch.crawler + +import io.github.oshai.kotlinlogging.KotlinLogging + +class ErrorRecorder { + + private val errorCounts: MutableMap = mutableMapOf() + private val logger = KotlinLogging.logger {} + + fun recordError(location: String) { + errorCounts[location] = errorCounts.getOrDefault(location, 0) + 1 + } + + fun printErrors() { + errorCounts.forEach { (location, count) -> + println("Error at $location: $count occurrences") + } + } +} \ No newline at end of file diff --git a/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPageCrawler.kt b/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPageCrawler.kt deleted file mode 100644 index d6641215..00000000 --- a/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPageCrawler.kt +++ /dev/null @@ -1,46 +0,0 @@ -package com.swm.idle.batch.crawler - -import com.swm.idle.batch.step.PostingReader -import org.openqa.selenium.By -import org.openqa.selenium.WebDriver -import org.openqa.selenium.support.ui.ExpectedConditions -import org.openqa.selenium.support.ui.WebDriverWait -import java.time.Duration -import java.time.LocalDate -import java.time.format.DateTimeFormatter - -class WorknetPageCrawler { - private var driver: WebDriver = DriverInitializer.init() - - fun initCounts(reader: PostingReader) { - reader.crawlingUrl = CrawlerConsts.CRAWLING_TARGET_URL_FORMAT.value - .replace("{yesterday}", LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"))) - .replace("{pageIndex}", "1") - - moveToPage(reader) - - reader.postingCount = driver - .findElement(By.xpath(CrawlerConsts.JOB_POSTING_COUNT.value)) - .text - .replace(",", "") - .toInt() - .takeIf { it > 0 } - ?: run { - driver.quit() - throw Exception("크롤링 할 공고가 없습니다.") - } - - reader.pageCount = (reader.postingCount + CrawlerConsts.JOB_POSTING_COUNT_PER_PAGE.getIntValue() - 1) / - CrawlerConsts.JOB_POSTING_COUNT_PER_PAGE.getIntValue() - reader.lastPageJobPostingCount = reader.postingCount % CrawlerConsts.JOB_POSTING_COUNT_PER_PAGE.getIntValue() - driver.quit() - } - - private fun moveToPage(reader: PostingReader) { - driver.get(reader.crawlingUrl) - WebDriverWait(driver, Duration.ofSeconds(10)) - .also { - it.until(ExpectedConditions.visibilityOfElementLocated(By.xpath(CrawlerConsts.JOB_POSTING_COUNT.value))) - } - } -} \ No newline at end of file diff --git a/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPageParser.kt b/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPageParser.kt new file mode 100644 index 00000000..ac4ae0b8 --- /dev/null +++ b/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPageParser.kt @@ -0,0 +1,96 @@ +package com.swm.idle.batch.crawler + +import io.github.oshai.kotlinlogging.KotlinLogging +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.support.ui.ExpectedConditions +import org.openqa.selenium.support.ui.WebDriverWait +import java.time.Duration +import java.time.LocalDate +import java.time.format.DateTimeFormatter + +class WorknetPageParser { + private var postingCount = 0 + private var crawlingUrl: String = "" + private var lastPageJobPostingCount: Int = 1 + private var pageCount: Int = 1 + + private val logger = KotlinLogging.logger { } + + init { + val driver = DriverInitializer.init() + driver.safeUse { + getCrawlingURL() + moveToPage(driver) + getPostingCount(driver) + calculatePageInfo() + LoggingPageResult() + } + } + + fun isOverPage(currentPage: Int): Boolean { + return currentPage > pageCount + } + + fun getFetchCount(currentPage: Int): Int { + if (currentPage == pageCount && lastPageJobPostingCount > 0) { + return lastPageJobPostingCount + } + return CrawlerConsts.JOB_POSTING_COUNT_PER_PAGE.getIntValue() + } + + fun getAccessURL(currentPage: Int): String { + return crawlingUrl.replace( + Regex("pageIndex=\\d+"), + "pageIndex=$currentPage") + } + + private fun getCrawlingURL() { + crawlingUrl = CrawlerConsts.CRAWLING_TARGET_URL_FORMAT.value + .replace("{yesterday}", LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"))) + .replace("{pageIndex}", "1") + } + + private fun moveToPage(driver: WebDriver) { + driver.get(crawlingUrl) + WebDriverWait(driver, Duration.ofSeconds(10)) + .until(ExpectedConditions.visibilityOfElementLocated( + By.xpath( + CrawlerConsts.JOB_POSTING_COUNT.value + ))) + } + + private fun getPostingCount(driver: WebDriver) { + postingCount = driver + .findElement(By.xpath(CrawlerConsts.JOB_POSTING_COUNT.value)) + .text + .replace(",", "") + .toInt() + .takeIf { it > 0 } + ?: run { + throw Exception("크롤링 할 공고가 없습니다.") + } + } + + private fun calculatePageInfo() { + pageCount = + (postingCount + CrawlerConsts.JOB_POSTING_COUNT_PER_PAGE.getIntValue() - 1) / + CrawlerConsts.JOB_POSTING_COUNT_PER_PAGE.getIntValue() + lastPageJobPostingCount = + postingCount % CrawlerConsts.JOB_POSTING_COUNT_PER_PAGE.getIntValue() + } + + private fun LoggingPageResult() { + logger.info { "PageCount : ${pageCount}" } + logger.info { "PostingCount : ${postingCount}" } + logger.info { "LastPage JobCount : ${lastPageJobPostingCount}" } + } + + private fun WebDriver.safeUse(block: (WebDriver) -> T): T { + try { + return block(this) + } finally { + this.quit() + } + } +} diff --git a/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPostCrawler.kt b/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPostCrawler.kt deleted file mode 100644 index 9e3f37c5..00000000 --- a/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPostCrawler.kt +++ /dev/null @@ -1,142 +0,0 @@ -package com.swm.idle.batch.crawler - -import com.swm.idle.batch.common.dto.CrawledJobPostingDto -import io.github.oshai.kotlinlogging.KotlinLogging -import org.openqa.selenium.By -import org.openqa.selenium.WebDriver -import org.openqa.selenium.support.ui.ExpectedConditions -import org.openqa.selenium.support.ui.WebDriverWait -import java.time.Duration -import java.time.LocalDate -import java.time.format.DateTimeFormatter -import org.openqa.selenium.WebElement - -class WorknetPostCrawler { - private val logger = KotlinLogging.logger { } - private var driver: WebDriver = DriverInitializer.init() - private var errorCountMap: MutableMap = mutableMapOf() - - fun crawlPosts(end: Int, url: String): List { - moveToPage(url) - - val crawledPostings = mutableListOf() - repeat(end) { i -> - val originalWindow = driver.windowHandle - val titleElement = findElementSafe(By.xpath("//*[@id=\"list${i+1}\"]/td[1]/div/div[2]/a")) ?: return@repeat - - moveToPostDetailWindow(titleElement, originalWindow) - - try { - val post: CrawledJobPostingDto = createPost() - crawledPostings.add(post) - } catch (e: Exception) { - logger.warn { "실패" } - } - - backWindow(originalWindow) - } - errorCountMap.asSequence().forEach { (key, value) -> println("$key -> $value") } - driver.quit() - return crawledPostings - } - - private fun moveToPage(url: String) { - driver.get(url) - WebDriverWait(driver, Duration.ofSeconds(10)) - .until( - ExpectedConditions.visibilityOfElementLocated(By.cssSelector("#list1")) - ) - } - - private fun createPost(): CrawledJobPostingDto { - return CrawledJobPostingDto( - title = extractText(CrawlerConsts.TITLE), - content = extractText(CrawlerConsts.CONTENT), - createdAt = extractText(CrawlerConsts.CREATED_AT), - payInfo = extractText(CrawlerConsts.PAY_INFO), - workSchedule = extractText(CrawlerConsts.WORK_SCHEDULE), - recruitmentProcess = extractText(CrawlerConsts.RECRUITMENT_PROCESS), - applyMethod = extractText(CrawlerConsts.APPLY_METHOD), - requiredDocument = extractText(CrawlerConsts.REQUIRED_DOCUMENT), - centerName = extractText(CrawlerConsts.CENTER_NAME), - applyDeadline = extractApplyDeadline(CrawlerConsts.APPLY_DEADLINE), - workTime = extractWorkTime(CrawlerConsts.WORK_TIME), - centerAddress = extractAddress( - CrawlerConsts.CLIENT_ADDRESS1, - CrawlerConsts.CLIENT_ADDRESS2 - ), - clientAddress = extractAddress( - CrawlerConsts.CENTER_ADDRESS1, - CrawlerConsts.CENTER_ADDRESS2, - CrawlerConsts.CENTER_ADDRESS3 - ), - directUrl = driver.currentUrl - ) - } - - - private inline fun errorRecord(location: String, action: () -> T): T { - return runCatching { action() } - .getOrElse { e -> - logError(location) - throw e - } - } - - private fun findElementSafe(by: By): WebElement? { - return runCatching { driver.findElement(by) }.getOrNull() - } - - private fun moveToPostDetailWindow(titleElement: WebElement, originalWindow: String) { - titleElement.click() - WebDriverWait(driver, Duration.ofSeconds(10)) - .until(ExpectedConditions.numberOfWindowsToBe(2)) - driver.switchTo().window(driver.windowHandles.first { it != originalWindow }) - } - - private fun extractText(con: CrawlerConsts): String { - return errorRecord(con.location) { driver.findElement(By.xpath(con.value)).text } - } - - private fun extractApplyDeadline(con: CrawlerConsts): String { - return errorRecord(con.location) { - driver.findElement(By.xpath(con.value)).text.let { - if (it.contains("채용시까지")) - LocalDate.now().plusDays(15).format(DateTimeFormatter.ofPattern("yyyyMMdd")) - else - it - } - } - } - - private fun extractAddress(vararg cons: CrawlerConsts): String { - for (con in cons) { - runCatching { - val address = driver.findElement(By.xpath(con.value)).text - return address.replace("지도보기", "").trim().replace(Regex("\\(\\d{5}\\)"), "").trim() - } .getOrElse { e -> - logError(con.location) - throw e - } - } - throw NoSuchElementException("Center address not found using any of the provided XPaths") - } - - private fun extractWorkTime(con: CrawlerConsts): String { - return errorRecord(con.location) { - driver.findElement(By.xpath(con.value)).text - .replace("도움말", "") - .replace("(근무시간)", "") - .replace("\n", "") - } - } - - private fun logError(location: String) { - errorCountMap[location] = errorCountMap.getOrDefault(location, 0) + 1 - } - - private fun backWindow(originalWindow: String?) { - driver.close() - driver.switchTo().window(originalWindow) - } -} \ No newline at end of file diff --git a/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPostParser.kt b/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPostParser.kt new file mode 100644 index 00000000..811e0b8c --- /dev/null +++ b/idle-batch/src/main/kotlin/com/swm/idle/batch/crawler/WorknetPostParser.kt @@ -0,0 +1,145 @@ +package com.swm.idle.batch.crawler + +import com.swm.idle.batch.common.dto.CrawledJobPostingDto +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.support.ui.ExpectedConditions +import org.openqa.selenium.support.ui.WebDriverWait +import java.time.Duration +import java.time.LocalDate +import java.time.format.DateTimeFormatter +import org.openqa.selenium.WebElement + +class WorknetPostParser { + private val errorRecorder = ErrorRecorder() + + fun crawlPosts(end: Int, url: String): List { + val result = mutableListOf() + val driver = DriverInitializer.init() + + driver.safeUse { + moveToPage(driver, url) + + for (index in 1..end) { + val originalWindow = driver.windowHandle + val titleElement = findElementOrNull(driver, By.xpath("//*[@id=\"list${index}\"]/td[1]/div/div[2]/a")) + ?: continue + + moveToPostDetailWindow(driver, titleElement, originalWindow) + putPostsToResult(driver, result) + backWindow(driver, originalWindow) + } + } + + errorRecorder.printErrors() + return result + } + + private fun WebDriver.safeUse(block: (WebDriver) -> T): T { + try { + return block(this) + } finally { + this.quit() + } + } + + private fun findElementOrNull(driver: WebDriver, by: By): WebElement? { + return runCatching { driver.findElement(by) }.getOrNull() + } + + private fun moveToPage(driver: WebDriver, url: String) { + driver.get(url) + WebDriverWait(driver, Duration.ofSeconds(10)) + .until( + ExpectedConditions.visibilityOfElementLocated(By.cssSelector("#list1")) + ) + } + + private fun moveToPostDetailWindow(driver: WebDriver, titleElement: WebElement, originalWindow: String) { + titleElement.click() + WebDriverWait(driver, Duration.ofSeconds(10)) + .until(ExpectedConditions.numberOfWindowsToBe(2)) + driver.switchTo().window(driver.windowHandles.first { it != originalWindow }) + } + + private fun putPostsToResult(driver: WebDriver, result: MutableList ) { + runCatching { + CrawledJobPostingDto( + title = extractText(driver,CrawlerConsts.TITLE), + content = extractText(driver, CrawlerConsts.CONTENT), + createdAt = extractText(driver, CrawlerConsts.CREATED_AT), + payInfo = extractText(driver, CrawlerConsts.PAY_INFO), + workSchedule = extractText(driver, CrawlerConsts.WORK_SCHEDULE), + recruitmentProcess = extractText(driver, CrawlerConsts.RECRUITMENT_PROCESS), + applyMethod = extractText(driver, CrawlerConsts.APPLY_METHOD), + requiredDocument = extractText(driver, CrawlerConsts.REQUIRED_DOCUMENT), + centerName = extractText(driver, CrawlerConsts.CENTER_NAME), + applyDeadline = extractApplyDeadline(driver, CrawlerConsts.APPLY_DEADLINE), + workTime = extractWorkTime(driver, CrawlerConsts.WORK_TIME), + centerAddress = extractAddress(driver, + CrawlerConsts.CLIENT_ADDRESS1, + CrawlerConsts.CLIENT_ADDRESS2 + ), + clientAddress = extractAddress(driver, + CrawlerConsts.CENTER_ADDRESS1, + CrawlerConsts.CENTER_ADDRESS2, + CrawlerConsts.CENTER_ADDRESS3 + ), + directUrl = driver.currentUrl + ) + }.onSuccess { post -> result.add(post) } + } + + private fun backWindow(driver:WebDriver, originalWindow: String?) { + driver.close() + driver.switchTo().window(originalWindow) + } + + private fun extractText(driver: WebDriver, con: CrawlerConsts): String { + return runCatching { + driver.findElement(By.xpath(con.value)).text + }.getOrElse { e-> + errorRecorder.recordError(con.location) + throw e + } + } + + private fun extractApplyDeadline(driver: WebDriver, con: CrawlerConsts): String { + return runCatching { + driver.findElement(By.xpath(con.value)).text.let { + if (it.contains("채용시까지")) + LocalDate.now().plusDays(15).format(DateTimeFormatter.ofPattern("yyyyMMdd")) + else + it + } + }.getOrElse { e-> + errorRecorder.recordError(con.location) + throw e + } + } + + private fun extractAddress(driver: WebDriver,vararg cons: CrawlerConsts): String { + runCatching { + for (con in cons) { + val address = driver.findElement(By.xpath(con.value)).text + return address.replace("지도보기", "").trim().replace(Regex("\\(\\d{5}\\)"), "").trim() + } + throw NoSuchElementException("Center address not found using any of the provided XPaths") + }.getOrElse { e-> + errorRecorder.recordError(cons[0].location) + throw e + } + } + + private fun extractWorkTime(driver: WebDriver, con: CrawlerConsts): String { + return runCatching { + driver.findElement(By.xpath(con.value)).text + .replace("도움말", "") + .replace("(근무시간)", "") + .replace("\n", "") + }.getOrElse { e-> + errorRecorder.recordError(con.location) + throw e + } + } +} \ No newline at end of file diff --git a/idle-batch/src/main/kotlin/com/swm/idle/batch/job/JobConfig.kt b/idle-batch/src/main/kotlin/com/swm/idle/batch/job/JobConfig.kt index c5feff94..56147d27 100644 --- a/idle-batch/src/main/kotlin/com/swm/idle/batch/job/JobConfig.kt +++ b/idle-batch/src/main/kotlin/com/swm/idle/batch/job/JobConfig.kt @@ -3,7 +3,6 @@ package com.swm.idle.batch.job import com.swm.idle.batch.common.dto.CrawledJobPostingDto import com.swm.idle.batch.step.PostingProcessor import com.swm.idle.batch.step.PostingReader -import com.swm.idle.batch.step.PostingReader.Companion.nextPage import com.swm.idle.domain.jobposting.entity.jpa.CrawledJobPosting import jakarta.persistence.EntityManagerFactory import org.springframework.batch.core.Step @@ -22,7 +21,6 @@ import org.springframework.batch.item.ItemWriter import org.springframework.batch.item.database.JpaItemWriter import org.springframework.core.task.SimpleAsyncTaskExecutor import org.springframework.transaction.PlatformTransactionManager -import java.util.concurrent.atomic.AtomicInteger @Configuration @EnableBatchProcessing @@ -34,7 +32,6 @@ class JobConfig( @Bean fun crawlingJob(): Job { - nextPage = AtomicInteger(1) return JobBuilder("crawlingJob", jobRepository) .start(crawlStep()) .preventRestart()