diff --git a/service-club/Dockerfile b/service-club/Dockerfile deleted file mode 100644 index 8e3fb70..0000000 --- a/service-club/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM gradle:5.5-jdk12 as builder -COPY --chown=gradle:gradle . /home/src -WORKDIR /home/src/service-club -RUN gradle shadowJar - - -FROM openjdk:12 - -WORKDIR / -COPY --from=builder /home/src/service-club/build/libs/service-club-0.0.1-all.jar ./service-club.jar -CMD java -jar ./service-club.jar diff --git a/service-club/build.gradle b/service-club/build.gradle deleted file mode 100644 index 237aae9..0000000 --- a/service-club/build.gradle +++ /dev/null @@ -1,14 +0,0 @@ -plugins { - id "kotlin" - id "application" - id "com.github.johnrengelman.shadow" version "5.1.0" -} -apply plugin: 'kotlinx-serialization' - -dependencies { - implementation(project(":common")) -} - -application { - mainClassName = "de.hpi.cloud.club.ClubServiceKt" -} diff --git a/service-club/src/main/kotlin/de/hpi/cloud/club/ClubService.kt b/service-club/src/main/kotlin/de/hpi/cloud/club/ClubService.kt deleted file mode 100644 index 3b37ea5..0000000 --- a/service-club/src/main/kotlin/de/hpi/cloud/club/ClubService.kt +++ /dev/null @@ -1,49 +0,0 @@ -package de.hpi.cloud.club - -import com.couchbase.client.java.Bucket -import com.couchbase.client.java.view.ViewQuery -import de.hpi.cloud.club.entities.Club -import de.hpi.cloud.club.entities.toProto -import de.hpi.cloud.club.v1test.ClubServiceGrpc -import de.hpi.cloud.club.v1test.GetClubRequest -import de.hpi.cloud.club.v1test.ListClubsRequest -import de.hpi.cloud.club.v1test.ListClubsResponse -import de.hpi.cloud.common.Service -import de.hpi.cloud.common.couchbase.VIEW_BY_ID -import de.hpi.cloud.common.couchbase.get -import de.hpi.cloud.common.couchbase.paginate -import de.hpi.cloud.common.entity.Id -import de.hpi.cloud.common.grpc.checkArgRequired -import de.hpi.cloud.common.grpc.throwNotFound -import de.hpi.cloud.common.grpc.unary -import de.hpi.cloud.common.protobuf.build -import io.grpc.stub.StreamObserver -import de.hpi.cloud.club.v1test.Club as ProtoClub - -fun main(args: Array) { - val service = Service("club", args.firstOrNull()?.toInt()) { ClubServiceImpl(it) } - service.blockUntilShutdown() -} - -class ClubServiceImpl(private val bucket: Bucket) : ClubServiceGrpc.ClubServiceImplBase() { - // region Club - override fun listClubs(request: ListClubsRequest?, responseObserver: StreamObserver?) = - unary(request, responseObserver, "listClubs") { req -> - val (clubs, newToken) = ViewQuery.from(Club.type, VIEW_BY_ID) - .paginate(bucket, req.pageSize, req.pageToken) - - ListClubsResponse.newBuilder().build { - addAllClubs(clubs.map { it.toProto(this@unary) }) - nextPageToken = newToken - } - } - - override fun getClub(request: GetClubRequest?, responseObserver: StreamObserver?) = - unary(request, responseObserver, "getClub") { req -> - checkArgRequired(req.id, "id") - - bucket.get(Id(req.id))?.toProto(this) - ?: throwNotFound(req.id) - } - // endregion -} diff --git a/service-club/src/main/kotlin/de/hpi/cloud/club/entities/Club.kt b/service-club/src/main/kotlin/de/hpi/cloud/club/entities/Club.kt deleted file mode 100644 index 44e4985..0000000 --- a/service-club/src/main/kotlin/de/hpi/cloud/club/entities/Club.kt +++ /dev/null @@ -1,40 +0,0 @@ -package de.hpi.cloud.club.entities - -import de.hpi.cloud.common.Context -import de.hpi.cloud.common.Party -import de.hpi.cloud.common.entity.Entity -import de.hpi.cloud.common.entity.Id -import de.hpi.cloud.common.entity.Wrapper -import de.hpi.cloud.common.protobuf.builder -import de.hpi.cloud.common.types.L10n -import kotlinx.serialization.Serializable -import de.hpi.cloud.club.v1test.Club as ProtoClub - -@Serializable -data class Club( - val title: L10n, - val abbreviation: String, - val headGroupId: Id, - val memberGroupId: Id -) : Entity() { - companion object : Entity.Companion("club") - - object ProtoSerializer : Entity.ProtoSerializer() { - override fun fromProto(proto: ProtoClub, context: Context): Club = Club( - title = L10n.single(context, proto.title), - abbreviation = proto.abbreviation, - headGroupId = Id(proto.headGroupId), - memberGroupId = Id(proto.memberGroupId) - ) - - override fun toProtoBuilder(entity: Club, context: Context): ProtoClub.Builder = - ProtoClub.newBuilder().builder(entity) { - title = it.title[context] - abbreviation = it.abbreviation - headGroupId = it.headGroupId.value - memberGroupId = it.memberGroupId.value - } - } -} - -fun Wrapper.toProto(context: Context): ProtoClub = Club.ProtoSerializer.toProto(this, context) diff --git a/service-course-crawler/build.gradle b/service-course-crawler/build.gradle deleted file mode 100644 index a3f81ba..0000000 --- a/service-course-crawler/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -buildscript { - repositories { - mavenCentral() - } - dependencies { - classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.3.50") - classpath("com.github.jengelman.gradle.plugins:shadow:5.1.0") - } -} - -plugins { - id "kotlin" - id "application" - id "com.github.johnrengelman.shadow" version "5.1.0" -} - -repositories { - jcenter() - maven { url "https://dl.bintray.com/hpi/hpi-cloud-mvn" } -} - -dependencies { - implementation(project(":common")) - - // Crawling - implementation("org.jsoup:jsoup:1.12.1") -} - -application { - mainClassName = "de.hpi.cloud.course.crawler.MainKt" -} diff --git a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/Course.kt b/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/Course.kt deleted file mode 100644 index 15c836e..0000000 --- a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/Course.kt +++ /dev/null @@ -1,59 +0,0 @@ -package de.hpi.cloud.course.crawler - -import de.hpi.cloud.common.Entity -import de.hpi.cloud.common.utils.couchbase.i18nSingle -import java.net.URL -import java.time.LocalDate -import java.time.format.DateTimeFormatter - -data class Course( - val courseSeries: CourseSeries, - val semester: Semester, - val lecturers: List, - val assistants: List, - val attendance: Int? = null, - val enrollmentDeadline: LocalDate?, - val website: URL, - val entityLanguage: String -) : Entity("course", 1) { - - override val id get() = "${courseSeries.id}_${semester.id}" - - override fun valueToMap() = mapOf( - "courseSeriesId" to courseSeries.id, - "semesterId" to semester.id, - "lecturers" to lecturers, - "assistants" to assistants, - "attendance" to attendance, - "enrollmentDeadline" to enrollmentDeadline?.format(DateTimeFormatter.ISO_DATE), - "website" to i18nSingle(website.toString(), courseSeries.entityLanguage) - ) - - - companion object { - inline fun build(block: Builder.() -> Unit) = - Builder().apply(block).build() - } - - class Builder { - lateinit var courseEntityLanguage: String - lateinit var courseSeries: CourseSeries - lateinit var semester: Semester - lateinit var lecturers: List - lateinit var assistants: List - var attendance: Int? = null - var enrollmentDeadline: LocalDate? = null - lateinit var website: URL - - fun build() = Course( - courseSeries, - semester, - lecturers, - assistants, - attendance, - enrollmentDeadline, - website, - courseEntityLanguage - ) - } -} diff --git a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/CourseDetail.kt b/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/CourseDetail.kt deleted file mode 100644 index 478d1fe..0000000 --- a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/CourseDetail.kt +++ /dev/null @@ -1,64 +0,0 @@ -package de.hpi.cloud.course.crawler - -import de.hpi.cloud.common.Entity -import de.hpi.cloud.common.utils.couchbase.i18nSingle -import java.net.URL - -data class CourseDetail( - val course: Course, - val teletask: URL?, - val programs: Map>, - val description: String?, - val requirements: String?, - val learning: String?, - val examination: String?, - val dates: String?, - val literature: String?, - val entityLanguage: String -) : Entity("courseDetail", 1) { - - override val id get() = course.id - - override fun valueToMap() = mapOf( - "teletask" to teletask?.toString(), - "programs" to programs - .mapKeys { entry -> entry.key.toString() } - .mapValues { entry -> entry.value.toList() }, - "description" to i18nSingle(description, entityLanguage), - "requirements" to i18nSingle(requirements, entityLanguage), - "learning" to i18nSingle(learning, entityLanguage), - "examination" to i18nSingle(examination, entityLanguage), - "dates" to i18nSingle(dates, entityLanguage), - "literature" to i18nSingle(literature, entityLanguage) - ) - - companion object { - inline fun build(block: Builder.() -> Unit) = - Builder().apply(block).build() - } - - class Builder { - lateinit var courseDetailEntityLanguage: String - lateinit var course: Course - var teletask: URL? = null - lateinit var programs: Map> - var description: String? = null - var requirements: String? = null - var learning: String? = null - var examination: String? = null - var dates: String? = null - var literature: String? = null - fun build() = CourseDetail( - course, - teletask, - programs, - description, - requirements, - learning, - examination, - dates, - literature, - courseDetailEntityLanguage - ) - } -} diff --git a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/CourseLanguage.kt b/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/CourseLanguage.kt deleted file mode 100644 index c85d70a..0000000 --- a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/CourseLanguage.kt +++ /dev/null @@ -1,12 +0,0 @@ -package de.hpi.cloud.course.crawler - -enum class CourseLanguage( - val title: String -) { - DE("Deutsch"), - EN("Englisch"); - - companion object { - fun parse(string: String) = values().first { cl -> cl.title.equals(string, ignoreCase = true) } - } -} diff --git a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/CourseSeries.kt b/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/CourseSeries.kt deleted file mode 100644 index 90b05bc..0000000 --- a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/CourseSeries.kt +++ /dev/null @@ -1,119 +0,0 @@ -package de.hpi.cloud.course.crawler - -import de.hpi.cloud.common.Entity -import de.hpi.cloud.common.utils.couchbase.i18nSingle - -data class CourseSeries( - val title: String, - val shortTitle: String, - val abbreviation: String, - val ects: Int?, - val hoursPerWeek: Int?, - val compulsory: Compulsory, - val courseLanguage: CourseLanguage, - val types: Set, - val entityLanguage: String -) : Entity("courseSeries", 1) { - init { - fun invalidDataset(fieldName: String): Nothing = error("Invalid dataset: $fieldName not set") - if (ects == null) invalidDataset("ECTS") - if (hoursPerWeek == null) invalidDataset("Hours") - } - - override val id - get() = title.toLowerCase() - .replace(Regex("[^a-z0-9]*"), "") - - override fun valueToMap() = mapOf( - "title" to i18nSingle(title, entityLanguage), - "shortTitle" to i18nSingle(shortTitle, entityLanguage), - "abbreviation" to i18nSingle(abbreviation, entityLanguage), - "ects" to ects, - "hoursPerWeek" to hoursPerWeek, - "compulsory" to compulsory.name, - "language" to courseLanguage.name, - "types" to types.map { it.toString() } - ) - - enum class Compulsory { - NON_COMPULSORY, - COMPULSORY, - BRIDGE; - - companion object { - fun parse(string: String) = when (string.toLowerCase()) { - "wahlpflichtmodul" -> NON_COMPULSORY - "pflichtmodul" -> COMPULSORY - "brückenmodul" -> BRIDGE - else -> error("Unknown occupancy status \"$string\"") - } - } - } - - enum class Type { - LECTURE, - SEMINAR, - PROJECT, - EXERCISE, - BLOCK_SEMINAR; - - override fun toString(): String { - return name.toLowerCase() - } - - companion object { - fun parse(string: String) = string.toLowerCase() - .split('/') - .flatMap { - when (it.trim()) { - "vorlesung" -> listOf(LECTURE) - "seminar" -> listOf(SEMINAR) - "projekt" -> listOf(PROJECT) - "übung" -> listOf(EXERCISE) - "blockseminar" -> listOf(BLOCK_SEMINAR) - "projektseminar" -> listOf(PROJECT, SEMINAR) - "vu" -> listOf(LECTURE, EXERCISE) - "sp" -> listOf(SEMINAR, PROJECT) - "s" -> listOf(SEMINAR) - "" -> listOf() - else -> { - println("Unknown type value \"$string\"") - listOf() - } - } - } - .filterNotNull() - .distinct() - .toSet() - } - } - - companion object { - inline fun build(block: Builder.() -> Unit) = - Builder().apply(block).build() - } - - class Builder { - lateinit var courseSeriesEntityLanguage: String - lateinit var title: String - lateinit var shortTitle: String - lateinit var abbreviation: String - var ects: Int? = null - var hoursPerWeek: Int? = null - lateinit var compulsory: Compulsory - lateinit var courseLanguage: CourseLanguage - lateinit var types: Set - - fun build() = CourseSeries( - title, - shortTitle, - abbreviation, - ects, - hoursPerWeek, - compulsory, - courseLanguage, - types, - courseSeriesEntityLanguage - ) - } -} diff --git a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/Degree.kt b/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/Degree.kt deleted file mode 100644 index e6101ed..0000000 --- a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/Degree.kt +++ /dev/null @@ -1,13 +0,0 @@ -package de.hpi.cloud.course.crawler - -enum class Degree( - val abbr: String -) { - BACHELOR("BA"), - MASTER("MA"); - - companion object { - fun parse(string: String) = values().firstOrNull { it.abbr.equals(string, ignoreCase = true) } - ?: values().first { it.name.equals(string, ignoreCase = true) } - } -} diff --git a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/HpiArchiveIdentifier.kt b/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/HpiArchiveIdentifier.kt deleted file mode 100644 index 561dbe5..0000000 --- a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/HpiArchiveIdentifier.kt +++ /dev/null @@ -1,8 +0,0 @@ -package de.hpi.cloud.course.crawler - -data class HpiArchiveIdentifier( - val spd: StudyPathDegree, - val sem: Semester -) { - override fun toString() = "$spd $sem" -} diff --git a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/HpiCourseDetailPageCrawler.kt b/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/HpiCourseDetailPageCrawler.kt deleted file mode 100644 index b01eec7..0000000 --- a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/HpiCourseDetailPageCrawler.kt +++ /dev/null @@ -1,235 +0,0 @@ -package de.hpi.cloud.course.crawler - -import de.hpi.cloud.common.utils.groupingSections -import de.hpi.cloud.common.utils.splitAsPair -import de.hpi.cloud.common.utils.thenTake -import de.hpi.cloud.common.utils.trim -import org.jsoup.Jsoup -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.jsoup.select.Elements -import java.net.URL -import java.time.LocalDate -import java.time.format.DateTimeFormatter -import java.time.format.DateTimeParseException -import java.time.format.FormatStyle -import java.util.* - -class HpiCourseDetailPageCrawler( - val url: URL -) { - - companion object { - val TITLE_REGEX = Regex("^(.*) \\((\\w+?) (.*?)\\)\$") - val LECTURER_ASSISTANTS_REGEX = Regex( - "(.*?)

", - setOf(RegexOption.DOT_MATCHES_ALL, RegexOption.MULTILINE) - ) - val SIMPLE_GERMAN_DATE_FORMAT = DateTimeFormatter.ofPattern("dd.MM.yyyy") - val LONG_GERMAN_DATE_FORMAT = DateTimeFormatter.ofLocalizedDate(FormatStyle.LONG).withLocale(Locale.GERMAN) - } - - private enum class RelevantSection( - val queryValues: List - ) { - GENERAL_INFORMATION("Allgemeine Information"), - PROGRAMS("Studiengänge & Module", "Studiengänge"), - DESCRIPTION("Beschreibung"), - REQUIREMENTS("Voraussetzungen"), - LITERATURE("Literatur"), - EXAMINATION("Leistungserfassung"), - DATES("Termine"), - LEARNING("Lern- und Lehrformen"); - - constructor(vararg queryValues: String) : this(queryValues.asList()) - - companion object { - val REVERSE_MAP = values().flatMap { section -> - section.queryValues.map { it to section } - }.toMap() - - fun parse(h1name: String) = REVERSE_MAP[h1name] - - fun isValid(string: String) = parse(string) != null - } - } - - fun query() = read(createDocumentQuery(url).get()) - - fun read(doc: Document): CourseDetail { - println(url) - val container = doc.selectFirst(".tx-ciuniversity-course") - val containerHtml = container.html() - - return CourseDetail.build { - course = Course.build { - courseSeries = CourseSeries.build { - website = url - - val titleMatch = TITLE_REGEX.matchEntire(container.selectFirst("h1").text().trim())!! - title = titleMatch.groups[1]!!.value - - // add shortTitle and abbreviation manually - shortTitle = title - abbreviation = "" - - semester = Semester.parse( - titleMatch.groups[3]!!.value, - titleMatch.groups[2]!!.value - ) - - val laa = readLecturerAndAssistants(containerHtml) - lecturers = laa.lecturers - assistants = laa.assistants - - container.children() - .groupingSections(true) { node -> - node.`is`("h2").thenTake { - node.text().trim().takeIf { - RelevantSection.isValid(it) - } - } - } - .mapValues { (_, elements) -> Elements(elements) } - .forEach { (key, elements) -> - // println(key + " " + "-".repeat(64 - key.length)) - when (RelevantSection.parse(key)) { - RelevantSection.GENERAL_INFORMATION -> { - elements - .select("ul.tx-ciuniversity-course-general-info > li") - .map { it.text().splitAsPair(":").trim() } - .forEach { (key, value) -> - when (key.toLowerCase()) { - "semesterwochenstunden" -> hoursPerWeek = value.toInt() - "ects" -> ects = value.toInt() - "benotet" -> { - } // ignored - "einschreibefrist" -> enrollmentDeadline = parseDeadline(value) - "maximale teilnehmerzahl" -> attendance = value.toInt() - "lehrform" -> types = CourseSeries.Type.parse(value) - "belegungsart" -> compulsory = CourseSeries.Compulsory.parse(value) - "lehrsprache" -> CourseLanguage.parse(value).apply { - courseLanguage = this - courseEntityLanguage = name - courseDetailEntityLanguage = name - courseSeriesEntityLanguage = name - } - else -> println("Unknown entry \"$key\"=\"$value\"") - } - } - } - RelevantSection.DESCRIPTION -> { - description = elements.outerHtml() - } - RelevantSection.PROGRAMS -> { - programs = elements - .select(".tx_dscclipclap") - .map { readStudyPathModules(it) } - .toMap() - } - RelevantSection.REQUIREMENTS -> { - requirements = elements.outerHtml() - } - RelevantSection.LEARNING -> { - learning = elements.outerHtml() - } - RelevantSection.EXAMINATION -> { - examination = elements.outerHtml() - } - RelevantSection.DATES -> { - dates = elements.outerHtml() - } - RelevantSection.LITERATURE -> { - literature = elements.outerHtml() - } - else -> { - elements.forEach { println(it.text()) } - } - } - } - } - } - } - } - - private fun parseDeadline(string: String): LocalDate? { - if (string.isBlank()) - return null - try { - return LocalDate.parse( - string - .replaceBefore("-", "") - .trim('-', ' '), - SIMPLE_GERMAN_DATE_FORMAT - ) - } catch (ex: DateTimeParseException) { - try { - return LocalDate.parse( - string.split(" ") - .subList(0, 3) - .joinToString(separator = " "), - LONG_GERMAN_DATE_FORMAT - ) - } catch (ex: IndexOutOfBoundsException) { - // error handling below - } catch (ex: DateTimeParseException) { - // error handling below - } - } - println("Could not parse enrollment deadline \"$string\". Please check manually.") - return null - } - - private fun readStudyPathModules(element: Element): Pair> { - val studyPathDegree = StudyPathDegree.parse(element.selectFirst(".tx_dscclipclap_header").text()) - val modules = element.selectFirst(".tx_dscclipclap_content") - .children() - .map { it.text() } - .toSet() - return studyPathDegree to modules - } - - private fun readLecturerAndAssistants(containerHtml: String): LecturerAndAssistants = LECTURER_ASSISTANTS_REGEX - .find(containerHtml)!! - .groups[1]!! - .value.trim() - .let { croppedHtml -> - LecturerAndAssistants.parse(croppedHtml) - } - - private data class LecturerAndAssistants( - val lecturers: List, - val assistants: List = listOf() - ) { - companion object { - fun parse(string: String): LecturerAndAssistants = string.run { - indexOf("Tutoren:").let { i -> - if (i == -1) { - LecturerAndAssistants( - readLecturers(this) - ) - } else { - LecturerAndAssistants( - readLecturers(this.substring(0, i)), - readAssistants(this.substring(i)) - ) - } - } - } - - private fun readLecturers(htmlSnippet: String) = htmlSnippet - .replaceAfter("
", "") // Hacky hack to counter "Website zum Kurs:" and others - .let { Jsoup.parse(it, HPI_BASE_URI.toString()) } - .select("i") - .map { it.selectFirst("a") } - .map { it.text() } - .filter { !it.startsWith("http", ignoreCase = true) } - - private fun readAssistants(htmlSnippet: String) = htmlSnippet - .let { Jsoup.parse(it, HPI_BASE_URI.toString()) } - .select("i > a") - .map { it.text() } - .filter { !it.startsWith("http", ignoreCase = true) } - } - } -} diff --git a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/HpiCourseListCrawler.kt b/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/HpiCourseListCrawler.kt deleted file mode 100644 index 296f461..0000000 --- a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/HpiCourseListCrawler.kt +++ /dev/null @@ -1,24 +0,0 @@ -package de.hpi.cloud.course.crawler - -class HpiCourseListCrawler( - val csi: HpiArchiveIdentifier -) { - constructor(studyPathName: String, degree: Degree, sem: Semester = CURRENT_SEMESTER) - : this(HpiArchiveIdentifier(StudyPathDegree(studyPathName, degree), sem)) - - val courseListBaseURI = HPI_BASE_URI.resolve("studium/lehrveranstaltungen/") - - val studyPathId = csi.spd.studyPathName.replace(' ', '-').toLowerCase() - val queryURL = courseListBaseURI.resolve("$studyPathId-${csi.spd.degree.abbr.toLowerCase()}/").toURL() - - fun listCourses(): Sequence { - val doc = createDocumentQuery(queryURL).get() - - val links = doc.select("table.contenttable tr a.courselink") - return links.asSequence() - .map { el -> el.attr("href") } - .map { HpiCourseDetailPageCrawler(HPI_BASE_URI.resolve(it).toURL()) } - } - - override fun toString() = csi.toString() -} diff --git a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/Main.kt b/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/Main.kt deleted file mode 100644 index 82b9065..0000000 --- a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/Main.kt +++ /dev/null @@ -1,89 +0,0 @@ -package de.hpi.cloud.course.crawler - -import com.couchbase.client.java.Bucket -import com.couchbase.client.java.view.ViewQuery -import de.hpi.cloud.common.utils.couchbase.VIEW_BY_ID -import de.hpi.cloud.common.utils.couchbase.withBucket -import org.jsoup.Connection -import org.jsoup.Jsoup -import java.net.URI -import java.net.URL - -const val NAME = "HPI-MobileDev-Crawler[Course]" -val USER_AGENT_STRING = "$NAME jsoup/1.12.1 Kotlin-runtime/${KotlinVersion.CURRENT}" - -val HPI_BASE_URI = URI("https://hpi.de/") - -val CURRENT_SEMESTER = Semester(2019, Semester.Term.WINTER) -val CRAWLERS = setOf( - HpiCourseListCrawler("IT-Systems Engineering", Degree.BACHELOR), - HpiCourseListCrawler("IT-Systems Engineering", Degree.MASTER), - HpiCourseListCrawler("Digital Health", Degree.MASTER), - HpiCourseListCrawler("Data Engineering", Degree.MASTER), - HpiCourseListCrawler("Cybersecurity", Degree.MASTER) -) - -var requestCount = 0 -fun createDocumentQuery(url: URL): Connection { - requestCount++ - return Jsoup - .connect(url.toString()) - .userAgent(USER_AGENT_STRING) -} - -fun main(args: Array) { - println("Starting $NAME") - withBucket("course") { bucket -> - if (args.contains("--clear")) - deleteOldData(bucket) - - println("Crawling current semester") - println("Using User-Agent=\"$USER_AGENT_STRING\"") - - var count = 0 - CRAWLERS.asSequence() - .flatMap { - println("Starting crawler for $it") - it.listCourses() - } - .map { - try { - it.query() - } catch (ex: Exception) { - println("error: $ex") - null - } - } - .filterNotNull() - .forEach { - println("Parsed course page with ID ${it.id}") - if (args.contains("--all") || args.contains("--upsert-course-details")) - bucket.upsert(it.toJsonDocument()) - if (args.contains("--all") || args.contains("--upsert-course")) - bucket.upsert(it.course.toJsonDocument()) - if (args.contains("--all") || args.contains("--upsert-course-series")) - bucket.upsert(it.course.courseSeries.toJsonDocument()) - count++ - } - println("Upserted $count course page") - println("Crawler used ${requestCount} server requests") - } -} - -fun deleteOldData(bucket: Bucket) { - fun Bucket.removeEverythingFromView(design: String) = bucket - .query(ViewQuery.from(design, VIEW_BY_ID)) - .allRows() - .forEach { - println("Remove $design ${it.id()}") - try { - bucket.remove(it.id()) - } catch (ex: Exception) { - // ignore strange "element not found" errors - } - } - println("Entered database cleanup mode\n") - bucket.removeEverythingFromView("course") - println() - bucket.removeEverythingFromView("courseSeries") -} diff --git a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/Semester.kt b/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/Semester.kt deleted file mode 100644 index 1d6d358..0000000 --- a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/Semester.kt +++ /dev/null @@ -1,44 +0,0 @@ -package de.hpi.cloud.course.crawler - -import de.hpi.cloud.common.Entity -import de.hpi.cloud.common.utils.cut - -data class Semester( - val year: Int, - val term: Term -) : Entity("semester", 1) { - - override val id get() = "$year${term.abbreviation}" - - override fun valueToMap() = mapOf( - "year" to year, - "term" to term.toString() - ) - - override fun toString() = id - - enum class Term( - val abbreviation: String, - val title: String - ) { - WINTER("ws", "Wintersemester"), - SUMMER("ss", "Sommersemester"); - - override fun toString(): String { - return name.toLowerCase() - } - - companion object { - fun parse(term: String) = values().firstOrNull { t -> term.startsWith(t.title, ignoreCase = true) } - ?: values().firstOrNull { t -> term.startsWith(t.name, ignoreCase = true) } - ?: values().first { t -> term.startsWith(t.abbreviation, ignoreCase = true) } - } - } - - companion object { - fun parse(year: String, term: String) = Semester( - year.cut('/').toInt(), - Term.parse(term) - ) - } -} diff --git a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/StudyPathDegree.kt b/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/StudyPathDegree.kt deleted file mode 100644 index bfda979..0000000 --- a/service-course-crawler/src/main/kotlin/de/hpi/cloud/course/crawler/StudyPathDegree.kt +++ /dev/null @@ -1,15 +0,0 @@ -package de.hpi.cloud.course.crawler - -import de.hpi.cloud.common.utils.splitAsPair - -data class StudyPathDegree( - val studyPathName: String, - val degree: Degree -) { - override fun toString(): String = "${studyPathName}-${degree.abbr}" - - companion object { - fun parse(string: String) = string.splitAsPair(" ", fromEnd = true) - .let { (spn, deg) -> StudyPathDegree(spn, Degree.parse(deg)) } - } -} diff --git a/service-course/build.gradle b/service-course/build.gradle deleted file mode 100644 index 1981817..0000000 --- a/service-course/build.gradle +++ /dev/null @@ -1,28 +0,0 @@ -buildscript { - repositories { - mavenCentral() - } - dependencies { - classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.3.40") - classpath("com.github.jengelman.gradle.plugins:shadow:5.1.0") - } -} - -plugins { - id "kotlin" - id "application" - id "com.github.johnrengelman.shadow" version "5.1.0" -} - -repositories { - jcenter() - maven { url "https://dl.bintray.com/hpi/hpi-cloud-mvn" } -} - -dependencies { - implementation(project(":common")) -} - -application { - mainClassName = "de.hpi.cloud.course.CourseServiceKt" -} diff --git a/service-course/src/main/kotlin/de/hpi/cloud/course/CourseService.kt b/service-course/src/main/kotlin/de/hpi/cloud/course/CourseService.kt deleted file mode 100644 index 31d8899..0000000 --- a/service-course/src/main/kotlin/de/hpi/cloud/course/CourseService.kt +++ /dev/null @@ -1,188 +0,0 @@ -package de.hpi.cloud.course - -import com.couchbase.client.java.Bucket -import com.couchbase.client.java.document.json.JsonObject -import com.couchbase.client.java.query.dsl.Expression.s -import com.couchbase.client.java.query.dsl.Expression.x -import com.couchbase.client.java.query.dsl.Sort.asc -import com.couchbase.client.java.query.dsl.Sort.desc -import com.couchbase.client.java.view.ViewQuery -import com.google.protobuf.GeneratedMessageV3 -import com.google.protobuf.UInt32Value -import de.hpi.cloud.common.Service -import de.hpi.cloud.common.utils.couchbase.* -import de.hpi.cloud.common.utils.getI18nString -import de.hpi.cloud.common.utils.grpc.buildWith -import de.hpi.cloud.common.utils.grpc.buildWithDocument -import de.hpi.cloud.common.utils.grpc.throwException -import de.hpi.cloud.common.utils.grpc.unary -import de.hpi.cloud.common.utils.protobuf.getDateUsingIsoFormat -import de.hpi.cloud.course.v1test.* -import io.grpc.Status -import io.grpc.stub.StreamObserver - -fun main(args: Array) { - val service = Service("course", args.firstOrNull()?.toInt()) { CourseServiceImpl(it) } - service.blockUntilShutdown() -} - -class CourseServiceImpl(private val bucket: Bucket) : CourseServiceGrpc.CourseServiceImplBase() { - companion object { - const val DESIGN_COURSE_SERIES = "courseSeries" - const val DESIGN_SEMESTER = "semester" - const val DESIGN_COURSE = "course" - const val DESIGN_COURSE_DETAIL = "courseDetail" - } - - // region CourseSeries - override fun listCourseSeries( - request: ListCourseSeriesRequest?, - responseObserver: StreamObserver? - ) = unary(request, responseObserver, "listCourseSeries") { req -> - val (courseSeries, newToken) = ViewQuery.from(DESIGN_COURSE_SERIES, VIEW_BY_ID) - .paginate(bucket, req.pageSize, req.pageToken) { it.parseCourseSeries(req) } - - ListCourseSeriesResponse.newBuilder().buildWith { - addAllCourseSeries(courseSeries) - nextPageToken = newToken - } - } - - override fun getCourseSeries(request: GetCourseSeriesRequest?, responseObserver: StreamObserver?) = - unary(request, responseObserver, "getCourseSeries") { req -> - if (req.id.isNullOrEmpty()) Status.INVALID_ARGUMENT.throwException("CourseSeries ID is required") - - bucket.get(DESIGN_COURSE_SERIES, VIEW_BY_ID, req.id) - ?.document()?.content()?.parseCourseSeries(req) - ?: Status.NOT_FOUND.throwException("CourseSeries with ID ${req.id} not found") - } - - private fun JsonObject.parseCourseSeries(request: GeneratedMessageV3) = - CourseSeries.newBuilder().buildWithDocument(this) { - id = getString(KEY_ID) - title = it.getI18nString("title", request) - shortTitle = it.getI18nString("shortTitle", request) - abbreviation = it.getI18nString("abbreviation", request) - ects = it.getInt("ects") - hoursPerWeek = it.getInt("hoursPerWeek") - compulsory = it.getString("compulsory").parseCourseSeriesCompulsory() - language = it.getString("language") - addAllTypes(it.getStringArray("types").mapNotNull { t -> t?.parseCourseSeriesType() }) - } - - private fun String.parseCourseSeriesCompulsory() = CourseSeries.Compulsory - .values().first { it.name.equals(this, ignoreCase = true) } - - private fun String.parseCourseSeriesType() = CourseSeries.Type - .values().first { it.name.equals(this, ignoreCase = true) } - // endregion - - // region Semester - override fun listSemesters( - request: ListSemestersRequest?, - responseObserver: StreamObserver? - ) = unary(request, responseObserver, "listSemesters") { req -> - val (semesters, newToken) = ViewQuery.from(DESIGN_SEMESTER, VIEW_BY_ID) - .paginate(bucket, req.pageSize, req.pageToken) { it.parseSemester() } - - ListSemestersResponse.newBuilder().buildWith { - addAllSemesters(semesters) - nextPageToken = newToken - } - } - - override fun getSemester(request: GetSemesterRequest?, responseObserver: StreamObserver?) = - unary(request, responseObserver, "getSemester") { req -> - if (req.id.isNullOrEmpty()) Status.INVALID_ARGUMENT.throwException("Argument ID is required") - - bucket.get(DESIGN_SEMESTER, VIEW_BY_ID, req.id) - ?.document()?.content()?.parseSemester() - ?: Status.NOT_FOUND.throwException("Semester with ID ${req.id} not found") - } - - private fun JsonObject.parseSemester() = Semester.newBuilder().buildWithDocument(this) { - id = getString(KEY_ID) - term = it.getString("term").parseSemesterTerm() - year = it.getInt("year") - } - - private fun String.parseSemesterTerm(): Semester.Term { - return Semester.Term.values().first { it.name.equals(this, true) } - } - // endregion - - // region Course - override fun listCourses(request: ListCoursesRequest?, responseObserver: StreamObserver?) = - unary(request, responseObserver, "listCourses") { req -> - val courseSeriesId = req.courseSeriesId?.trim()?.takeIf { it.isNotEmpty() } - val semesterId = req.semesterId?.trim()?.takeIf { it.isNotEmpty() } - - val (courses, newToken) = paginate(bucket, { - where( - and( - x(KEY_TYPE).eq(s("course")), - v("courseSeriesId").eq(s(courseSeriesId)).takeIf { courseSeriesId != null }, - v("semesterId").eq(s(semesterId)).takeIf { semesterId != null }) - ) - .orderBy( - asc(v("courseSeriesId")), - desc(v("semesterId")) - ) - }, req.pageSize, req.pageToken) { it.parseCourse(req) } - - ListCoursesResponse.newBuilder().buildWith { - addAllCourses(courses) - nextPageToken = newToken - } - } - - override fun getCourse(request: GetCourseRequest?, responseObserver: StreamObserver?) = - unary(request, responseObserver, "getCourse") { req -> - if (req.id.isNullOrEmpty()) Status.INVALID_ARGUMENT.throwException("Course ID is required") - - bucket.get(DESIGN_COURSE, VIEW_BY_ID, req.id) - ?.document()?.content()?.parseCourse(req) - ?: Status.NOT_FOUND.throwException("Course with ID ${req.id} not found") - } - - private fun JsonObject.parseCourse(request: GeneratedMessageV3) = - Course.newBuilder().buildWithDocument(this) { - id = getString(KEY_ID) - courseSeriesId = it.getString("courseSeriesId") - semesterId = it.getString("semesterId") - addAllLecturers(it.getStringArray("lecturers").filterNotNull()) - addAllAssistants(it.getStringArray("assistants").filterNotNull()) - it.getInt("attendance")?.let { c -> attendance = UInt32Value.of(c) } - it.getDateUsingIsoFormat("enrollment_deadline ")?.let { d -> enrollmentDeadline = d } - it.getI18nString("website", request)?.let { w -> website = w } - } - // endregion - - // region CourseDetail - override fun getCourseDetail(request: GetCourseDetailRequest?, responseObserver: StreamObserver?) = - unary(request, responseObserver, "getCourseDetail") { req -> - if (req.courseId.isNullOrEmpty()) Status.INVALID_ARGUMENT.throwException("Argument course_id is required") - - bucket.get(DESIGN_COURSE_DETAIL, VIEW_BY_ID, req.courseId) - ?.document()?.content()?.parseCourseDetail(req) - ?: Status.NOT_FOUND.throwException("CourseDetail with ID ${req.courseId} not found") - } - - private fun JsonObject.parseCourseDetail(request: GeneratedMessageV3) = - CourseDetail.newBuilder().buildWithDocument(this) { - courseId = getString(KEY_ID) - it.getString("teletask")?.let { t -> teletask = t } - putAllPrograms(it.getObject("programs").toMap() - .mapValues { p -> - @Suppress("UNCHECKED_CAST") - CourseDetail.ProgramList.newBuilder().addAllPrograms(p.value as List).build() - }) - it.getI18nString("description", request)?.let { d -> description = d } - it.getI18nString("requirements", request)?.let { r -> requirements = r } - it.getI18nString("learning", request)?.let { l -> learning = l } - it.getI18nString("examination", request)?.let { e -> examination = e } - it.getI18nString("dates", request)?.let { d -> dates = d } - it.getI18nString("literature", request)?.let { l -> literature = l } - } - // endregion -} diff --git a/service-crashreporting/Dockerfile b/service-crashreporting/Dockerfile deleted file mode 100644 index e06b631..0000000 --- a/service-crashreporting/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM gradle:5.5-jdk12 as builder -COPY --chown=gradle:gradle . /home/src -WORKDIR /home/src/service-crashreporting -RUN gradle shadowJar - - -FROM openjdk:12 - -WORKDIR / -COPY --from=builder /home/src/service-crashreporting/build/libs/service-crashreporting-0.0.1-all.jar ./service-crashreporting.jar -CMD java -jar ./service-crashreporting.jar diff --git a/service-crashreporting/build.gradle b/service-crashreporting/build.gradle deleted file mode 100644 index 73e6ef5..0000000 --- a/service-crashreporting/build.gradle +++ /dev/null @@ -1,14 +0,0 @@ -plugins { - id "kotlin" - id "application" - id "com.github.johnrengelman.shadow" version "5.1.0" -} -apply plugin: 'kotlinx-serialization' - -dependencies { - implementation(project(":common")) -} - -application { - mainClassName = "de.hpi.cloud.crashreporting.CrashReportingServiceKt" -} diff --git a/service-crashreporting/src/main/kotlin/de/hpi/cloud/crashreporting/CrashReportingService.kt b/service-crashreporting/src/main/kotlin/de/hpi/cloud/crashreporting/CrashReportingService.kt deleted file mode 100644 index 4e59603..0000000 --- a/service-crashreporting/src/main/kotlin/de/hpi/cloud/crashreporting/CrashReportingService.kt +++ /dev/null @@ -1,70 +0,0 @@ -package de.hpi.cloud.crashreporting - -import com.couchbase.client.java.Bucket -import de.hpi.cloud.common.Service -import de.hpi.cloud.common.couchbase.tryInsert -import de.hpi.cloud.common.entity.Id -import de.hpi.cloud.common.entity.parse -import de.hpi.cloud.common.grpc.checkArgRequired -import de.hpi.cloud.common.grpc.unary -import de.hpi.cloud.crashreporting.entities.CrashReport -import de.hpi.cloud.crashreporting.entities.toProto -import de.hpi.cloud.crashreporting.v1test.CrashReportingServiceGrpc -import de.hpi.cloud.crashreporting.v1test.CreateCrashReportRequest -import io.grpc.stub.StreamObserver -import de.hpi.cloud.crashreporting.v1test.CrashReport as ProtoCrashReport - -fun main(args: Array) { - val service = Service("crashreporting", args.firstOrNull()?.toInt()) { CrashReportingServiceImpl(it) } - service.blockUntilShutdown() -} - -class CrashReportingServiceImpl(private val bucket: Bucket) : - CrashReportingServiceGrpc.CrashReportingServiceImplBase() { - override fun createCrashReport( - request: CreateCrashReportRequest?, - responseObserver: StreamObserver? - ) = unary(request, responseObserver, "createCrashReport") { req -> - checkArgRequired(req.hasCrashReport(), "crash_report") - checkArgRequired(req.crashReport.appName, "crash_report.app_name") - checkArgRequired(req.crashReport.appVersion, "crash_report.app_version") - checkArgRequired(req.crashReport.appVersionCode != 0, "crash_report.app_version_code") - if (req.crashReport.hasDevice()) { - checkArgRequired( - req.crashReport.device.brand, - "crash_report.device.brand", - ifArgSet = "crash_report.device" - ) - checkArgRequired( - req.crashReport.device.model, - "crash_report.device.model", - ifArgSet = "crash_report.device" - ) - } - if (req.crashReport.hasOperatingSystem()) { - checkArgRequired( - req.crashReport.operatingSystem.os, - "crash_report.operating_system.os", - ifArgSet = "crash_report.operating_system" - ) - checkArgRequired( - req.crashReport.operatingSystem.version, - "crash_report.operating_system.version", - ifArgSet = "crash_report.operating_system" - ) - } - checkArgRequired(req.crashReport.hasTimestamp(), "crash_report.timestamp") - checkArgRequired(req.crashReport.exception, "crash_report.exception") - checkArgRequired(req.crashReport.stackTrace, "crash_report.stack_trace") - - val id = Id.fromClientSupplied(req.crashReport.id, bucket) - - val crashReport = req.crashReport.parse(this) - val wrapper = crashReport.createNewWrapper(this, id) - - bucket.tryInsert(wrapper) - - wrapper.toProto(this) - } - // endregion -} diff --git a/service-crashreporting/src/main/kotlin/de/hpi/cloud/crashreporting/entities/CrashReport.kt b/service-crashreporting/src/main/kotlin/de/hpi/cloud/crashreporting/entities/CrashReport.kt deleted file mode 100644 index f0aabf6..0000000 --- a/service-crashreporting/src/main/kotlin/de/hpi/cloud/crashreporting/entities/CrashReport.kt +++ /dev/null @@ -1,101 +0,0 @@ -package de.hpi.cloud.crashreporting.entities - -import de.hpi.cloud.common.Context -import de.hpi.cloud.common.entity.Entity -import de.hpi.cloud.common.entity.Wrapper -import de.hpi.cloud.common.entity.parse -import de.hpi.cloud.common.protobuf.build -import de.hpi.cloud.common.protobuf.builder -import de.hpi.cloud.common.serializers.json.InstantSerializer -import de.hpi.cloud.common.serializers.proto.toProto -import kotlinx.serialization.Serializable -import java.time.Instant -import de.hpi.cloud.common.serializers.proto.ProtoSerializer as BaseProtoSerializer -import de.hpi.cloud.crashreporting.v1test.CrashReport as ProtoCrashReport - -@Serializable -data class CrashReport( - val appName: String, - val appVersion: String, - val appVersionCode: Int, - val device: Device, - val operatingSystem: OperatingSystem, - val timestamp: @Serializable(InstantSerializer::class) Instant, - val exception: String, - val stackTrace: String, - val log: String -) : Entity() { - companion object : Entity.Companion("crashReport") - object ProtoSerializer : Entity.ProtoSerializer() { - override fun fromProto(proto: ProtoCrashReport, context: Context): CrashReport = CrashReport( - appName = proto.appName.trim(), - appVersion = proto.appVersion.trim(), - appVersionCode = proto.appVersionCode, - device = proto.device.parse(context), - operatingSystem = proto.operatingSystem.parse(context), - timestamp = proto.timestamp.parse(context), - exception = proto.exception.trim(), - stackTrace = proto.stackTrace.trim(), - log = proto.log.trim() - ) - - override fun toProtoBuilder(entity: CrashReport, context: Context): ProtoCrashReport.Builder = - ProtoCrashReport.newBuilder().builder(entity) { - appName = it.appName - appVersion = it.appVersion - appVersionCode = it.appVersionCode - device = it.device.toProto(context) - operatingSystem = it.operatingSystem.toProto(context) - timestamp = it.timestamp.toProto(context) - exception = it.exception - stackTrace = it.stackTrace - log = it.log - } - } - - @Serializable - data class Device( - val brand: String, - val model: String - ) { - object ProtoSerializer : BaseProtoSerializer { - override fun fromProto(proto: ProtoCrashReport.Device, context: Context): Device = Device( - brand = proto.brand.trim(), - model = proto.model.trim() - ) - - override fun toProto(entity: Device, context: Context): ProtoCrashReport.Device = - ProtoCrashReport.Device.newBuilder().build(entity) { - brand = it.brand - model = it.model - } - } - - fun toProto(context: Context) = ProtoSerializer.toProto(this, context) - } - - @Serializable - data class OperatingSystem( - val os: String, - val version: String - ) { - object ProtoSerializer : BaseProtoSerializer { - override fun fromProto(proto: ProtoCrashReport.OperatingSystem, context: Context): OperatingSystem = - OperatingSystem( - os = proto.os.trim(), - version = proto.version.trim() - ) - - override fun toProto(persistable: OperatingSystem, context: Context): ProtoCrashReport.OperatingSystem = - ProtoCrashReport.OperatingSystem.newBuilder().build(persistable) { - os = it.os - version = it.version - } - } - - fun toProto(context: Context) = ProtoSerializer.toProto(this, context) - } -} - -fun Wrapper.toProto(context: Context): ProtoCrashReport = - CrashReport.ProtoSerializer.toProto(this, context) diff --git a/service-runner/build.gradle b/service-runner/build.gradle deleted file mode 100644 index 5fe139b..0000000 --- a/service-runner/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -buildscript { - repositories { - mavenCentral() - } - dependencies { - classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.3.50") - classpath("com.github.jengelman.gradle.plugins:shadow:5.1.0") - } -} - -plugins { - id "kotlin" - id "application" - id "com.github.johnrengelman.shadow" version "5.1.0" -} - -repositories { - jcenter() - maven { url "https://dl.bintray.com/hpi/hpi-cloud-mvn" } - maven { url 'https://jitpack.io' } -} - -dependencies { - api(project(":common")) - - implementation("io.github.config4k:config4k:0.4.1") -} - -application { - mainClassName = "de.hpi.cloud.runner.MainKt" -} diff --git a/service-runner/src/main/kotlin/de/hpi/cloud/runner/CmdService.kt b/service-runner/src/main/kotlin/de/hpi/cloud/runner/CmdService.kt deleted file mode 100644 index 0161898..0000000 --- a/service-runner/src/main/kotlin/de/hpi/cloud/runner/CmdService.kt +++ /dev/null @@ -1,37 +0,0 @@ -package de.hpi.cloud.runner - -import java.io.File -import java.time.LocalDateTime -import java.time.format.DateTimeFormatter -import java.util.logging.Logger - -interface CmdService { - val name: String - val cmd: String - val args: List - val workingDir: File - val logger: Logger get() = Logger.getLogger(name) - - val command get() = listOf(cmd, *args.toTypedArray()) - - private fun getLogFile() = workingDir.toPath().resolve("logs/runner-service-$name-${getDateString()}.log").toFile() - private fun getDateString() = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss")) - - fun start() { - val logFile = getLogFile().apply { - parentFile.mkdirs() - createNewFile() - } - - logger.info("Starting $name as \"${command.joinToString(separator = " ")}\" in working directory \"${workingDir.absolutePath}\"") - ProcessBuilder(command) - .redirectErrorStream(true) - .redirectOutput(logFile) - .directory(workingDir) - .start() - .onExit() - .thenAccept { - logger.info("Finished $name with status code ${it.exitValue()}") - } - } -} diff --git a/service-runner/src/main/kotlin/de/hpi/cloud/runner/Main.kt b/service-runner/src/main/kotlin/de/hpi/cloud/runner/Main.kt deleted file mode 100644 index cef7f60..0000000 --- a/service-runner/src/main/kotlin/de/hpi/cloud/runner/Main.kt +++ /dev/null @@ -1,25 +0,0 @@ -package de.hpi.cloud.runner - -import com.typesafe.config.Config -import com.typesafe.config.ConfigFactory -import java.io.File -import java.util.concurrent.Executors -import java.util.logging.Logger - -val configFile = File("services.conf") -val logger: Logger = Logger.getLogger("Scheduler") - -fun main() { - println("Loading config from: ${configFile.absolutePath}") - val config = ConfigFactory.parseString(configFile.readText()) - val services = readConfig(config) - val scheduler = Executors.newScheduledThreadPool( - if (config.hasPath("threads")) config.getInt("threads") - else 4 - ) - services.forEach { it.schedule(scheduler) } -} - -fun readConfig(config: Config): List = config - .getConfigList("services") - .mapNotNull { SimpleRuntimeConfiguration.parse(it) } diff --git a/service-runner/src/main/kotlin/de/hpi/cloud/runner/SimpleCmdService.kt b/service-runner/src/main/kotlin/de/hpi/cloud/runner/SimpleCmdService.kt deleted file mode 100644 index d9501a7..0000000 --- a/service-runner/src/main/kotlin/de/hpi/cloud/runner/SimpleCmdService.kt +++ /dev/null @@ -1,39 +0,0 @@ -package de.hpi.cloud.runner - -import com.typesafe.config.Config -import java.io.File - -class SimpleCmdService( - override val name: String, - override val cmd: String, - vararg arguments: String, - override val workingDir: File = File(".") -) : CmdService { - override val args = arguments.asList() - - companion object { - fun parse(config: Config) = SimpleCmdService( - config.getString("name"), - config.getString("command"), - *config.getStringList("arguments").toTypedArray() - ) - } -} - -class SimpleJavaService( - override val name: String, - val jarFile: File, - vararg jarArguments: String -) : CmdService { - override val args = jarArguments.asList() - override val workingDir: File = jarFile.parentFile ?: error("File does not exist \"$jarFile\"") - override val cmd = "java -jar $jarFile" - - companion object { - fun parse(config: Config) = SimpleJavaService( - config.getString("name"), - File(config.getString("jar")), - *config.getStringList("arguments").toTypedArray() - ) - } -} diff --git a/service-runner/src/main/kotlin/de/hpi/cloud/runner/SimpleRuntimeConfiguration.kt b/service-runner/src/main/kotlin/de/hpi/cloud/runner/SimpleRuntimeConfiguration.kt deleted file mode 100644 index c614ee6..0000000 --- a/service-runner/src/main/kotlin/de/hpi/cloud/runner/SimpleRuntimeConfiguration.kt +++ /dev/null @@ -1,69 +0,0 @@ -package de.hpi.cloud.runner - -import com.typesafe.config.Config -import de.hpi.cloud.runner.utils.LocalDayTime -import java.time.DayOfWeek -import java.time.Duration -import java.time.LocalDateTime -import java.time.LocalTime -import java.util.concurrent.ScheduledExecutorService -import java.util.concurrent.ScheduledFuture -import java.util.concurrent.TimeUnit - -infix fun CmdService.at(times: Set) = SimpleRuntimeConfiguration( - this, - times -) - -class SimpleRuntimeConfiguration( - val service: CmdService, - val dayTimes: Set -) { - fun schedule(scheduler: ScheduledExecutorService) = dayTimes - .forEach { schedule(scheduler, it) } - - private fun schedule(scheduler: ScheduledExecutorService, localDayTime: LocalDayTime): ScheduledFuture<*> = - LocalDateTime.now().let { now -> - Duration.between( - now, - now.with(localDayTime.nextOrSameAdjuster()) - ).seconds.let { secondsUntilNextCall -> - logger.info("Scheduled ${service.name} - running next in $secondsUntilNextCall seconds every ${LocalDayTime.SecondsOfWeek.SECONDS_IN_WEEK} seconds") - scheduler.scheduleAtFixedRate( - { service.start() }, - secondsUntilNextCall, - LocalDayTime.SecondsOfWeek.SECONDS_IN_WEEK, - TimeUnit.SECONDS - ) - } - } - - companion object { - fun parse(config: Config): SimpleRuntimeConfiguration? = - parseSpecs(config.getString("type"), config.getConfig("specs")) - ?.at(parseSchedules(config.getConfigList("schedules"))) - - private fun parseSpecs(type: String, config: Config): CmdService? { - return when (type.toLowerCase()) { - "java" -> SimpleJavaService.parse(config) - "cmd" -> SimpleCmdService.parse(config) - else -> { - println("Unknown type of service config \"$config\"") - null - } - } - } - - private fun parseSchedules(config: List): Set { - fun Config.parseLocalDayTime() = LocalDayTime( - getString("weekday") - .let { weekday -> - DayOfWeek.values() - .first { it.name.equals(weekday, ignoreCase = true) } - }, - LocalTime.parse(getString("time")) - ) - return config.map { it.parseLocalDayTime() }.toSet() - } - } -} diff --git a/service-runner/src/main/kotlin/de/hpi/cloud/runner/utils/LocalDayTime.kt b/service-runner/src/main/kotlin/de/hpi/cloud/runner/utils/LocalDayTime.kt deleted file mode 100644 index e2e74a9..0000000 --- a/service-runner/src/main/kotlin/de/hpi/cloud/runner/utils/LocalDayTime.kt +++ /dev/null @@ -1,65 +0,0 @@ -package de.hpi.cloud.runner.utils - -import java.time.DayOfWeek -import java.time.LocalTime -import java.time.temporal.* - -infix fun DayOfWeek.at(time: LocalTime) = LocalDayTime(this, time) -val TemporalField.maximumValue get() = this.range().maximum - -class LocalDayTime( - val dayOfWeek: DayOfWeek, - val localTime: LocalTime -) { - override fun toString() = "$dayOfWeek $localTime" - - fun nextOrSameAdjuster(): TemporalAdjuster { - val wantedValue = SecondsOfWeek.getFrom(this) - return TemporalAdjuster { temporal: Temporal -> - val currentValue = temporal.getLong(SecondsOfWeek) - if (wantedValue == currentValue) { - temporal - } else { - val diff = currentValue - wantedValue - temporal.plus( - if (diff >= 0) SecondsOfWeek.SECONDS_IN_WEEK - diff else -diff, - ChronoUnit.SECONDS - ) - } - } - } - - object SecondsOfWeek : TemporalField { - val SECONDS_IN_DAY = (ChronoField.SECOND_OF_DAY.maximumValue + 1) - - override fun isTimeBased() = true - override fun isDateBased() = true - - override fun getBaseUnit() = ChronoUnit.SECONDS - override fun getRangeUnit() = ChronoUnit.WEEKS - override fun range(): ValueRange = ValueRange.of(0, ChronoField.DAY_OF_WEEK.maximumValue * SECONDS_IN_DAY - 1) - override fun rangeRefinedBy(temporal: TemporalAccessor) = range() - val SECONDS_IN_WEEK = maximumValue + 1 - - override fun isSupportedBy(temporal: TemporalAccessor) = - temporal.isSupported(ChronoField.SECOND_OF_DAY) && temporal.isSupported(ChronoField.DAY_OF_WEEK) - - private fun getFrom(dayOfWeek: Int, localTime: Int): Long = ( - localTime - + SECONDS_IN_DAY - * (dayOfWeek - 1) - ) - - fun getFrom(dayOfWeek: DayOfWeek, localTime: LocalTime) = getFrom(dayOfWeek.value, localTime.toSecondOfDay()) - fun getFrom(localDayTime: LocalDayTime) = getFrom(localDayTime.dayOfWeek, localDayTime.localTime) - override fun getFrom(temporal: TemporalAccessor) = - getFrom(temporal[ChronoField.DAY_OF_WEEK], temporal[ChronoField.SECOND_OF_DAY]) - - @Suppress("UNCHECKED_CAST") - override fun adjustInto(temporal: R, newValue: Long): R = ( - temporal - .with(ChronoField.DAY_OF_WEEK, 1 + newValue / SECONDS_IN_DAY) - .with(ChronoField.SECOND_OF_DAY, newValue % SECONDS_IN_DAY) - ) as R - } -}