From 808dfd1b627aafa12f007fdb4acebe024057cb94 Mon Sep 17 00:00:00 2001
From: Jan-Niclas Struewer <j.n.struewer@gmail.com>
Date: Thu, 20 Apr 2023 10:42:24 +0200
Subject: [PATCH] long running tasks now don't block the start of a new task
---
.../iem/dataprovider/git/GitService.kt | 4 ++--
.../{git => taskManager}/TaskManager.kt | 23 +++++++++++++------
src/main/resources/application.properties | 3 ++-
3 files changed, 20 insertions(+), 10 deletions(-)
rename src/main/kotlin/de/fraunhofer/iem/dataprovider/{git => taskManager}/TaskManager.kt (57%)
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/git/GitService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/git/GitService.kt
index 4ba78604..bce20555 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/git/GitService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/git/GitService.kt
@@ -1,8 +1,8 @@
package de.fraunhofer.iem.dataprovider.git
+import de.fraunhofer.iem.dataprovider.taskManager.TaskManager
import org.springframework.stereotype.Service
-import kotlinx.coroutines.*
-import kotlinx.coroutines.channels.*
+
@Service
class GitService(private val taskManager: TaskManager) {
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/git/TaskManager.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
similarity index 57%
rename from src/main/kotlin/de/fraunhofer/iem/dataprovider/git/TaskManager.kt
rename to src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
index 5d335ada..e974fa7f 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/git/TaskManager.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
@@ -1,5 +1,6 @@
-package de.fraunhofer.iem.dataprovider.git
+package de.fraunhofer.iem.dataprovider.taskManager
+import de.fraunhofer.iem.dataprovider.logger.getLogger
import jakarta.annotation.PostConstruct
import jakarta.annotation.PreDestroy
import kotlinx.coroutines.*
@@ -7,6 +8,10 @@ import kotlinx.coroutines.channels.Channel
import org.springframework.stereotype.Component
import kotlin.coroutines.CoroutineContext
+enum class TaskType{
+ REPO_CHANGED
+}
+data class Task(val type: TaskType, val payload: Any)
@Component
class TaskManager : CoroutineScope {
@@ -16,21 +21,25 @@ class TaskManager : CoroutineScope {
// TODO: this could be dangerous to give it unlimited memory
private val tasks = Channel<String>(Channel.UNLIMITED)
+ private val numCores = Runtime.getRuntime().availableProcessors()
+ private val logger = getLogger(javaClass)
@PostConstruct
fun startWorkers() {
- // TODO: spawn threads equal to number of cores
- repeat(10) {
- println("[${Thread.currentThread().name}] launching coroutine")
+ logger.info("Starting $numCores workers.")
+ repeat(numCores-1) {
+ logger.info("[${Thread.currentThread().name}] launching coroutine")
launchWorker(it)
}
}
private fun launchWorker(id: Int) = launch {
for (task in tasks) {
- println("[${Thread.currentThread().name}] Processor #$id received $task")
- executeTask(task)
- println("[${Thread.currentThread().name}] Processor #$id finished $task")
+ launch {
+ logger.debug("[${Thread.currentThread().name}] Processor #$id received $task")
+ executeTask(task)
+ logger.debug("[${Thread.currentThread().name}] Processor #$id finished $task")
+ }
}
}
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
index f763c595..db287326 100644
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -1,4 +1,5 @@
spring.config.import=optional:classpath:.env[.properties]
OPENCODE_GITLAB_URL=https://gitlab.opencode.de/
OPENCODE_GITLAB_TOKEN=${OPENCODE_TOKEN}
-spring.r2dbc.url=${R2DBC_URL}
\ No newline at end of file
+spring.r2dbc.url=${R2DBC_URL}
+logging.level.root=DEBUG
\ No newline at end of file
--
GitLab