From 47f66824f2f8dd8429e7b02bd10200679cbac7ad Mon Sep 17 00:00:00 2001
From: Jan-Niclas Struewer <j.n.struewer@gmail.com>
Date: Wed, 26 Apr 2023 13:16:54 +0200
Subject: [PATCH] code cleanup
---
.../iem/dataprovider/git/GitService.kt | 21 --------
.../gitlab/GitlabConfiguration.kt | 12 -----
.../dataprovider/gitlab/GitlabController.kt | 18 +++++--
.../iem/dataprovider/gitlab/GitlabService.kt | 18 +++----
.../gitlab/OpenCodeGitlabConfiguration.kt | 24 +++++++++
.../dataprovider/taskManager/TaskManager.kt | 49 ++++---------------
.../iem/dataprovider/taskManager/Worker.kt | 7 +--
.../dataprovider/taskManager/tasks/GitTask.kt | 24 +++++++++
.../taskManager/tasks/GitlabTask.kt | 22 +++++++++
.../dataprovider/taskManager/tasks/Task.kt | 24 +++++++++
src/main/resources/application.properties | 4 +-
11 files changed, 134 insertions(+), 89 deletions(-)
delete mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/git/GitService.kt
delete mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabConfiguration.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/OpenCodeGitlabConfiguration.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitTask.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitlabTask.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/Task.kt
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/git/GitService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/git/GitService.kt
deleted file mode 100644
index 1b40b291..00000000
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/git/GitService.kt
+++ /dev/null
@@ -1,21 +0,0 @@
-package de.fraunhofer.iem.dataprovider.git
-
-import de.fraunhofer.iem.dataprovider.taskManager.GitTask
-import de.fraunhofer.iem.dataprovider.taskManager.Task
-import de.fraunhofer.iem.dataprovider.taskManager.TaskManager
-import de.fraunhofer.iem.dataprovider.taskManager.TaskType
-import org.springframework.stereotype.Service
-
-@Service
-class GitService(private val taskManager: TaskManager) {
-
- val list = listOf("https://github.com/eclipse/jgit.git", "https://github.com/secure-software-engineering/phasar", "https://github.com/secure-software-engineering/FlowDroid")
-
- suspend fun cloneGit() {
- println("clone git called")
- for (e in list) {
- val t = GitTask(TaskType.REPO_CHANGED, e, taskManager::addTask)
- taskManager.addTask(t)
- }
- }
-}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabConfiguration.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabConfiguration.kt
deleted file mode 100644
index 1d9eda17..00000000
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabConfiguration.kt
+++ /dev/null
@@ -1,12 +0,0 @@
-package de.fraunhofer.iem.dataprovider.gitlab
-
-import org.springframework.beans.factory.annotation.Value
-import org.springframework.stereotype.Component
-
-@Component
-class GitlabConfiguration {
- @Value("\${OPENCODE_GITLAB_URL}")
- lateinit var host: String
- @Value("\${OPENCODE_GITLAB_TOKEN}")
- lateinit var accessToken: String
-}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabController.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabController.kt
index c062fcb3..829194f0 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabController.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabController.kt
@@ -6,16 +6,28 @@ import org.springframework.web.bind.annotation.RequestBody
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RestController
-data class RepositoryChangedDto(val repoId: Long)
+enum class Platform {
+ OPEN_CODE,
+ GITHUB,
+}
+data class RepositoryChangedDto(val repoId: Long, val platform: Platform)
@RestController
@RequestMapping("/gitlab")
class GitlabController(private val gitlabService: GitlabService) {
private val logger = getLogger(javaClass)
+ // TODO: if the parsing of the request body fails we send a stack
+ // trace back this should be changed to a custom error.
@PostMapping("/repoChanged")
suspend fun repoChanged(@RequestBody repositoryChangedDto: RepositoryChangedDto) {
- logger.info("Repo changed POST request for ID ${repositoryChangedDto.repoId} received.")
- gitlabService.queryRepo(repositoryChangedDto.repoId)
+ logger.info("Repo changed POST request for ID ${repositoryChangedDto.repoId} on platform ${repositoryChangedDto.platform} received.")
+ when (repositoryChangedDto.platform) {
+ Platform.OPEN_CODE -> gitlabService.queryOpenCodeProject(repositoryChangedDto.repoId)
+ else -> {
+ logger.info("Platform currently not supported.")
+ // TODO: send fitting http response
+ }
+ }
}
}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabService.kt
index 13128991..ec09be77 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabService.kt
@@ -1,21 +1,19 @@
package de.fraunhofer.iem.dataprovider.gitlab
-import de.fraunhofer.iem.dataprovider.git.GitService
import de.fraunhofer.iem.dataprovider.logger.getLogger
-import org.eclipse.jgit.api.Git
-import org.gitlab4j.api.GitLabApi
+import de.fraunhofer.iem.dataprovider.taskManager.TaskManager
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.GetGitlabProjectTask
import org.springframework.stereotype.Service
+
+
@Service
-class GitlabService(gitlabConfiguration: GitlabConfiguration, val gitService: GitService) {
+class GitlabService(private val openCodeGitlabConfiguration: OpenCodeGitlabConfiguration, private val taskManager: TaskManager) {
private val logger = getLogger(javaClass)
- private var gitlabApi = GitLabApi(gitlabConfiguration.host, gitlabConfiguration.accessToken)
- suspend fun queryRepo(repoId: Long) {
-// val project = gitlabApi.projectApi.getProject(repoId)
-// logger.info(project.toString())
-// val projectUri = project.sshUrlToRepo
- gitService.cloneGit()
+
+ suspend fun queryOpenCodeProject(repoId: Long) {
+ taskManager.addTask(GetGitlabProjectTask(repoId = repoId, gitlabConfiguration = openCodeGitlabConfiguration, responseChannel = taskManager::addTask))
}
}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/OpenCodeGitlabConfiguration.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/OpenCodeGitlabConfiguration.kt
new file mode 100644
index 00000000..88fa1a70
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/OpenCodeGitlabConfiguration.kt
@@ -0,0 +1,24 @@
+package de.fraunhofer.iem.dataprovider.gitlab
+
+import org.springframework.beans.factory.annotation.Value
+import org.springframework.stereotype.Component
+
+interface GitlabConfiguration {
+ var host: String
+ var accessToken: String
+}
+@Component
+class OpenCodeGitlabConfiguration: GitlabConfiguration {
+ @Value("\${OPENCODE_GITLAB_URL}")
+ override lateinit var host: String
+ @Value("\${OPENCODE_GITLAB_TOKEN}")
+ override lateinit var accessToken: String
+}
+
+@Component
+class GitHubConfiguration: GitlabConfiguration {
+ @Value("\${GITHUB_URL}")
+ override lateinit var host: String
+ @Value("\${GITHUB_TOKEN}")
+ override lateinit var accessToken: String
+}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
index d45f6b74..160edb33 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
@@ -1,46 +1,14 @@
package de.fraunhofer.iem.dataprovider.taskManager
import de.fraunhofer.iem.dataprovider.logger.getLogger
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.GetGitlabProjectTask
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.ITask
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.TaskType
import jakarta.annotation.PreDestroy
import kotlinx.coroutines.*
import kotlinx.coroutines.channels.actor
import org.springframework.stereotype.Component
-enum class TaskType {
- REPO_CHANGED,
- DONE
-}
-interface Task {
- val type: TaskType
- suspend fun execute()
-}
-
-class GitTask(
- override val type: TaskType,
- private val gitUrl: String,
- private val responseChannel: suspend (task: Task) -> Unit
-): Task {
- override suspend fun execute() {
- println("Cloning $gitUrl")
- delay(5000)
- responseChannel(DoneTask("Done 1 with gitUrl $gitUrl"))
- delay(5000)
- responseChannel(DoneTask("Done 2 with gitUrl $gitUrl"))
- // val git: Git = Git.cloneRepository()
-// .setURI("https://github.com/eclipse/jgit.git")
-// .setDirectory("/path/to/repo")
-// .call()
- }
-}
-
-class DoneTask(private val message: String? = null): Task {
- override val type: TaskType = TaskType.DONE
- override suspend fun execute() {
- println("[${Thread.currentThread().name}] I'm done $message")
- }
-}
-
-
/**
* The Task Manager takes tasks and distributes them to
* underlying workers. Internally it uses a channel
@@ -55,18 +23,18 @@ class TaskManager: IWorker {
private val worker = Worker("Worker")
// Should be used for long-running tasks, which DON'T use CPU power and
// are non-blocking
- // private val ioWorker = Worker("IO-Worker", CoroutineScope(Dispatchers.IO))
+ private val ioWorker = Worker("IO-Worker", CoroutineScope(Dispatchers.IO))
// Should be used for long-running CPU heavy tasks, which potentially block.
// private val customWorker = getCustomThreadpoolWorker()
private val logger = getLogger(javaClass)
private val mainScope = CoroutineScope(Dispatchers.Default)
private val actor = mainScope.taskActor()
- override suspend fun addTask(task: Task) {
+ override suspend fun addTask(task: ITask) {
actor.send(task)
}
- private fun CoroutineScope.taskActor() = actor<Task> {
+ private fun CoroutineScope.taskActor() = actor<ITask> {
for (task in channel) {
// For test purposes we have the Done task, which prints a control
// message printed in the task manager.
@@ -74,7 +42,10 @@ class TaskManager: IWorker {
task.execute()
}
logger.info("[${Thread.currentThread().name}] add task called in Task Manager $task")
- worker.addTask(task)
+ when(task) {
+ is GetGitlabProjectTask -> ioWorker.addTask(task)
+ else -> worker.addTask(task)
+ }
logger.info("[${Thread.currentThread().name}] add task finished in Task Manager $task")
}
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/Worker.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/Worker.kt
index e8c8d058..7715df29 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/Worker.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/Worker.kt
@@ -1,6 +1,7 @@
package de.fraunhofer.iem.dataprovider.taskManager
import de.fraunhofer.iem.dataprovider.logger.getLogger
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.ITask
import kotlinx.coroutines.*
import kotlinx.coroutines.channels.Channel
@@ -9,7 +10,7 @@ interface IWorker {
/**
* Adds a new task to the worker's internal tasks channel.
*/
- suspend fun addTask(task: Task)
+ suspend fun addTask(task: ITask)
// TODO: check if we can use autoclosable interface here.
/**
@@ -40,12 +41,12 @@ class Worker(private val name: String,
): IWorker {
// TODO: this could be dangerous to give it unlimited memory
- private val tasks = Channel<Task>(Channel.UNLIMITED)
+ private val tasks = Channel<ITask>(Channel.UNLIMITED)
private val logger = getLogger(javaClass)
- override suspend fun addTask(task: Task) {
+ override suspend fun addTask(task: ITask) {
tasks.send(task)
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitTask.kt
new file mode 100644
index 00000000..09ff4624
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitTask.kt
@@ -0,0 +1,24 @@
+package de.fraunhofer.iem.dataprovider.taskManager.tasks
+
+import org.eclipse.jgit.api.Git
+import java.io.File
+
+
+data class GitProject(val name: String, val uri: String)
+class GitTask(
+ override val type: TaskType,
+ private val gitProject: GitProject,
+ private val responseChannel: suspend (task: ITask) -> Unit
+): Task() {
+ override suspend fun execute() {
+ logger.info("Cloning ${gitProject.name}")
+
+ val git: Git = Git.cloneRepository()
+ .setURI(gitProject.uri)
+ .setDirectory(
+ File("/tmp/opencode/${gitProject.name}-${taskID}")) // TODO: add output path
+ .call()
+ git.close()
+ responseChannel(DoneTask("Successfully cloned ${gitProject.name}"))
+ }
+}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitlabTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitlabTask.kt
new file mode 100644
index 00000000..96ac3b99
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitlabTask.kt
@@ -0,0 +1,22 @@
+package de.fraunhofer.iem.dataprovider.taskManager.tasks
+
+import de.fraunhofer.iem.dataprovider.gitlab.GitlabConfiguration
+import org.gitlab4j.api.GitLabApi
+
+class GetGitlabProjectTask(
+ override val type: TaskType = TaskType.REPO_CHANGED,
+ private val repoId: Long,
+ private val gitlabConfiguration: GitlabConfiguration,
+ private val responseChannel: suspend (task: ITask) -> Unit): Task() {
+
+ private val gitlabApi: GitLabApi = GitLabApi(gitlabConfiguration.host, gitlabConfiguration.accessToken)
+ override suspend fun execute() {
+ logger.info("Gitlab config ${gitlabConfiguration.host} ${gitlabConfiguration.accessToken}")
+ val project = gitlabApi.projectApi.getProject(repoId)
+ logger.info(project.toString())
+ val projectUri = project.sshUrlToRepo
+ val gitProject = GitProject(project.name, projectUri)
+ logger.info("Retrieved project ${project.path} and URI $projectUri")
+ responseChannel(GitTask(this.type, gitProject, responseChannel))
+ }
+}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/Task.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/Task.kt
new file mode 100644
index 00000000..5168bacf
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/Task.kt
@@ -0,0 +1,24 @@
+package de.fraunhofer.iem.dataprovider.taskManager.tasks
+
+import de.fraunhofer.iem.dataprovider.logger.getLogger
+import java.util.*
+
+enum class TaskType {
+ REPO_CHANGED,
+ DONE
+}
+interface ITask {
+ val type: TaskType
+ suspend fun execute()
+}
+
+sealed class Task(protected val taskID: UUID = UUID.randomUUID()): ITask {
+ protected val logger = getLogger(javaClass)
+}
+
+class DoneTask(private val message: String? = null): Task() {
+ override val type: TaskType = TaskType.DONE
+ override suspend fun execute() {
+ logger.info("[${Thread.currentThread().name}] I'm done $message")
+ }
+}
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
index db287326..7ea02d25 100644
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -1,5 +1,7 @@
spring.config.import=optional:classpath:.env[.properties]
OPENCODE_GITLAB_URL=https://gitlab.opencode.de/
OPENCODE_GITLAB_TOKEN=${OPENCODE_TOKEN}
+GITHUB_URL=https://gitlab.opencode.de/
+GITHUB_TOKEN=${GITHUB_TOKEN}
spring.r2dbc.url=${R2DBC_URL}
-logging.level.root=DEBUG
\ No newline at end of file
+#logging.level.root=DEBUG
\ No newline at end of file
--
GitLab