Skip to content
Snippets Groups Projects
Verified Commit 16abceac authored by Jan-Niclas Strüwer's avatar Jan-Niclas Strüwer
Browse files

simplified task setup

parent f2b7b243
No related branches found
No related tags found
No related merge requests found
......@@ -2,7 +2,10 @@ package de.fraunhofer.iem.dataprovider.taskManager
import de.fraunhofer.iem.dataprovider.gitlab.GitConfiguration
import de.fraunhofer.iem.dataprovider.logger.getLogger
import de.fraunhofer.iem.dataprovider.taskManager.tasks.*
import de.fraunhofer.iem.dataprovider.taskManager.tasks.CloneGitTask
import de.fraunhofer.iem.dataprovider.taskManager.tasks.GetGitlabProjectIdTask
import de.fraunhofer.iem.dataprovider.taskManager.tasks.GitProject
import de.fraunhofer.iem.dataprovider.taskManager.tasks.OdcTask
import jakarta.annotation.PreDestroy
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
......@@ -15,7 +18,14 @@ import java.util.*
sealed class Event
class RepoChangedEvent(val gitConfiguration: GitConfiguration, val repoId: Long) : Event()
class SequenceTaskDoneEvent(val sequenceId: UUID) : Event()
sealed class TaskDone : Event() {
abstract val taskId: UUID
}
class GitCloneDone(override val taskId: UUID, val outputDirectory: String) : TaskDone()
class GetGitlabProjectDone(override val taskId: UUID, val gitProject: GitProject) : TaskDone()
class ProcessTaskDone(override val taskId: UUID, val message: String) : TaskDone()
/**
* The Task Manager takes tasks and distributes them to
......@@ -69,84 +79,36 @@ class TaskManager {
private inner class MyActor {
private val logger = getLogger(javaClass)
private val taskSequences = HashMap<UUID, TaskSequence>()
suspend fun onReceive(event: Event) {
logger.info("[${Thread.currentThread().name}] add task called in Task Manager $event")
when (event) {
is RepoChangedEvent -> {
val taskSequence = TaskSequence.getRepoChangedSequence(
event.repoId,
event.gitConfiguration,
responseChannel = ::addEvent
)
taskSequences[taskSequence.sequenceID] = taskSequence
taskSequence.next().forEach {
worker.addTask(it)
}
ioWorker.addTask(GetGitlabProjectIdTask(event.repoId, event.gitConfiguration, ::addEvent))
}
is SequenceTaskDoneEvent -> handleSequenceTaskDone(event.sequenceId)
}
logger.info("[${Thread.currentThread().name}] add task finished in Task Manager $event")
}
is GetGitlabProjectDone -> {
worker.addTask(
CloneGitTask(
event.gitProject,
::addEvent,
"/tmp/opencode"
)
) //TODO: remove hardcoded variable
}
private suspend fun handleSequenceTaskDone(id: UUID) {
is GitCloneDone -> {
worker.addTask(OdcTask(event.outputDirectory, event.outputDirectory, ::addEvent))
}
val sequence = taskSequences.getOrDefault(id, TaskSequence.EmptySequence)
if (sequence.hasNext()) {
sequence.next().forEach {
when (it) {
is GetGitlabProjectTask -> ioWorker.addTask(it)
else -> worker.addTask(it)
}
else -> {
logger.info("Received event without special handling associated $event")
}
} else {
taskSequences.remove(id)
logger.info("[${Thread.currentThread().name}] sequence with id $id finished.")
}
logger.info("[${Thread.currentThread().name}] add task finished in Task Manager $event")
}
}
}
private class TaskSequence(
private val tasks: List<List<ITask>>,
val sequenceID: UUID = UUID.randomUUID()
) : Iterator<List<ITask>> {
companion object Factory {
val EmptySequence = TaskSequence(listOf())
fun getRepoChangedSequence(
repoId: Long,
gitConfiguration: GitConfiguration,
responseChannel: suspend (task: Event) -> Unit
): TaskSequence {
val id = UUID.randomUUID()
val gitlabTask = GetGitlabProjectIdTask(repoId, gitConfiguration, responseChannel, id)
val gitTask = GitTask(TaskType.REPO_CHANGED, gitlabTask::getResult, responseChannel, id)
return TaskSequence(
listOf(
listOf(gitlabTask),
listOf(gitTask),
listOf(
JavaTask(TaskType.REPO_CHANGED, "-version", responseChannel = responseChannel, sequenceId = id),
JavaTask(TaskType.REPO_CHANGED, "-help", responseChannel = responseChannel, sequenceId = id)
)
),
id
)
}
}
private val it = tasks.iterator()
override fun next(): List<ITask> {
return it.next()
}
override fun hasNext(): Boolean {
return it.hasNext()
}
}
package de.fraunhofer.iem.dataprovider.taskManager.tasks
import de.fraunhofer.iem.dataprovider.taskManager.Event
import de.fraunhofer.iem.dataprovider.taskManager.GitCloneDone
import org.eclipse.jgit.api.Git
import java.nio.file.Paths
data class GitProject(val name: String, val uri: String)
class CloneGitTask(
private val gitProject: GitProject,
override val responseChannel: suspend (task: Event) -> Unit,
private val outputPath: String,
) : Task() {
override suspend fun execute() {
logger.info("Cloning ${gitProject.name}")
val outputDirectory = Paths.get(outputPath, "${gitProject.name}-${taskID}")
val git: Git = Git.cloneRepository()
.setURI(gitProject.uri)
.setDirectory(outputDirectory.toFile())
.call()
git.close()
responseChannel(GitCloneDone(taskID, outputDirectory.toString()))
logger.info("Finished cloning ${gitProject.name}")
}
}
\ No newline at end of file
package de.fraunhofer.iem.dataprovider.taskManager.tasks
import de.fraunhofer.iem.dataprovider.taskManager.Event
import de.fraunhofer.iem.dataprovider.taskManager.SequenceTaskDoneEvent
import java.util.*
data class GitProject(val name: String, val uri: String)
class GitTask(
override val type: TaskType,
private val getGitProject: () -> GitProject,
// private val outputPath: String,
private val responseChannel: suspend (task: Event) -> Unit,
override val sequenceId: UUID?
): Task() {
override suspend fun execute() {
val gitProject = getGitProject()
logger.info("Cloning ${gitProject.name}")
if (sequenceId != null) {
responseChannel(SequenceTaskDoneEvent(sequenceId!!))
}
// val git: Git = Git.cloneRepository()
// .setURI(gitProject.uri)
// .setDirectory(
// File("/tmp/opencode/${gitProject.name}-${taskID}")) // TODO: add output path
// .call()
// git.close()
// responseChannel(DoneTask("Successfully cloned ${gitProject.name}"))
}
}
\ No newline at end of file
......@@ -2,48 +2,39 @@ package de.fraunhofer.iem.dataprovider.taskManager.tasks
import de.fraunhofer.iem.dataprovider.gitlab.GitConfiguration
import de.fraunhofer.iem.dataprovider.taskManager.Event
import de.fraunhofer.iem.dataprovider.taskManager.SequenceTaskDoneEvent
import de.fraunhofer.iem.dataprovider.taskManager.GetGitlabProjectDone
import org.gitlab4j.api.GitLabApi
import java.util.*
sealed class GetGitlabProjectTask(
override val type: TaskType = TaskType.REPO_CHANGED,
protected open val repoId: Any,
private val gitlabConfiguration: GitConfiguration,
private val responseChannel: suspend (task: Event) -> Unit,
override val sequenceId: UUID? = null
): Task() {
sealed class GetGitlabProjectTask : Task() {
protected abstract val repoId: Any
protected abstract val gitlabConfiguration: GitConfiguration
private val gitlabApi: GitLabApi = GitLabApi(gitlabConfiguration.host, gitlabConfiguration.accessToken)
private lateinit var gitProject: GitProject
override suspend fun execute() {
logger.info("Gitlab config ${gitlabConfiguration.host} ${gitlabConfiguration.accessToken}")
val project = gitlabApi.projectApi.getProject(repoId)
logger.info(project.toString())
val projectUri = project.httpUrlToRepo
gitProject = GitProject(project.name, projectUri)
logger.info("Retrieved project ${project.path} and url $projectUri")
if (sequenceId != null) {
responseChannel(SequenceTaskDoneEvent(sequenceId!!))
}
// responseChannel(GitTask(this.type, gitProject, responseChannel))
}
val gitProject = GitProject(project.name, projectUri)
fun getResult(): GitProject {
return gitProject
responseChannel(GetGitlabProjectDone(taskID, gitProject))
logger.info("Retrieved project ${project.path} and url $projectUri")
}
}
class GetGitlabProjectPathTask(
override val repoId: String,
private val gitlabConfiguration: GitConfiguration,
private val responseChannel: suspend (task: Event) -> Unit,
override val sequenceId: UUID? = null
): GetGitlabProjectTask(repoId = repoId, gitlabConfiguration = gitlabConfiguration, responseChannel = responseChannel )
override val gitlabConfiguration: GitConfiguration,
override val responseChannel: suspend (task: Event) -> Unit,
) : GetGitlabProjectTask()
class GetGitlabProjectIdTask(
override val repoId: Long,
private val gitlabConfiguration: GitConfiguration,
private val responseChannel: suspend (task: Event) -> Unit,
override val sequenceId: UUID? = null
): GetGitlabProjectTask(repoId = repoId, gitlabConfiguration = gitlabConfiguration, responseChannel = responseChannel )
override val gitlabConfiguration: GitConfiguration,
override val responseChannel: suspend (task: Event) -> Unit
) : GetGitlabProjectTask()
package de.fraunhofer.iem.dataprovider.taskManager.tasks
import de.fraunhofer.iem.dataprovider.taskManager.Event
import de.fraunhofer.iem.dataprovider.taskManager.ProcessTaskDone
import org.springframework.core.io.ClassPathResource
import org.springframework.core.io.Resource
class OdcTask(projectPath: String, outputPath: String, override val responseChannel: suspend (task: Event) -> Unit) :
ProcessTask() {
override val flags: Array<String> = arrayOf(outputPath, projectPath)
private val resource: Resource = ClassPathResource("scripts/odc.sh")
override val execPath: String = resource.toString()
override suspend fun handleProcessReturn(p: Process) {
logger.info(resource.toString())
val returnMessage = "Odc finished with exit code ${p.exitValue()}"
val output = String(p.inputStream.readAllBytes())
logger.info("Process output $output")
responseChannel(ProcessTaskDone(taskID, returnMessage))
}
}
package de.fraunhofer.iem.dataprovider.taskManager.tasks
import de.fraunhofer.iem.dataprovider.taskManager.Event
import de.fraunhofer.iem.dataprovider.taskManager.SequenceTaskDoneEvent
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import java.util.*
sealed class ProcessTask(protected open vararg val flags: String): Task() {
sealed class ProcessTask : Task() {
protected abstract val flags: Array<String>
protected abstract val execPath: String
private val mainScope = CoroutineScope(Dispatchers.Default)
override suspend fun execute() {
val p = ProcessBuilder(execPath, *flags)
.start()
val process = ProcessBuilder(execPath, *flags).start()
p.onExit().thenApply { p1 ->
process.onExit().thenApply { p1 ->
mainScope.launch {
handleProcessReturn(p1)
}
......@@ -25,22 +23,3 @@ sealed class ProcessTask(protected open vararg val flags: String): Task() {
abstract suspend fun handleProcessReturn(p: Process)
}
/**
* Dummy Task to simulate running external processes.
*/
class JavaTask(
override val type: TaskType,
override vararg val flags: String,
private val responseChannel: suspend (task: Event) -> Unit,
override val sequenceId: UUID?
): ProcessTask() {
override val execPath: String = "java"
override suspend fun handleProcessReturn(p: Process) {
// responseChannel(DoneTask("Done with external process. Exit value ${p.exitValue()}"))
if (sequenceId != null) {
responseChannel(SequenceTaskDoneEvent(sequenceId!!))
}
}
}
\ No newline at end of file
package de.fraunhofer.iem.dataprovider.taskManager.tasks
import de.fraunhofer.iem.dataprovider.logger.getLogger
import de.fraunhofer.iem.dataprovider.taskManager.Event
import java.util.*
enum class TaskType {
REPO_CHANGED,
DONE
}
interface ITask {
val type: TaskType
suspend fun execute()
val taskID: UUID
}
sealed class Task(open val sequenceId: UUID? = null) : ITask {
sealed class Task : ITask {
override val taskID: UUID = UUID.randomUUID()
protected val logger = getLogger(javaClass)
}
class DoneTask(private val message: String? = null): Task() {
override val type: TaskType = TaskType.DONE
override suspend fun execute() {
logger.info("[${Thread.currentThread().name}] I'm done $message")
}
protected abstract val responseChannel: suspend (task: Event) -> Unit
}
#!/bin/sh
DC_VERSION="latest"
DC_DIRECTORY=${1}/OWASP-Dependency-Check
DC_PROJECT="dependency-check scan: ${2}"
DATA_DIRECTORY="$DC_DIRECTORY/data"
CACHE_DIRECTORY="$DC_DIRECTORY/data/cache"
if [ ! -d "$DATA_DIRECTORY" ]; then
echo "Initially creating persistent directory: $DATA_DIRECTORY"
mkdir -p "$DATA_DIRECTORY"
fi
if [ ! -d "$CACHE_DIRECTORY" ]; then
echo "Initially creating persistent directory: $CACHE_DIRECTORY"
mkdir -p "$CACHE_DIRECTORY"
fi
# Make sure we are using the latest version
docker pull owasp/dependency-check:$DC_VERSION
docker run --rm \
-e user="$USER" \
-u $(id -u ${USER}):$(id -g ${USER}) \
--volume ${2}:/src:z \
--volume "$DATA_DIRECTORY":/usr/share/dependency-check/data:z \
--volume "$DC_DIRECTORY"/odc-reports:/report:z \
owasp/dependency-check:$DC_VERSION \
--scan /src \
--format "ALL" \
--project "$DC_PROJECT" \
--out /report
# Use suppression like this: (where /src == $pwd)
# --suppression "/src/security/dependency-check-suppression.xml"
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Consent

On this website, we use the web analytics service Matomo to analyze and review the use of our website. Through the collected statistics, we can improve our offerings and make them more appealing for you. Here, you can decide whether to allow us to process your data and set corresponding cookies for these purposes, in addition to technically necessary cookies. Further information on data protection—especially regarding "cookies" and "Matomo"—can be found in our privacy policy. You can withdraw your consent at any time.