diff --git a/.jpb/jpb-settings.xml b/.jpb/jpb-settings.xml
new file mode 100644
index 0000000000000000000000000000000000000000..9e8fa78703d266a57b04865c38a859b6f17e8e9f
--- /dev/null
+++ b/.jpb/jpb-settings.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+ <component name="JpaPluginProjectSettings">
+ <option name="lastSelectedLanguage" value="Kotlin"/>
+ </component>
+</project>
\ No newline at end of file
diff --git a/build.gradle.kts b/build.gradle.kts
index b68ab8b6df49eda8685df3a8eb6ed342a564d750..5bf465c9fe3856729b6f1a245449f0eccc8b2a89 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -1,11 +1,22 @@
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
- id("org.springframework.boot") version "3.0.5"
+ id("org.springframework.boot") version "3.0.6"
id("io.spring.dependency-management") version "1.1.0"
+ id("org.jetbrains.kotlin.plugin.allopen") version "1.8.21"
kotlin("jvm") version "1.7.22"
kotlin("plugin.spring") version "1.7.22"
- kotlin("plugin.serialization") version "1.8.21"
+ kotlin("plugin.jpa") version "1.7.22"
+ kotlin("plugin.serialization") version "1.8.21"
+}
+
+allOpen {
+ annotation("javax.persistence.Entity")
+ annotation("javax.persistence.Embeddable")
+ annotation("javax.persistence.MappedSuperclass")
+ annotation("jakarta.persistence.Entity")
+ annotation("jakarta.persistence.Embeddable")
+ annotation("jakarta.persistence.MappedSuperclass")
}
group = "de.fraunhofer.iem"
@@ -23,7 +34,8 @@ repositories {
}
dependencies {
- implementation("org.springframework.boot:spring-boot-starter-data-r2dbc")
+ implementation("org.springframework.boot:spring-boot-starter-data-jpa")
+ implementation("org.springframework.boot:spring-boot-starter-validation")
implementation("org.springframework.boot:spring-boot-starter-security")
implementation("org.springframework.boot:spring-boot-starter-webflux")
implementation("com.fasterxml.jackson.module:jackson-module-kotlin")
@@ -32,12 +44,10 @@ dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactive")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactor")
implementation("org.gitlab4j:gitlab4j-api:6.0.0-rc.1")
- implementation("org.eclipse.jgit:org.eclipse.jgit:6.5.0.202303070854-r")
- implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.5.0")
- // for deployment replace with io.r2dbc:r2dbc-postgresql
- implementation("io.r2dbc:r2dbc-h2")
+ implementation("org.eclipse.jgit:org.eclipse.jgit:6.5.0.202303070854-r")
+ implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.5.0")
developmentOnly("org.springframework.boot:spring-boot-devtools")
- runtimeOnly("com.h2database:h2")
+ runtimeOnly("org.postgresql:postgresql")
testImplementation("org.springframework.boot:spring-boot-starter-test")
testImplementation("org.springframework.security:spring-security-test")
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.4")
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/DataProviderApplication.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/DataProviderApplication.kt
index 8953f77e2ec40f9f5a889971fce14cb5bf5a704d..4b84022572a922582ec86c2b03484617cb575209 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/DataProviderApplication.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/DataProviderApplication.kt
@@ -3,20 +3,6 @@ package de.fraunhofer.iem.dataprovider
import org.springframework.boot.autoconfigure.SpringBootApplication
import org.springframework.boot.runApplication
-// Taken from https://spring.io/guides/tutorials/spring-webflux-kotlin-rsocket/
-//@Configuration
-//class Config {
-//
-// @Bean
-// fun initializer(connectionFactory: ConnectionFactory): ConnectionFactoryInitializer {
-// val initializer = ConnectionFactoryInitializer()
-// initializer.setConnectionFactory(connectionFactory)
-// val populator = CompositeDatabasePopulator()
-// populator.addPopulators(ResourceDatabasePopulator(ClassPathResource("./sql/schema.sql")))
-// initializer.setDatabasePopulator(populator)
-// return initializer
-// }
-//}
@SpringBootApplication
class DataProviderApplication
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt
new file mode 100644
index 0000000000000000000000000000000000000000..fb409de39884cd3ec58f0a4f13d1c09b04502750
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt
@@ -0,0 +1,43 @@
+package de.fraunhofer.iem.dataprovider.gitlab
+
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRun
+import jakarta.persistence.*
+import java.util.*
+
+@Entity
+@Table(name = "repository")
+class Repository {
+
+ @Id
+ @GeneratedValue(strategy = GenerationType.AUTO)
+ @Column(name = "id", nullable = false)
+ var id: UUID? = null
+
+ @OrderBy("time_stamp DESC")
+ @OneToMany(mappedBy = "repository", cascade = [CascadeType.ALL], orphanRemoval = true)
+ var toolRuns: MutableList<ToolRun> = mutableListOf()
+
+ @Column(name = "name")
+ var name: String? = null
+
+ @Column(name = "repo_id")
+ var repoId: Long? = null
+
+ @Enumerated
+ @Column(name = "platform")
+ var platform: Platform? = null
+
+ @Column(name = "url")
+ var url: String? = null
+
+ fun addToolResults(toolRuns: Collection<ToolRun>) {
+ toolRuns.forEach {
+ this.addToolResult(it)
+ }
+ }
+
+ fun addToolResult(toolRun: ToolRun) {
+ toolRuns.add(toolRun)
+ toolRun.repository = this
+ }
+}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/RepositoryRepository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/RepositoryRepository.kt
new file mode 100644
index 0000000000000000000000000000000000000000..23fa8211136a14ab2b5cc0f127af6219a7f20326
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/RepositoryRepository.kt
@@ -0,0 +1,13 @@
+package de.fraunhofer.iem.dataprovider.gitlab;
+
+import org.springframework.data.jpa.repository.JpaRepository
+import java.util.*
+
+interface RepositoryRepository : JpaRepository<Repository, UUID> {
+
+
+ fun existsByRepoIdAndPlatform(repoId: Long, platform: Platform): Boolean
+
+
+ fun findByRepoIdAndPlatform(repoId: Long, platform: Platform): Repository?
+}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt
index 46b4dba60980920739e7dab92ee1f9ce6dc93f73..bf90606284b159bcb5d01d5cb6cdec6868a3b90a 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt
@@ -1,62 +1,69 @@
package de.fraunhofer.iem.dataprovider.sarif
import de.fraunhofer.iem.dataprovider.toolResult.*
+import de.fraunhofer.iem.dataprovider.toolResult.Rule
+import de.fraunhofer.iem.dataprovider.toolResult.Tool
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import java.io.IOException
import java.nio.file.Path
+import java.sql.Timestamp
+import java.time.Instant
+import java.util.*
-fun Rule.asDbObject(): RuleDb {
- return RuleDb(
- ruleId = this.id,
- name = this.name,
- shortDescription = this.shortDescription.text,
- helpUri = this.helpUri
- )
+fun de.fraunhofer.iem.dataprovider.sarif.Rule.asDbObject(): Rule {
+ val rule = Rule()
+ rule.sarifRuleId = this.id
+ rule.shortDescription = this.shortDescription.text
+ return rule
}
-fun Tool.asDbObject(): ToolDb {
- return ToolDb(
- fullName = this.driver.fullName,
- name = this.driver.name,
- rules = this.driver.rules.map { it.asDbObject() },
- downloadUri = this.driver.downloadUri,
- guid = this.driver.guid,
- informationUri = this.driver.informationUri,
- language = this.driver.language,
- organization = this.driver.organization,
- semanticVersion = this.driver.semanticVersion,
- version = this.driver.version,
- )
-}
-fun Location.asDbObject(): ResultLocationDb {
- return ResultLocationDb(
- artifactLocationUri = this.physicalLocation.artifactLocation.uri,
- endColumn = this.physicalLocation.region.endColumn,
- endLine = this.physicalLocation.region.endLine,
- startColumn = this.physicalLocation.region.startColumn,
- startLine = this.physicalLocation.region.startLine,
- )
+fun de.fraunhofer.iem.dataprovider.sarif.Tool.asDbObject(): Tool {
+ val tool = Tool()
+ tool.addRules(this.driver.rules.map { it.asDbObject() })
+ tool.name = this.driver.fullName
+ tool.version = this.driver.version
+ return tool
}
-fun Result.asDbObject(): ToolResultDb {
- return ToolResultDb(
- level = this.level,
- locations = this.locations.map { it.asDbObject() },
- message = this.message.text,
- ruleId = this.ruleId
- )
+fun Location.asDbObject(): ResultLocation {
+ val resultLocation = ResultLocation()
+ resultLocation.endColumn = this.physicalLocation.region.endColumn
+ resultLocation.endLine = this.physicalLocation.region.endLine
+ resultLocation.startColumn = this.physicalLocation.region.startColumn
+ resultLocation.startLine = this.physicalLocation.region.startLine
+ resultLocation.artifactLocation = this.physicalLocation.artifactLocation.uri
+ return resultLocation
}
-fun Sarif.asDbObject(): List<ToolResultsDb> {
+fun Result.asDbObject(rules: Collection<Rule>): ToolResult {
+ val toolResult = ToolResult()
+
+ when (this.level.uppercase(Locale.getDefault())) {
+ "WARNING" -> toolResult.level = Level.Warning
+ "ERROR" -> toolResult.level = Level.Error
+ "NOTE" -> toolResult.level = Level.Note
+ else -> toolResult.level = Level.None
+ }
+
+ rules
+ .filter { it.sarifRuleId == this.ruleId }
+ .forEach { it.addResultLocation(toolResult) }
+
+ toolResult.message = this.message.text
+ toolResult.addLocations(this.locations.map { it.asDbObject() })
+ return toolResult
+}
+fun Sarif.asDbObject(): List<ToolRun> {
return this.runs.map { run ->
- ToolResultsDb(
- results = run.results.map { it.asDbObject() },
- tool = run.tool.asDbObject(),
- )
+ val tr = ToolRun()
+ tr.tool = run.tool.asDbObject()
+ tr.addToolResults(run.results.map { it.asDbObject(tr.tool!!.rules) })
+ tr.timeStamp = Timestamp.from(Instant.now())
+ tr
}
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
index 7a50404ec5fd737e36b5768918c6734683b7c0b1..c6488980bf5f51bf85325a6779cdebbe2c598b30 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
@@ -1,8 +1,14 @@
package de.fraunhofer.iem.dataprovider.taskManager
import de.fraunhofer.iem.dataprovider.gitlab.GitConfiguration
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.logger.getLogger
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.*
+import de.fraunhofer.iem.dataprovider.sarif.Sarif
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.CloneGitTask
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.DetektTask
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.GetGitlabProjectTask
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.GitRepository
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
import jakarta.annotation.PreDestroy
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@@ -21,17 +27,23 @@ sealed class TaskDone : Event() {
abstract val taskId: UUID
}
-class GitCloneDone(override val taskId: UUID, val outputDirectory: String) : TaskDone()
-class GetGitlabProjectDone(override val taskId: UUID, val gitProject: GitProject) : TaskDone()
+class GitCloneDone(override val taskId: UUID, val repoId: UUID, val outputDirectory: String) : TaskDone()
+class GetGitlabProjectDone(override val taskId: UUID, val repoId: UUID, val gitRepository: GitRepository) : TaskDone()
class ProcessTaskDone(override val taskId: UUID, val message: String) : TaskDone()
+class DetektDone(override val taskId: UUID, val sarif: Sarif) : TaskDone()
+
/**
* The Task Manager takes tasks and distributes them to
* underlying workers. Internally it uses a channel
* to manage incoming tasks.
*/
@Component
-class TaskManager(private val config: Config) {
+class TaskManager(
+ private val config: Config,
+ private val repositoryRepository: RepositoryRepository,
+ private val toolRunRepository: ToolRunRepository
+) {
// The used default dispatcher is ok for CPU-bound workloads. However,
// if they block for a long time it's better to use a custom thread
@@ -83,13 +95,22 @@ class TaskManager(private val config: Config) {
when (event) {
is RepoChangedEvent -> {
- ioWorker.addTask(GetGitlabProjectTask(event.repoId, event.gitConfiguration, ::addEvent))
+ ioWorker.addTask(
+ GetGitlabProjectTask(
+ event.repoId,
+ event.gitConfiguration,
+ ::addEvent,
+ repositoryRepository
+ )
+ )
}
is GetGitlabProjectDone -> {
- worker.addTask(
+
+ ioWorker.addTask(
CloneGitTask(
- event.gitProject,
+ event.gitRepository,
+ event.repoId,
::addEvent,
config.gitProjectPath
)
@@ -99,8 +120,17 @@ class TaskManager(private val config: Config) {
is GitCloneDone -> {
// TODO: analysis results path should be unique or best not placed in the git folder
val analysisResultsPath = Paths.get(event.outputDirectory, "tool-results").toString()
- worker.addTask(OdcTask(event.outputDirectory, analysisResultsPath, ::addEvent))
- worker.addTask(DetektTask(event.outputDirectory, analysisResultsPath, ::addEvent))
+// worker.addTask(OdcTask(event.outputDirectory, analysisResultsPath, ::addEvent))
+ worker.addTask(
+ DetektTask(
+ event.outputDirectory,
+ analysisResultsPath,
+ ::addEvent,
+ event.repoId,
+ repositoryRepository,
+ toolRunRepository
+ )
+ )
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/CloneGitTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/CloneGitTask.kt
index 3cd1753c6daa2a269936d10a7f4b0755c2fd440b..187f4aeba00b6632449585554e92bc1bcacd5960 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/CloneGitTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/CloneGitTask.kt
@@ -1,33 +1,45 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
+import de.fraunhofer.iem.dataprovider.gitlab.Platform
+import de.fraunhofer.iem.dataprovider.gitlab.Repository
import de.fraunhofer.iem.dataprovider.taskManager.Event
import de.fraunhofer.iem.dataprovider.taskManager.GitCloneDone
import org.eclipse.jgit.api.Git
import java.nio.file.Paths
+import java.util.*
-data class GitProject(val name: String, val uri: String)
+data class GitRepository(val name: String, val uri: String, val id: Long, val platform: Platform)
+
+fun GitRepository.toDbObject(): Repository {
+ val repo = Repository()
+ repo.name = this.name
+ repo.repoId = this.id
+ repo.platform = this.platform
+ return repo
+}
class CloneGitTask(
- private val gitProject: GitProject,
+ private val gitRepository: GitRepository,
+ private val repoDbId: UUID,
override val responseChannel: suspend (task: Event) -> Unit,
private val outputPath: String,
) : Task() {
override suspend fun execute() {
- val outputDirName = "${gitProject.name}-${taskID}"
+ val outputDirName = "${gitRepository.name}-${taskID}"
val outputDirectory = Paths.get(outputPath, outputDirName)
- logger.info("Cloning ${gitProject.name} into $outputDirectory")
+ logger.info("Cloning ${gitRepository.name} into $outputDirectory")
val git: Git = Git.cloneRepository()
- .setURI(gitProject.uri)
+ .setURI(gitRepository.uri)
.setDirectory(outputDirectory.toFile())
.call()
git.close()
- responseChannel(GitCloneDone(taskID, outputDirectory.toString()))
+ responseChannel(GitCloneDone(taskID, repoDbId, outputDirectory.toString()))
- logger.info("Finished cloning ${gitProject.name}")
+ logger.info("Finished cloning ${gitRepository.name}")
}
}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt
index 852c39d2e2489a664fb707a75f30c457683917b0..18d46f475692b94bf787c7a36b30bafa89c504ea 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt
@@ -1,37 +1,32 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
-import de.fraunhofer.iem.dataprovider.sarif.getSarifFromFilePath
+
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
+import de.fraunhofer.iem.dataprovider.sarif.Sarif
+import de.fraunhofer.iem.dataprovider.taskManager.DetektDone
import de.fraunhofer.iem.dataprovider.taskManager.Event
-import de.fraunhofer.iem.dataprovider.taskManager.ProcessTaskDone
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
import org.springframework.core.io.ClassPathResource
import org.springframework.core.io.Resource
+import java.nio.file.Path
import java.nio.file.Paths
+import java.util.*
-class DetektTask(projectPath: String, outputPath: String, override val responseChannel: suspend (task: Event) -> Unit) :
- ProcessTask() {
+class DetektTask(
+ projectPath: String, outputPath: String, override val responseChannel: suspend (task: Event) -> Unit,
+ override val repoId: UUID,
+ override val repository: RepositoryRepository,
+ override val toolRunRepository: ToolRunRepository
+) : SarifTask() {
private val resource: Resource = ClassPathResource("scripts/detekt.sh")
override val flags: Array<String> = arrayOf(resource.file.absolutePath, projectPath, outputPath)
-
override val execPath: String = "/bin/sh"
+ override val resultPath: Path = Paths.get(outputPath, "detekt", "report.sarif")
- private val resultPath = Paths.get(outputPath, "detekt", "report.sarif")
- override suspend fun handleProcessReturn(p: Process) {
- logger.info(resource.toString())
-
- val returnMessage = "Odc finished with exit code ${p.exitValue()}"
-// val output = String(p.inputStream.readAllBytes())
-// logger.info("Process output $output")
-
- val sarifResult = getSarifFromFilePath(resultPath)
-
- sarifResult.runs.forEach {
- it.results.forEach {
- logger.info("run in sarif result message: ${it.message}")
- }
- }
- responseChannel(ProcessTaskDone(taskID, returnMessage))
+ override suspend fun sendResult(sarif: Sarif) {
+ responseChannel(DetektDone(this.taskID, sarif))
}
}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitlabTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitlabTask.kt
index 7ac7a188107d48f3a1271a2e64b88b21590d1b61..3a465cbee48b05ad7ffa0f8b09f1539bdcb77a57 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitlabTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitlabTask.kt
@@ -1,6 +1,8 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
import de.fraunhofer.iem.dataprovider.gitlab.GitConfiguration
+import de.fraunhofer.iem.dataprovider.gitlab.Platform
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.taskManager.Event
import de.fraunhofer.iem.dataprovider.taskManager.GetGitlabProjectDone
import org.gitlab4j.api.GitLabApi
@@ -8,7 +10,8 @@ import org.gitlab4j.api.GitLabApi
class GetGitlabProjectTask(
private val repoId: Long, private val gitlabConfiguration: GitConfiguration,
- override val responseChannel: suspend (task: Event) -> Unit
+ override val responseChannel: suspend (task: Event) -> Unit,
+ private val repository: RepositoryRepository
) : Task() {
@@ -20,10 +23,17 @@ class GetGitlabProjectTask(
val project = gitlabApi.projectApi.getProject(repoId)
logger.info(project.toString())
val projectUri = project.httpUrlToRepo
- val gitProject = GitProject(project.path, projectUri)
- responseChannel(GetGitlabProjectDone(taskID, gitProject))
+ val gitRepository = GitRepository(project.path, projectUri, repoId, Platform.OPEN_CODE)
+
+ var repo = repository.findByRepoIdAndPlatform(gitRepository.id, gitRepository.platform)
+ if (repo == null) {
+ logger.info("Repository ${gitRepository.id} ${gitRepository.platform} doesn't exist in database.")
+ repo = repository.save(gitRepository.toDbObject())
+ }
logger.info("Retrieved project ${project.path} and url $projectUri")
+
+ responseChannel(GetGitlabProjectDone(taskID, repo.id!!, gitRepository))
}
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt
index 46c6a45a2de05871de9a25cb286d004b50c29193..b9fe3691b0d954200e131faa1e01cdea0969c338 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt
@@ -1,10 +1,18 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
+import de.fraunhofer.iem.dataprovider.sarif.Sarif
+import de.fraunhofer.iem.dataprovider.sarif.asDbObject
+import de.fraunhofer.iem.dataprovider.sarif.getSarifFromFilePath
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
+import org.springframework.transaction.annotation.Transactional
import java.nio.file.Files
+import java.nio.file.Path
import java.nio.file.Paths
+import java.util.*
sealed class ProcessTask : Task() {
protected abstract val flags: Array<String>
@@ -33,4 +41,35 @@ sealed class ProcessTask : Task() {
}
abstract suspend fun handleProcessReturn(p: Process)
+
+}
+@Transactional
+sealed class SarifTask : ProcessTask() {
+ protected abstract val repoId: UUID
+ protected abstract val repository: RepositoryRepository
+ protected abstract val toolRunRepository: ToolRunRepository
+ protected abstract val resultPath: Path
+
+
+ override suspend fun handleProcessReturn(p: Process) {
+
+ // TODO: check process exit codes. Current problem, if an analysis
+ // has findings it might return an exit code != 0 even tho the process finished correctly
+
+ logger.info("Handle Process return in $javaClass")
+ val sarifResult = getSarifFromFilePath(resultPath)
+ val repo = repository.findById(repoId)
+ val sarifDb = sarifResult.asDbObject()
+ val results = toolRunRepository.saveAll(sarifDb)
+
+ repo.ifPresent {
+// Hibernate.initialize(it.toolRuns) // TODO: this fails because the session seems to be closed and add won't work because toolRuns are loaded lazily
+ it.addToolResults(results)
+ repository.save(it)
+ }
+ sendResult(sarifResult)
+ }
+
+ abstract suspend fun sendResult(sarif: Sarif)
+
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRessults.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRessults.kt
deleted file mode 100644
index 924f5a964644117e7ec22c66415b5f0e6012f87a..0000000000000000000000000000000000000000
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRessults.kt
+++ /dev/null
@@ -1,46 +0,0 @@
-package de.fraunhofer.iem.dataprovider.toolResult
-
-import org.springframework.data.annotation.Id
-import org.springframework.data.relational.core.mapping.Table
-
-@Table("TOOL_RESULTS")
-data class ToolResultsDb(
- val results: List<ToolResultDb>,
- val tool: ToolDb,
- @Id var id: String? = null
-)
-
-data class ToolResultDb(
- val level: String,
- val locations: List<ResultLocationDb>,
- val message: String,
- val ruleId: String,
-)
-
-data class ResultLocationDb(
- val artifactLocationUri: String,
- val endColumn: Int,
- val endLine: Int,
- val startColumn: Int,
- val startLine: Int,
-)
-
-data class ToolDb(
- val fullName: String,
- val name: String,
- val rules: List<RuleDb>,
- val downloadUri: String? = null,
- val guid: String? = null,
- val informationUri: String? = null,
- val language: String? = null,
- val organization: String? = null,
- val semanticVersion: String? = null,
- val version: String? = null,
-)
-
-data class RuleDb(
- val ruleId: String,
- val name: String,
- val shortDescription: String,
- val helpUri: String? = null,
-)
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsPersistenceService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsPersistenceService.kt
deleted file mode 100644
index 785e693aa9db9b2ec3a55d5276587d418fc061a2..0000000000000000000000000000000000000000
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsPersistenceService.kt
+++ /dev/null
@@ -1,13 +0,0 @@
-package de.fraunhofer.iem.dataprovider.toolResult
-
-import de.fraunhofer.iem.dataprovider.sarif.Sarif
-import org.springframework.stereotype.Service
-
-@Service
-class ToolResultsPersistenceService(val toolResultsRepository: ToolResultsRepository) {
-
- suspend fun save(sarif: Sarif) {
-// toolResultsRepository.save(sarif.asDbObject())
- }
-
-}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsRepository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsRepository.kt
deleted file mode 100644
index 4d2c3d26edf4884df317ff081cfe9aee8d7c9b3e..0000000000000000000000000000000000000000
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsRepository.kt
+++ /dev/null
@@ -1,6 +0,0 @@
-package de.fraunhofer.iem.dataprovider.toolResult
-
-import org.springframework.data.repository.kotlin.CoroutineCrudRepository
-
-interface ToolResultsRepository : CoroutineCrudRepository<ToolResultsDb, String> {
-}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
new file mode 100644
index 0000000000000000000000000000000000000000..ed71bb931b55d70852c927aecc7ea62325be1338
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
@@ -0,0 +1,218 @@
+package de.fraunhofer.iem.dataprovider.toolResult
+
+import de.fraunhofer.iem.dataprovider.gitlab.Repository
+import jakarta.persistence.*
+import org.hibernate.Hibernate
+import org.hibernate.annotations.JdbcTypeCode
+import org.hibernate.type.SqlTypes
+import java.sql.Timestamp
+import java.util.*
+
+enum class Level {
+ Warning, Error, Note, None
+}
+
+enum class Kind {
+ pass, open, informational, notApplicable
+}
+
+@Entity
+@Table(name = "tool_run")
+class ToolRun {
+ @Id
+ @GeneratedValue(strategy = GenerationType.AUTO)
+ @Column(name = "id", nullable = false)
+ var id: UUID? = null
+
+
+ @OneToOne(orphanRemoval = true, cascade = [CascadeType.ALL])
+ @JoinColumn(name = "tool_id")
+ var tool: Tool? = null
+
+ @JdbcTypeCode(SqlTypes.TIMESTAMP)
+ @Column(name = "time_stamp")
+ var timeStamp: Timestamp? = null
+
+ @ManyToOne(fetch = FetchType.EAGER)
+ var repository: Repository? = null
+
+ @OneToMany(mappedBy = "toolRun", cascade = [CascadeType.ALL], orphanRemoval = true)
+ var toolResults: MutableSet<ToolResult> = mutableSetOf()
+
+ fun addToolResults(toolResults: Collection<ToolResult>) {
+ toolResults.forEach {
+ this.addToolResult(it)
+ }
+ }
+
+ fun addToolResult(toolResult: ToolResult) {
+ toolResults.add(toolResult)
+ toolResult.toolRun = this
+ }
+}
+
+@Entity
+@Table(name = "tool")
+class Tool {
+ @Id
+ @GeneratedValue(strategy = GenerationType.AUTO)
+ @Column(name = "id", nullable = false)
+ var id: UUID? = null
+
+
+ @Column(name = "name", length = 500)
+ var name: String? = null
+
+ @Column(name = "version")
+ var version: String? = null
+
+ @OneToMany(mappedBy = "tool", cascade = [CascadeType.ALL], orphanRemoval = true)
+ var rules: MutableSet<Rule> = mutableSetOf()
+
+ fun addRules(rules: Collection<Rule>) {
+ rules.forEach {
+ this.addRule(it)
+ }
+ }
+
+ fun addRule(rule: Rule) {
+ rules.add(rule)
+ rule.tool = this
+ }
+}
+
+
+@Entity
+@Table(name = "tool_result")
+class ToolResult {
+ @Id
+ @GeneratedValue(strategy = GenerationType.AUTO)
+ @Column(name = "id", nullable = false)
+ var id: UUID? = null
+
+ @ManyToOne
+ var rule: Rule? = null
+
+ @OneToMany(mappedBy = "toolResult", cascade = [CascadeType.ALL], orphanRemoval = true)
+ var resultLocations: MutableSet<ResultLocation> = mutableSetOf()
+
+ fun addLocations(locations: Collection<ResultLocation>) {
+ locations.forEach {
+ this.addLocation(it)
+ }
+ }
+
+ fun addLocation(location: ResultLocation) {
+ resultLocations.add(location)
+ location.toolResult = this
+ }
+
+ @Enumerated
+ @Column(name = "level")
+ var level: Level? = null
+
+ @Enumerated
+ @Column(name = "kind")
+ var kind: Kind? = null
+
+ @Column(name = "message", columnDefinition = "TEXT")
+ var message: String? = null
+
+ @ManyToOne
+ var toolRun: ToolRun? = null
+
+ override fun equals(other: Any?): Boolean {
+ if (this === other) return true
+ if (other == null || Hibernate.getClass(this) != Hibernate.getClass(other)) return false
+ other as ToolResult
+
+ return id != null && id == other.id
+ }
+
+ override fun hashCode(): Int = javaClass.hashCode()
+}
+
+@Entity
+@Table(name = "result_location")
+class ResultLocation {
+ @Id
+ @GeneratedValue(strategy = GenerationType.AUTO)
+ @Column(name = "id", nullable = false)
+ var id: UUID? = null
+
+ @ManyToOne
+ var toolResult: ToolResult? = null
+
+ @Column(name = "artifact_location", columnDefinition = "TEXT")
+ var artifactLocation: String? = null
+
+ @Column(name = "end_column")
+ var endColumn: Int? = null
+
+ @Column(name = "end_line")
+ var endLine: Int? = null
+
+ @Column(name = "start_column")
+ var startColumn: Int? = null
+
+ @Column(name = "start_line")
+ var startLine: Int? = null
+
+ override fun equals(other: Any?): Boolean {
+ if (this === other) return true
+ if (other == null || Hibernate.getClass(this) != Hibernate.getClass(other)) return false
+ other as ResultLocation
+
+ return id != null && id == other.id
+ }
+
+ override fun hashCode(): Int = javaClass.hashCode()
+}
+
+@Entity
+@Table(name = "rule")
+class Rule {
+ @Id
+ @GeneratedValue(strategy = GenerationType.AUTO)
+ @Column(name = "id", nullable = false)
+ var id: UUID? = null
+
+
+ @Column(name = "sarif_rule_id")
+ var sarifRuleId: String? = null
+
+ @Column(name = "short_description", columnDefinition = "TEXT")
+ var shortDescription: String? = null
+
+ @OneToMany(mappedBy = "rule", cascade = [CascadeType.ALL], orphanRemoval = true)
+ var toolResults: MutableSet<ToolResult> = mutableSetOf()
+
+ fun addResultLocations(toolResults: Collection<ToolResult>) {
+ toolResults.forEach {
+ this.addResultLocation(it)
+ }
+ }
+
+ fun addResultLocation(toolResult: ToolResult) {
+ toolResults.add(toolResult)
+ toolResult.rule = this
+ }
+
+ @ElementCollection
+ @CollectionTable(name = "rule_messages", joinColumns = [JoinColumn(name = "owner_id")])
+ @Column(name = "message", length = 1000)
+ var messages: MutableList<String> = mutableListOf()
+
+ @ManyToOne
+ var tool: Tool? = null
+
+ override fun equals(other: Any?): Boolean {
+ if (this === other) return true
+ if (other == null || Hibernate.getClass(this) != Hibernate.getClass(other)) return false
+ other as Rule
+
+ return id != null && id == other.id
+ }
+
+ override fun hashCode(): Int = javaClass.hashCode()
+}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunRepository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunRepository.kt
new file mode 100644
index 0000000000000000000000000000000000000000..4a907621bbee277cc305a2a2751e59a93c5a8dac
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunRepository.kt
@@ -0,0 +1,6 @@
+package de.fraunhofer.iem.dataprovider.toolResult
+
+import org.springframework.data.jpa.repository.JpaRepository
+import java.util.*
+
+interface ToolRunRepository : JpaRepository<ToolRun, UUID>
\ No newline at end of file
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
index 10edc1c6cfd14c290796fb23bd18e44afdc98647..2601e3563a0a2f2763ffab9c06935584370e73a5 100644
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -3,8 +3,16 @@ OPENCODE_GITLAB_URL=https://gitlab.opencode.de/
OPENCODE_GITLAB_TOKEN=${OPENCODE_TOKEN}
GITHUB_URL=https://gitlab.opencode.de/
GITHUB_TOKEN=${GITHUB_TOKEN}
-spring.r2dbc.url=${R2DBC_URL}
-
GIT_PROJECT_PATH=${GIT_PROJECT_PATH}
ODC_OUTPUT_PATH=${ODC_OUTPUT_PATH}
+spring.datasource.url=jdbc:postgresql://localhost:5432/dataprovider
+spring.datasource.username=sa
+spring.datasource.password=password
+spring.jpa.generate-ddl=true
+spring.jpa.show-sql=true
+spring.jpa.hibernate.ddl-auto=update
+spring.jpa.properties.hibernate.format_sql=true
+spring.jpa.properties.hibernate.enable_lazy_load_no_trans=true
+spring.jpa.properties.hibernate.bytecode.use_reflection_optimizer=false
+spring.jpa.open-in-view=false
#logging.level.root=DEBUG
\ No newline at end of file
diff --git a/src/main/resources/db/docker-compose.yml b/src/main/resources/db/docker-compose.yml
new file mode 100644
index 0000000000000000000000000000000000000000..977879021d81bc7f7e64ad0a58252fbcc9c5b2fa
--- /dev/null
+++ b/src/main/resources/db/docker-compose.yml
@@ -0,0 +1,11 @@
+version: '3.7'
+services:
+ db:
+ image: postgres:latest
+ restart: always
+ environment:
+ - POSTGRES_USER=sa
+ - POSTGRES_PASSWORD=password
+ - POSTGRES_DB=dataprovider
+ ports:
+ - 5432:5432
\ No newline at end of file