Skip to content
Snippets Groups Projects
Verified Commit cc4d95f7 authored by Jan-Niclas Strüwer's avatar Jan-Niclas Strüwer
Browse files

Integrated usage of dummy API. Careful hardcoded paths in code !

parent c76f775d
No related branches found
No related tags found
1 merge request!8Dev
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 2033 additions and 13385 deletions
build.gradle.kts
+ 4
− 1
View file @ cc4d95f7
......@@ -22,7 +22,6 @@ allOpen {
group = "de.fraunhofer.iem"
version = "0.0.2-SNAPSHOT"
java.sourceCompatibility = JavaVersion.VERSION_17
configurations {
compileOnly {
extendsFrom(configurations.annotationProcessor.get())
......@@ -47,6 +46,10 @@ dependencies {
implementation("org.eclipse.jgit:org.eclipse.jgit:6.6.0.202305301015-r")
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.5.1")
implementation("org.springframework.boot:spring-boot-starter-actuator")
implementation("io.ktor:ktor-client-core:2.3.3")
implementation("io.ktor:ktor-client-cio:2.3.3")
implementation("io.ktor:ktor-client-content-negotiation:2.3.3")
implementation("io.ktor:ktor-serialization-kotlinx-json:2.3.3")
developmentOnly("org.springframework.boot:spring-boot-devtools")
runtimeOnly("org.postgresql:postgresql")
testImplementation("org.springframework.boot:spring-boot-starter-test") {
......
settings.gradle.kts
+ 1
− 1
View file @ cc4d95f7
rootProject.name = "data-provider"
rootProject.name = "data-provider"
\ No newline at end of file
src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
+ 11
− 30
View file @ cc4d95f7
......@@ -12,8 +12,7 @@ import de.fraunhofer.iem.dataprovider.taskManager.tasks.dataQuery.GetGitlabProje
import de.fraunhofer.iem.dataprovider.taskManager.tasks.dataQuery.GetRepositoryDetailsTask
import de.fraunhofer.iem.dataprovider.taskManager.tasks.kpiCalculation.MetricsTask
import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.occmd.OccmdTask
import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.ort.OrtAdvisorTask
import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.ort.OrtAnalyzerTask
import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.ort.OrtApiTask
import de.fraunhofer.iem.dataprovider.tool.service.ToolService
import de.fraunhofer.iem.dataprovider.toolRun.service.ToolRunService
import jakarta.annotation.PreDestroy
......@@ -35,10 +34,10 @@ class TaskManager(
private val config: Config,
private val openCodeGitlabConfiguration: OpenCodeGitlabApiConfiguration,
private val toolRunService: ToolRunService,
private val toolService: ToolService,
private val repositoryService: RepositoryService,
private val kpiService: KPIService,
private val dependencyService: DependencyService,
private val toolService: ToolService
private val dependencyService: DependencyService
) {
// The used default dispatcher is ok for CPU-bound workloads. However,
......@@ -127,39 +126,21 @@ class TaskManager(
groupId,
toolRunService
)
val ortAnalyzerTask = OrtAnalyzerTask(
event.outputDirectory,
config.toolResultsTargetDirectory,
::addEvent,
event.repoId,
groupId
ioWorker.addTask(
OrtApiTask(
::addEvent,
event.repoId,
dependencyService,
toolService,
repositoryService
)
)
groupTaskManager.addTaskToGroup(groupId, ortAnalyzerTask.taskID)
groupTaskManager.addTaskToGroup(groupId, occmdTask.taskID)
worker.addTask(ortAnalyzerTask)
worker.addTask(occmdTask)
}
is OrtAnalyzerDoneEvent -> {
logger.info("Ort analyzer done event received")
val ortAdvisorTask = OrtAdvisorTask(
event.resultFileOutputDirectoryPath,
config.toolResultsTargetDirectory,
::addEvent,
event.repoId,
event.groupID,
dependencyService,
event.taskId,
toolService,
repositoryService
)
worker.addTask(ortAdvisorTask)
}
is GroupTaskDoneEvent -> {
logger.info("GroupTaskDoneEvent received")
groupTaskManager.taskInGroupFinished(event.groupId, event.taskId)
......
src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/ort/OrtAnalyzerTask.kt deleted 100644 → 0
+ 0
− 50
View file @ c76f775d
package de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.ort
import de.fraunhofer.iem.dataprovider.dependency.dto.DependencyCreateDto
import de.fraunhofer.iem.dataprovider.taskManager.events.Event
import de.fraunhofer.iem.dataprovider.taskManager.events.OrtAnalyzerDoneEvent
import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.ToolProcessTask
import org.springframework.core.io.ClassPathResource
import org.springframework.core.io.Resource
import java.io.File
import java.nio.file.Path
import java.nio.file.Paths
import java.util.*
class OrtAnalyzerTask(
repositoryDirectoryPath: String,
outputDirectoryPath: String,
override val responseChannel: suspend (event: Event) -> Unit,
override val repoId: UUID,
override val groupID: UUID? = null
) : ToolProcessTask<List<DependencyCreateDto>>() {
private val resource: Resource = ClassPathResource("scripts/ort/ort_analyzer.sh")
override val outputDirectory = Paths.get(outputDirectoryPath, "ort-analyzer", taskID.toString()).toString()
override val resultFileOutputPath: Path = Paths.get(outputDirectory, "analyzer-result.json")
private val copiedRepositoryPath = Paths.get(outputDirectory, "copiedRepo")
override val flags = arrayOf(resource.file.absolutePath, copiedRepositoryPath.toString(), outputDirectory)
override suspend fun parseProcessResults(resultPath: Path): List<DependencyCreateDto> {
return emptyList()
}
override suspend fun storeResultsInDb(result: List<DependencyCreateDto>) {
}
override suspend fun handleProcessReturn(p: Process) {
responseChannel(OrtAnalyzerDoneEvent(outputDirectory, repoId, taskID, groupID))
}
override fun cleanUp() {
File(copiedRepositoryPath.toUri()).deleteRecursively()
}
init {
val repoFile = File(repositoryDirectoryPath)
val copiedRepo = File(copiedRepositoryPath.toUri())
repoFile.copyRecursively(copiedRepo)
}
}
\ No newline at end of file
src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/ort/OrtAdvisorTask.kt → src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/ort/OrtApiTask.kt
+ 49
− 53
View file @ cc4d95f7
......@@ -8,49 +8,55 @@ import de.fraunhofer.iem.dataprovider.dependency.enumeration.VulnerabilityScorin
import de.fraunhofer.iem.dataprovider.dependency.service.DependencyService
import de.fraunhofer.iem.dataprovider.repository.service.RepositoryService
import de.fraunhofer.iem.dataprovider.taskManager.events.Event
import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.ToolProcessTask
import de.fraunhofer.iem.dataprovider.taskManager.tasks.Task
import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.ort.dto.OrtResultDto
import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.ort.json.AdvisorResultJson
import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.ort.json.OrtJson
import de.fraunhofer.iem.dataprovider.tool.dto.CreateToolDto
import de.fraunhofer.iem.dataprovider.tool.service.ToolService
import io.ktor.client.*
import io.ktor.client.call.*
import io.ktor.client.engine.cio.*
import io.ktor.client.plugins.contentnegotiation.*
import io.ktor.client.request.*
import io.ktor.client.statement.*
import io.ktor.serialization.kotlinx.json.*
import kotlinx.serialization.json.*
import org.springframework.core.io.ClassPathResource
import org.springframework.core.io.Resource
import java.io.IOException
import java.nio.file.Path
import java.nio.file.Paths
import java.util.*
class OrtAdvisorTask(
analyzerResultLocation: String,
outputDirectoryPath: String,
class OrtApiTask(
override val responseChannel: suspend (event: Event) -> Unit,
override val repoId: UUID,
override val groupID: UUID?,
val repoId: UUID,
private val dependencyService: DependencyService,
override val taskID: UUID,
private val toolService: ToolService,
private val repositoryService: RepositoryService
) : ToolProcessTask<List<DependencyCreateDto>>() {
private val resource: Resource = ClassPathResource("scripts/ort/ort_advisor.sh")
override val outputDirectory = Paths.get(outputDirectoryPath, "ort-advisor", taskID.toString()).toString()
override val resultFileOutputPath: Path = Paths.get(outputDirectory, "advisor-result.json")
override val flags = arrayOf(resource.file.absolutePath, analyzerResultLocation, outputDirectory)
) : Task() {
val json = Json { ignoreUnknownKeys = true }
override suspend fun execute() {
val client = HttpClient(CIO) {
install(ContentNegotiation) {
json(
json
)
}
}
override suspend fun parseProcessResults(resultPath: Path): List<DependencyCreateDto> {
val response: HttpResponse = client.get("http://localhost:3000/ort")
val ortJson = response.body<OrtJson>()
println(response.status)
println(ortJson)
client.close()
val ortDto = ortJsonToDto(ortJson)
val dependencies = mutableListOf<DependencyCreateDto>()
val ortJson = getOrtResultsFromFile(resultPath)
ortJson.packages.forEach { p ->
ortDto.packages.forEach { p ->
if (p.nameAndVersion != null) {
dependencies.add(DependencyCreateDto(name = p.nameAndVersion))
}
}
ortJson.advisorResults.forEach { advisorResult ->
ortDto.advisorResults.forEach { advisorResult ->
var dependency = dependencies.find { it.name == advisorResult.identifier }
......@@ -78,38 +84,11 @@ class OrtAdvisorTask(
}
}
return dependencies
}
// TODO: Requires a database purge / association with the tool run
override suspend fun storeResultsInDb(result: List<DependencyCreateDto>) {
val dependencies = mutableListOf<DependencyEntity>()
result.forEach { dependency ->
val dependencyEntity = dependency.toDbObject()
dependencyService.save(dependencyEntity)
dependencies.add(dependencyEntity)
}
val tool = toolService.findOrCreateTool(CreateToolDto("ORT", "ORT", "docker-snapshot"))
val repo = repositoryService.findRepoByID(repoId)
if (repo != null) {
repo.toolEntities.add(tool)
repo.dependencyEntities.addAll(dependencies)
repositoryService.save(repo)
}
storeResultsInDb(dependencies)
}
}
fun getOrtResultsFromFile(resultPath: Path): OrtResultDto {
val resFile = resultPath.toFile()
if (resFile.exists()) {
private fun ortJsonToDto(rawJson: OrtJson): OrtResultDto {
val results = mutableListOf<AdvisorResultJson>()
val resString = resFile.readText()
val json = Json { ignoreUnknownKeys = true }
val rawJson = json.decodeFromString<OrtJson>(resString)
rawJson.advisor?.results?.advisorResults?.forEach { (key, value) ->
if (value is JsonArray) {
value.forEach { res ->
......@@ -132,5 +111,22 @@ fun getOrtResultsFromFile(resultPath: Path): OrtResultDto {
return OrtResultDto(advisorResults = results, packages = packages)
}
throw IOException("File not found / Path is no file.")
// TODO: Requires a database purge / association with the tool run
private suspend fun storeResultsInDb(result: List<DependencyCreateDto>) {
val dependencies = mutableListOf<DependencyEntity>()
result.forEach { dependency ->
val dependencyEntity = dependency.toDbObject()
dependencyService.save(dependencyEntity)
dependencies.add(dependencyEntity)
}
val tool = toolService.findOrCreateTool(CreateToolDto("ORT", "ORT", "docker-snapshot"))
val repo = repositoryService.findRepoByID(repoId)
if (repo != null) {
repo.toolEntities.add(tool)
repo.dependencyEntities.addAll(dependencies)
repositoryService.save(repo)
}
}
}
tools/db/api.json
+ 1968
− 13250
View file @ cc4d95f7
This diff is collapsed.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Consent

On this website, we use the web analytics service Matomo to analyze and review the use of our website. Through the collected statistics, we can improve our offerings and make them more appealing for you. Here, you can decide whether to allow us to process your data and set corresponding cookies for these purposes, in addition to technically necessary cookies. Further information on data protection—especially regarding "cookies" and "Matomo"—can be found in our privacy policy. You can withdraw your consent at any time.