clouditor · co-de-pot · Jul 17, 2023 · Jul 17, 2023 · Jul 17, 2023 · Jul 27, 2023
diff --git a/README.md b/README.md
@@ -19,7 +19,22 @@ Note that this project primarily serves as a research prototype, so please do no
 ## Setup and Build
 
 ### Python Setup
-Please follow the instruction at https://github.com/ninia/jep/wiki/Getting-Started#installing-jep to install jep. Also, make sure you are using Python 3.10.
+
+You need to install [jep](https://github.com/ninia/jep/). This can either be system-wide or in a virtual environment. Your jep version has to match the version used by the CPG (see [version catalog](./gradle/libs.versions.toml)).
+
+Currently, only Python 3.{9,10,11,12,13} is supported.
+
+##### System Wide
+
+Follow the instructions at https://github.com/ninia/jep/wiki/Getting-Started#installing-jep.
+
+##### Virtual Env
+
+- `python3 -m venv ~/.virtualenvs/cpg`
+- `source ~/.virtualenvs/cpg/bin/activate`
+- `pip3 install jep`
+
+Through the `JepSingleton`, the CPG library will look for well known paths on Linux and OS X. `JepSingleton` will prefer a virtualenv with the name `cpg`, this can be adjusted with the environment variable `CPG_PYTHON_VIRTUALENV`.
 
 ### Build
 Note that we recommend using Java 17. First, the graph classes need to be built from the Ontology definitions by calling `./build-ontology.sh`. We aim to automate this process using a Gradle plugin in the future. Then, build using `./gradlew installDist`.

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -1,7 +1,7 @@
 plugins {
     // Apply the Kotlin JVM plugin to add support for Kotlin.
-    id("org.jetbrains.kotlin.jvm") version "1.7.20-Beta" apply false
-    kotlin("plugin.serialization") version "1.7.20-Beta" apply false
+    id("org.jetbrains.kotlin.jvm") version "1.9.20" apply false
+    kotlin("plugin.serialization") version "1.9.20" apply false
     id("com.diffplug.spotless") version "5.12.1"
 }
 
@@ -15,7 +15,7 @@ subprojects {
         mavenCentral()
 
         ivy {
-            setUrl("https://download.eclipse.org/tools/cdt/releases/10.2/cdt-10.2.0/plugins")
+            setUrl("https://download.eclipse.org/tools/cdt/releases/11.3/cdt-11.3.1/plugins")
             metadataSources {
                 artifact()
             }

diff --git a/cloudpg/build.gradle.kts b/cloudpg/build.gradle.kts
@@ -1,5 +1,3 @@
-import org.jetbrains.kotlin.com.intellij.openapi.vfs.StandardFileSystems.jar
-
 /*
  * This file was generated by the Gradle 'init' task.
  *
@@ -10,7 +8,7 @@ plugins {
     application
     idea
     `maven-publish`
-    id("org.jetbrains.kotlinx.benchmark") version "0.4.4"
+    id("org.jetbrains.kotlinx.benchmark") version "0.4.10"
     // use this plugin to set all classes open which is required for kotlinx benchmark: id("org.jetbrains.kotlin.plugin.allopen") version "1.7.20-Beta"
 }
 
@@ -31,13 +29,13 @@ java {
     sourceSets["main"].java {
         srcDir("${generatedDir}/main/java")
     }
-    sourceCompatibility = JavaVersion.VERSION_11
-    targetCompatibility = JavaVersion.VERSION_11
+    sourceCompatibility = JavaVersion.VERSION_17
+    targetCompatibility = JavaVersion.VERSION_17
 }
 
 tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile>().configureEach {
     kotlinOptions {
-        jvmTarget = "11"
+        jvmTarget = "17"
     }
 }
 
@@ -50,6 +48,7 @@ publishing {
 }
 
 repositories {
+    mavenLocal()
     mavenCentral()
 
     maven { setUrl("https://jitpack.io") }
@@ -59,32 +58,40 @@ repositories {
     }
 
     ivy {
-        setUrl("https://download.eclipse.org/tools/cdt/releases/10.3/cdt-10.3.2/plugins")
+        setUrl("https://download.eclipse.org/tools/cdt/releases/")
         metadataSources {
             artifact()
         }
+
         patternLayout {
-            artifact("/[organisation].[module]_[revision].[ext]")
+            artifact("[organisation].[module]_[revision].[ext]")
         }
     }
 }
 
 dependencies {
     implementation("org.junit.jupiter:junit-jupiter:5.7.0")
-    val version = "4.6.0"
+    //implementation("com.github.Fraunhofer-AISEC:cpg:4e7c0b862e")
+    //implementation("de.fraunhofer.aisec:cpg-language-cxx:8.3.0")
+    val version = "9.0.2"
+
+    implementation("de.fraunhofer.aisec:cpg-language-typescript:9.0.2")
 
     implementation("de.fraunhofer.aisec:cpg-core:$version")
     implementation("de.fraunhofer.aisec:cpg-analysis:$version")
     implementation("de.fraunhofer.aisec:cpg-language-go:$version")
     implementation("de.fraunhofer.aisec:cpg-language-python:$version")
+    implementation("de.fraunhofer.aisec:cpg-language-typescript:$version")
+    implementation("de.fraunhofer.aisec:cpg-language-java:$version")
+    implementation("de.fraunhofer.aisec:cpg-language-cxx:$version")
 
-    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.+")
-    implementation ("org.xmlunit:xmlunit-core:2.9.0")
+    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.17.2")
+    implementation("org.xmlunit:xmlunit-core:2.9.0")
     implementation("org.xmlunit:xmlunit-matchers:2.9.0")
 
-    api("org.neo4j", "neo4j-ogm-core", "3.2.31")
-    api("org.neo4j", "neo4j-ogm", "3.2.21")
-    api("org.neo4j", "neo4j-ogm-bolt-driver", "3.2.21")
+    api("org.neo4j", "neo4j-ogm-core", "4.0.10")
+    api("org.neo4j", "neo4j-ogm", "4.0.10")
+    api("org.neo4j", "neo4j-ogm-bolt-driver", "4.0.10")
 
     implementation(platform("org.jetbrains.kotlin:kotlin-bom"))
 
@@ -118,7 +125,7 @@ dependencies {
 
 application {
     // Define the main class for the application.
-    mainClassName = "io.clouditor.graph.AppKt"
+    mainClass = "io.clouditor.graph.AppKt"
 }
 
 tasks.named("compileJava") {
@@ -159,7 +166,7 @@ kotlin {
     sourceSets {
         test {
             dependencies {
-                implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.4.4")
+                implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.4.10")
             }
         }
     }

diff --git a/cloudpg/src/main/java/io/clouditor/graph/App.kt b/cloudpg/src/main/java/io/clouditor/graph/App.kt
@@ -4,24 +4,23 @@
 package io.clouditor.graph
 
 import de.fraunhofer.aisec.cpg.*
-import de.fraunhofer.aisec.cpg.frontends.golang.GoLanguageFrontend
-import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguageFrontend
-import de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguageFrontend
+import de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage
+import de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage
+import de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage
+import de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage
+import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage
+import de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguage
+import de.fraunhofer.aisec.cpg.graph.Component
 import de.fraunhofer.aisec.cpg.graph.Node
+import de.fraunhofer.aisec.cpg.graph.allChildren
 import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
-import de.fraunhofer.aisec.cpg.graph.graph
 import de.fraunhofer.aisec.cpg.helpers.Benchmark
-import io.clouditor.graph.frontends.ruby.RubyLanguageFrontend
 import io.clouditor.graph.nodes.Builder
 import io.clouditor.graph.passes.*
 import io.clouditor.graph.passes.golang.*
 import io.clouditor.graph.passes.java.JaxRsClientPass
-import io.clouditor.graph.passes.java.JaxRsPass
-import io.clouditor.graph.passes.java.SpringBootPass
 import io.clouditor.graph.passes.js.FetchPass
-import io.clouditor.graph.passes.js.JSHttpPass
 import io.clouditor.graph.passes.python.*
-import io.clouditor.graph.passes.ruby.WebBrickPass
 import io.clouditor.graph.testing.LocalTestingPass
 import java.nio.file.Path
 import java.util.concurrent.Callable
@@ -35,12 +34,6 @@ import picocli.CommandLine
     mixinStandardHelpOptions = true,
     description = ["Builds the Cloud Property Graph and persists it into a graph database."]
 )
-@OptIn(
-    ExperimentalTypeScript::class,
-    ExperimentalPython::class,
-    ExperimentalGolang::class,
-    ExperimentalGraph::class
-)
 object App : Callable<Int> {
     @CommandLine.Option(
         names = ["-k8s-n", "--kubernetes-namespaces"],
@@ -87,18 +80,24 @@ object App : Callable<Int> {
                 .build()
 
         val sessionFactory =
-            SessionFactory(configuration, "de.fraunhofer.aisec.cpg.graph", "io.clouditor.graph")
+            SessionFactory(
+                configuration,
+                "de.fraunhofer.aisec.cpg.graph",
+                "io.clouditor.graph",
+                "de.fraunhofer.aisec.cpg.frontends"
+            )
         val session = sessionFactory.openSession()
 
         val result = doTranslate()
 
         val nodes = mutableListOf<Node>()
-        nodes.addAll(result.graph.nodes)
-        nodes.addAll(result.translationUnits)
+        val translationUnits =
+            result.components.stream().flatMap { it.translationUnits.stream() }.toList()
+        nodes.addAll(result.allChildren())
         nodes.addAll(result.images)
         nodes.addAll(result.builders)
         nodes.addAll(result.computes)
-        nodes.addAll(result.translationUnits)
+        nodes.addAll(translationUnits)
         nodes.addAll(result.additionalNodes)
 
         session.beginTransaction().use { transaction ->
@@ -122,68 +121,53 @@ object App : Callable<Int> {
         val builder =
             TranslationConfiguration.builder()
                 .topLevel(rootPath.toFile())
-                .sourceLocations(paths.map { rootPath.resolve(it).toFile() })
-                .defaultPasses()
-                .defaultLanguages()
-                .registerLanguage(
-                    RubyLanguageFrontend::class.java,
-                    RubyLanguageFrontend.RUBY_EXTENSIONS
-                )
-                .registerLanguage(
-                    TypeScriptLanguageFrontend::class.java,
-                    TypeScriptLanguageFrontend.TYPESCRIPT_EXTENSIONS +
-                        TypeScriptLanguageFrontend.JAVASCRIPT_EXTENSIONS
-                )
-                .registerLanguage(
-                    PythonLanguageFrontend::class.java,
-                    PythonLanguageFrontend.PY_EXTENSIONS
-                )
-                .registerLanguage(
-                    GoLanguageFrontend::class.java,
-                    GoLanguageFrontend.GOLANG_EXTENSIONS
-                )
+                .sourceLocations(paths.map { rootPath.resolve(it).toFile().normalize() })
+                .registerLanguage(JavaLanguage())
+                .registerLanguage(CPPLanguage())
+                .registerLanguage(CLanguage())
+                .registerLanguage(TypeScriptLanguage())
+                .registerLanguage(PythonLanguage())
+                .registerLanguage(GoLanguage())
                 .debugParser(true)
-                .registerPass(GitHubWorkflowPass())
-                .registerPass(SpringBootPass())
-                .registerPass(JaxRsPass())
-                .registerPass(GolangHttpPass())
-                .registerPass(GinGonicPass())
-                .registerPass(WebBrickPass())
-                .registerPass(JSHttpPass())
-                .registerPass(FlaskPass())
+                .defaultPasses()
+                .registerPass(GitHubWorkflowPass::class)
+                // .registerPass(SpringBootPass::class)
+                // .registerPass(JaxRsPass::class)
+                .registerPass(GolangHttpPass::class)
+                .registerPass(GinGonicPass::class)
+                // .registerPass(WebBrickPass::class)
+                // .registerPass(JSHttpPass::class)
+                .registerPass(FlaskPass::class)
                 .apply {
                     if (localMode) {
                         // register the localTestingPass after the HTTP Passes since it needs HTTP
                         // request handlers
-                        registerPass(LocalTestingPass())
-                        registerPass(GolangHttpRequestPass())
+                        registerPass(LocalTestingPass::class)
+                        registerPass(GolangHttpRequestPass::class)
                     } else {
-                        registerPass(AzurePass())
-                        registerPass(AzureClientSDKPass())
-                        registerPass(KubernetesPass())
-                        registerPass(IngressInvocationPass())
+                        registerPass(AzurePass::class)
+                        registerPass(AzureClientSDKPass::class)
+                        registerPass(KubernetesPass::class)
+                        registerPass(IngressInvocationPass::class)
                     }
                 }
-                .registerPass(CryptographyPass())
-                .registerPass(GoCryptoPass())
-                .registerPass(JaxRsClientPass())
-                .registerPass(FetchPass())
-                .registerPass(RequestsPass())
-                .registerPass(PythonLogPass())
-                .registerPass(GolangLogPass())
-                .registerPass(GormDatabasePass())
-                .registerPass(PyMongoPass())
-                .registerPass(Psycopg2Pass())
+                .registerPass(CryptographyPass::class)
+                .registerPass(GoCryptoPass::class)
+                .registerPass(JaxRsClientPass::class)
+                .registerPass(FetchPass::class)
+                .registerPass(RequestsPass::class)
+                .registerPass(PythonLogPass::class)
+                .registerPass(GolangLogPass::class)
+                .registerPass(GormDatabasePass::class)
+                .registerPass(PyMongoPass::class)
+                .registerPass(Psycopg2Pass::class)
                 .processAnnotations(true)
 
         if (labelsEnabled) {
-            val edgesCache: BidirectionalEdgesCachePass = BidirectionalEdgesCachePass()
-            val labelPass: LabelExtractionPass = LabelExtractionPass()
-            labelPass.edgesCachePass = edgesCache
             builder
-                .registerPass(DFGExtensionPass())
-                .registerPass(edgesCache)
-                .registerPass(labelPass)
+                .registerPass(DFGExtensionPass::class)
+                .registerPass(BidirectionalEdgesCachePass::class)
+                .registerPass(LabelExtractionPass::class)
                 .matchCommentsToNodes(true)
         }
 
@@ -196,7 +180,6 @@ object App : Callable<Int> {
     }
 }
 
-@OptIn(ExperimentalPython::class, ExperimentalTypeScript::class, ExperimentalGolang::class)
 fun main(args: Array<String>): Unit = exitProcess(CommandLine(App).execute(*args))
 
 val TranslationResult.images: MutableList<Image>
@@ -212,13 +195,14 @@ val TranslationResult.computes: MutableList<Compute>
         this.scratch.computeIfAbsent("computes") { mutableListOf<Compute>() } as
             MutableList<Compute>
 
-val TranslationResult.additionalNodes: MutableList<Node>
-    get() =
-        this.scratch.computeIfAbsent("additionalNodes") { mutableListOf<Node>() } as
-            MutableList<Node>
-
 fun TranslationResult.findApplicationByTU(tu: TranslationUnitDeclaration): Application? {
-    return this.additionalNodes.filterIsInstance(Application::class.java).firstOrNull {
+    return this.additionalNodes.filterIsInstance<Application>().firstOrNull() {
+        it.translationUnits.contains(tu)
+    }
+}
+
+fun TranslationResult.findComponentByTU(tu: TranslationUnitDeclaration): Component? {
+    return this.additionalNodes.filterIsInstance<Component>().firstOrNull() {
         it.translationUnits.contains(tu)
     }
 }