Merge pull request #44 from secure-software-engineering/hotfix/reduce-jar-size

Reduce JAR size and remove training JARs

Author: oshando

swan-pipeline/pom.xml

@@ -258,28 +258,55 @@
                     </execution>
                 </executions>
             </plugin>
-
-            <!-- This creates a runnable JAR that can be used from the command line. -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-assembly-plugin</artifactId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.4</version>
                 <executions>
                     <execution>
                         <phase>package</phase>
                         <goals>
-                            <goal>single</goal>
+                            <goal>shade</goal>
                         </goals>
                         <configuration>
-                            <archive>
-                                <manifest>
-                                    <mainClass>
-                                        de.fraunhofer.iem.swan.Main
-                                    </mainClass>
-                                </manifest>
-                            </archive>
-                            <descriptorRefs>
-                                <descriptorRef>jar-with-dependencies</descriptorRef>
-                            </descriptorRefs>
+                            <transformers>
+                                <!-- adding Main-Class to manifest file -->
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>de.fraunhofer.iem.swan.Main</mainClass>
+                                </transformer>
+                            </transformers>
+                            <minimizeJar>true</minimizeJar>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                                <filter>
+                                    <artifact>edu.stanford.nlp:stanford-corenlp:models-english</artifact>
+                                    <excludes>
+                                        <exclude>edu/stanford/nlp/models/srparser/**</exclude>
+                                        <exclude>edu/stanford/nlp/models/ner/**</exclude>
+                                        <exclude>edu/stanford/nlp/models/sentiment/**</exclude>
+                                    </excludes>
+                                </filter>
+                                <filter>
+                                    <artifact>org.bytedeco</artifact>
+                                    <excludes>
+                                        <exclude>**</exclude>
+                                    </excludes>
+                                </filter>
+                                <!--filter>
+                                    <artifact>org.nd4j</artifact>
+                                    <excludes>
+                                        <exclude>**</exclude>
+                                    </excludes>
+                                </filter-->
+                            </filters>
                         </configuration>
                     </execution>
                 </executions>

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/features/doc/nlp/CoreNLPExecutor.java

@@ -28,7 +28,7 @@ public class CoreNLPExecutor {
     public CoreNLPExecutor() {
         logger.info("Initializing CoreNLP pipeline");
         properties = new Properties();
-        properties.setProperty("annotators", "tokenize, ssplit, pos, lemma, ner, parse");
+        properties.setProperty("annotators", "tokenize, ssplit, pos, lemma, parse");
         pipeline = new StanfordCoreNLP(properties);
     }
 

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/io/DependencyManager.java

@@ -1,6 +1,8 @@
 package de.fraunhofer.iem.swan.io;
 
 import dev.jeka.core.api.depmanagement.*;
+import dev.jeka.core.api.depmanagement.resolution.JkDependencyResolver;
+import dev.jeka.core.api.depmanagement.resolution.JkResolvedDependencyNode;
 import edu.stanford.nlp.util.StringUtils;
 
 import java.nio.file.Path;
@@ -29,25 +31,23 @@ public class DependencyManager {
     public List<Path> getSourceJar(String module) {
 
         JkDependencySet sources = JkDependencySet.of()
-                .and(module)
-                .withDefaultScopes(JkScope.SOURCES);
+                .and(module);
 
-        return resolver.resolve(sources, JkScope.SOURCES).getFiles().getEntries();
+        return resolver.resolve(sources).getFiles().getEntries();
     }
 
     public String getDependencies(String moduleDescription) {
         JkDependencySet deps = JkDependencySet.of()
-                .and(moduleDescription)
-                .withDefaultScopes(JkScope.COMPILE);
+                .and(moduleDescription);
 
          resolver = JkDependencyResolver.of().addRepos(JkRepo.ofMavenCentral());
 
         //Get list of dependencies for module
-        List<JkDependencyNode> nodes = resolver.resolve(deps, JkScope.COMPILE).getDependencyTree().toFlattenList();
+        List<JkResolvedDependencyNode> nodes = resolver.resolve(deps).getDependencyTree().toFlattenList();
 
         Set<String> dependencies = new HashSet<>();
 
-        for (JkDependencyNode node : nodes)
+        for (JkResolvedDependencyNode node : nodes)
             dependencies.add(node.getModuleInfo().getModuleId().toString()
                     + ":" + node.getModuleInfo().getResolvedVersion().toString());
 

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/model/toolkit/MLPlan.java

@@ -5,7 +5,7 @@
 import ai.libs.jaicore.ml.core.dataset.serialization.ArffDatasetAdapter;
 import ai.libs.jaicore.ml.core.filter.SplitterUtil;
 import ai.libs.jaicore.ml.weka.classification.learner.IWekaClassifier;
-import ai.libs.mlplan.multiclass.wekamlplan.MLPlanWekaBuilder;
+import ai.libs.mlplan.weka.MLPlanWekaBuilder;
 import de.fraunhofer.iem.swan.model.MonteCarloValidator;
 import de.fraunhofer.iem.swan.util.Util;
 import org.api4.java.ai.ml.core.dataset.schema.attribute.IAttribute;
@@ -50,6 +50,7 @@ public MLPlan() {
     public HashMap<String, ArrayList<Double>> evaluateDataset(Instances instances1) {
 
         String arffFilePath = Util.exportInstancesToArff(instances1);
+        ArffDatasetAdapter arffDatasetAdapter = new ArffDatasetAdapter();
 
         String mClass = Util.getClassName(instances1);
 
@@ -58,7 +59,7 @@ public HashMap<String, ArrayList<Double>> evaluateDataset(Instances instances1)
         //Initialize dataset using ARFF file path
         ILabeledDataset<?> dataset = null;
         try {
-            dataset = ArffDatasetAdapter.readDataset(new File(arffFilePath));
+            dataset = arffDatasetAdapter.readDataset(new File(arffFilePath));
         } catch (DatasetDeserializationFailedException e) {
             e.printStackTrace();
         }
@@ -95,14 +96,14 @@ public HashMap<String, ArrayList<Double>> evaluateDataset(Instances instances1)
                 //optimizedClassifier.fit(split.get(0));
 
                 String trainPath = "swan/swan_core/swan-out/mlplan/train-methods-dataset.arff";
-                ArffDatasetAdapter.serializeDataset(new File(trainPath), split.get(0));
+                arffDatasetAdapter.serializeDataset(new File(trainPath), split.get(0));
                 ArffLoader trainLoader = new ArffLoader();
                 trainLoader.setFile(new File(trainPath));
                 Instances trainInstances = trainLoader.getDataSet();
                 trainInstances.setClassIndex(trainInstances.numAttributes() - 1);
 
                 String testPath = "swan/swan_core/swan-out/mlplan/test-methods-dataset.arff";
-                ArffDatasetAdapter.serializeDataset(new File(testPath), split.get(1));
+                arffDatasetAdapter.serializeDataset(new File(testPath), split.get(1));
                 ArffLoader testLoader = new ArffLoader();
                 testLoader.setFile(new File(testPath));
                 Instances testInstances = testLoader.getDataSet();
@@ -148,6 +149,7 @@ public void evaluateDataset(Instances instances, int k) {
 
         //arffFilePath = "swan/swan_core/src/main/resources/waveform.arff";
         String arffFilePath = Util.exportInstancesToArff(instances);
+        ArffDatasetAdapter arffDatasetAdapter = new ArffDatasetAdapter();
 
         String mClass = Util.getClassName(instances);
 
@@ -156,7 +158,7 @@ public void evaluateDataset(Instances instances, int k) {
         //Initialize dataset using ARFF file path
         ILabeledDataset<?> dataset = null;
         try {
-            dataset = ArffDatasetAdapter.readDataset(new File(arffFilePath));
+            dataset = arffDatasetAdapter.readDataset(new File(arffFilePath));
         } catch (DatasetDeserializationFailedException e) {
             e.printStackTrace();
         }
@@ -177,7 +179,7 @@ public void evaluateDataset(Instances instances, int k) {
 
                     //   System.out.println(attribute.getName());
                 }
-                ArffDatasetAdapter.serializeDataset(new File("swan/swan_core/swan-out/mlplan/methods-dataset.arff"), split.get(1));
+                arffDatasetAdapter.serializeDataset(new File("swan/swan_core/swan-out/mlplan/methods-dataset.arff"), split.get(1));
 
 
                 for (int x = 0; x < split.get(1).size(); x++) {

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/training/TrainingSetUpdater.java

@@ -1,10 +1,9 @@
 package de.fraunhofer.iem.swan.training;
 
-import de.fraunhofer.iem.swan.features.code.soot.SourceFileLoader;
-import de.fraunhofer.iem.swan.io.dataset.SrmList;
-import de.fraunhofer.iem.swan.util.Util;
 import de.fraunhofer.iem.swan.data.Category;
 import de.fraunhofer.iem.swan.data.Method;
+import de.fraunhofer.iem.swan.io.dataset.SrmList;
+import de.fraunhofer.iem.swan.util.Util;
 import org.apache.commons.io.FileUtils;
 
 import java.io.File;
@@ -115,7 +114,7 @@ public static Set<Method> extractMethodData(HashSet<String> classes, HashSet<Str
 
         System.out.println("Classes: Total/" + classes.size());
 
-        String classpath = Util.buildCP(TRAINING_JARS);
+        String classpath = "Util.buildCP(TRAINING_JARS)";
 
         HashMap<String, String> methodSet = new HashMap<>();
 
@@ -124,15 +123,15 @@ public static Set<Method> extractMethodData(HashSet<String> classes, HashSet<Str
             methodSet.put(t.substring(0, t.indexOf("|")), t.substring(t.indexOf("|") + 1));
         }
 
-        SourceFileLoader sourceFileLoader = new SourceFileLoader(classpath);
-        sourceFileLoader.loadMethodsFromTestLib();
+      //  SourceFileLoader sourceFileLoader = new SourceFileLoader(classpath);
+        //sourceFileLoader.loadMethodsFromTestLib();
 
-        Util.createSubclassAnnotations(sourceFileLoader.getMethods(), classpath);
+       // Util.createSubclassAnnotations(sourceFileLoader.getMethods(), classpath);
 
         HashSet<Method> trainingMethods = new HashSet<>();
 
 
-        for (Method m : sourceFileLoader.getMethods()) {
+        for (Method m :trainingMethods) {
 
             String sig = m.getClassName() + "." + m.getName();
 

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/util/Util.java

@@ -130,7 +130,7 @@ public static String getClassname(String path) {
         className = className.substring(0, className.length() - ".class".length());
         if (className.contains("$"))
             className = className.substring(0, className.indexOf("$") - 1);
-        System.out.println(className);
+
         return className;
     }