Add Jackson annotations and refactor SRM file loader

Author: oshando

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/SwanPipeline.java

@@ -1,5 +1,6 @@
 package de.fraunhofer.iem.swan;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import de.fraunhofer.iem.swan.cli.SwanOptions;
 import de.fraunhofer.iem.swan.data.Category;
 import de.fraunhofer.iem.swan.data.Method;
@@ -54,86 +55,9 @@ public void run() throws IOException, InterruptedException {
 
         long startAnalysisTime = System.currentTimeMillis();
 
-        // Cache the list of classes and the CP.
-        Set<String> testClasses = Util.getAllClassesFromDirectory(options.getTestData());
-        String testCp = Util.buildCP(options.getTestData());
-
-        logger.info("Loading train data from {}", options.getTrainData());
-        String trainingCp = Util.buildCP(options.getTrainData());
-
-        // Cache the methods from the training set.
-        parser = new Parser(trainingCp);
-        parser.loadTrainingSet(Collections.singleton(options.getDatasetJson()));
-        logger.info("{} training methods, distribution={}",
-                parser.methods().size(), Util.countCategories(parser.methods(), false));
-
-        //Remove methods that do not have method doc comments
-        parser.removeUndocumentedMethods();
-        logger.info("Remove undocumented training methods. Remaining {}, distribution={}",
-                parser.methods().size(), Util.countCategories(parser.methods(), true));
-
-        // Cache the methods from the testing set.
-        logger.info("Loading test data from {}", options.getTestData());
-        loader = new Loader(testCp);
-        loader.loadTestSet(testClasses, parser.methods());
-
-        // Cache the features.
-        logger.info("Loading feature instances");
-        featureHandler = new FeatureHandler(trainingCp + System.getProperty("path.separator") + testCp);
-        featureHandler.initializeFeatures();
-
-        Set<Method> methods = new HashSet<>();
-
-        for (Method method : parser.methods()) {
-            List<String> words = Arrays.asList(StringUtils.split(method.getJavadoc().getMethodComment(), " "));
-            if (method.getJavadoc().getMethodComment().length() > 0 || words.size() > 1
-            ) {
-                //  if (method.getDiscovery().contentEquals("manual")) {
-                methods.add(method);
-                //if(method.getCategoriesTrained().contains(Category.AUTHENTICATION_NEUTRAL))
-                //   System.out.println(method.getJavaSignature());
-            }
-        }
-
-        //Populate SWAN feature attributes
-        docFeatureHandler = null;
-        InstancesHandler.FeatureSet feature = InstancesHandler.FeatureSet.getValue(Integer.parseInt(options.getFeatureSet()));
-
-        switch (feature) {
-            case SWANDOC_MANUAL:
-            case SWAN_SWANDOC_MANUAL:
-
-                docFeatureHandler = new DocFeatureHandler(methods);
-                docFeatureHandler.initialiseManualFeatureSet();
-                docFeatureHandler.evaluateManualFeatureData();
-                break;
-            case SWANDOC_WORD_EMBEDDING:
-            case SWAN_SWANDOC_WORD_EMBEDDING:
-
-                docFeatureHandler = new DocFeatureHandler(methods);
-                docFeatureHandler.initialiseAutomaticFeatureSet();
-                docFeatureHandler.evaluateAutomaticFeatureData();
-                break;
-        }
-
-        // Prepare classifier.
-        logger.info("Preparing classifier");
-        writer = new Writer(loader.methods());
-        learner = new Learner(writer);
-
-        Learner.Mode learnerMode = Learner.Mode.valueOf(options.getLearningMode().toUpperCase());
-
-        /*
-            FIRST PHASE - binary classification for each of the categories.
-            (1) Classify: source, sink, sanitizer,
-            auth-no-change, auth-unsafe-state, auth-safe-state
-            (2) Classify: relevant
-         */
-
-        //Store predictions for each classifier and iteration.
-        predictions = new HashMap<>();
-        for (int x = 0; x < 10; x++)
-            predictions.put(Integer.toString(x), new HashSet<>());
+        // Load methods in training dataset
+        SrmList dataset = SrmListUtils.importFile(options.getDatasetJson(), options.getTrainDataDir());
+        logger.info("Loaded {} training methods, distribution={}", dataset.getMethods().size(), Util.countCategories(dataset.getMethods()));
 
         //Load methods from the test set
         logger.info("Loading test JARs in {}", options.getTestDataDir());

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/data/Category.java

@@ -1,5 +1,7 @@
 package de.fraunhofer.iem.swan.data;
 
+import com.fasterxml.jackson.annotation.JsonValue;
+
 /**
  * Categories for the learner.
  *
@@ -38,6 +40,11 @@ public boolean isCwe() {
         return cwe;
     }
 
+    @JsonValue
+    public String getId() {
+        return id;
+    }
+
     @Override
     public String toString() {
         return id;
@@ -52,8 +59,8 @@ public static Category getCategoryForCWE(String cweName) {
 
     public static Category fromText(String text) {
 
-        for(Category cat: Category.values()){
-            if(cat.name().equalsIgnoreCase(text)){
+        for (Category cat : Category.values()) {
+            if (cat.name().equalsIgnoreCase(text)) {
                 return cat;
             }
         }

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/data/Javadoc.java

@@ -1,14 +1,18 @@
 
 package de.fraunhofer.iem.swan.data;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 /**
  * Stores the Javadoc comment for a method and the class it belongs to.
  *
  * @author Oshando Johnson on 23.07.20
  */
 public class Javadoc {
 
+    @JsonProperty("method")
     private String methodComment;
+    @JsonProperty("class")
     private String classComment;
 
     public Javadoc() {

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/data/RelevantPart.java

@@ -4,38 +4,41 @@
  * POJO for the Relevant parts of a method
  *
  * @author Goran Piskachev
- *
  */
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 import java.util.ArrayList;
 import java.util.List;
 
 public class RelevantPart {
-  private boolean returnValue = false;
-  private List<Integer> parameterIndeces = new ArrayList<Integer>();
 
-  public RelevantPart(boolean rT, List<Integer> parInd) {
-    setReturnValue(rT);
-    setParameterIndeces(parInd);
-  }
+    @JsonProperty("return")
+    private boolean returnValue = false;
+    private List<Integer> parameters = new ArrayList<Integer>();
+
+    public RelevantPart(boolean rT, List<Integer> parInd) {
+        setReturnValue(rT);
+        setParameters(parInd);
+    }
 
-  public RelevantPart() {
+    public RelevantPart() {
 
-  }
+    }
 
-  public List<Integer> getParameterIndeces() {
-    return parameterIndeces;
-  }
+    public List<Integer> getParameters() {
+        return parameters;
+    }
 
-  public void setParameterIndeces(List<Integer> parameterIndeces) {
-    this.parameterIndeces = parameterIndeces;
-  }
+    public void setParameters(List<Integer> parameters) {
+        this.parameters = parameters;
+    }
 
-  public boolean getReturnValue() {
-    return returnValue;
-  }
+    public boolean getReturnValue() {
+        return returnValue;
+    }
 
-  public void setReturnValue(boolean returnValue) {
-    this.returnValue = returnValue;
-  }
+    public void setReturnValue(boolean returnValue) {
+        this.returnValue = returnValue;
+    }
 }

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/features/doc/nlp/DocCommentVector.java

@@ -1,8 +1,7 @@
 package de.fraunhofer.iem.swan.features.doc.nlp;
 
 import de.fraunhofer.iem.swan.data.Method;
-import de.fraunhofer.iem.swan.io.dataset.Parser;
-
+import de.fraunhofer.iem.swan.io.dataset.SrmList;
 import edu.stanford.nlp.util.StringUtils;
 import org.apache.commons.io.FileUtils;
 import org.deeplearning4j.models.paragraphvectors.ParagraphVectors;
@@ -15,7 +14,6 @@
 import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;
 import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
 import org.nd4j.common.io.ClassPathResource;
-import org.nd4j.linalg.cpu.nativecpu.NDArray;
 
 import java.io.File;
 import java.io.IOException;
@@ -42,15 +40,15 @@ public static void main(String[] args) {
 
         //Export method and class doc comments
 
-        Parser parser = new Parser();
-        parser.parse("/swan-dataset.json");
+        SrmList srmListUtils = new SrmList();
+        //parser.parse("/swan-dataset.json");
 
 //        for(Method method: parser.getMethods()){
 //            if(method.getJavadoc().getMethodComment().length()>0)
 //            System.out.println(NLPUtils.cleanText(method.getJavadoc().getMergedComments()));
 //        }
 
-        for(Method method: parser.getMethods()){
+        for(Method method: srmListUtils.getMethods()){
             List<String> words = StringUtils.split(method.getJavadoc().getMethodComment(), " ");
             if(method.getJavadoc().getMethodComment().length()>0 && words.size()>1)
                 System.out.println(NLPUtils.cleanFirstSentence(method.getJavadoc().getMethodComment())+" "+NLPUtils.cleanFirstSentence(method.getJavadoc().getClassComment()));

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/training/TrainingSetUpdater.java

@@ -1,9 +1,8 @@
 package de.fraunhofer.iem.swan.training;
 
-import de.fraunhofer.iem.swan.features.code.soot.Loader;
-import de.fraunhofer.iem.swan.io.dataset.Parser;
+import de.fraunhofer.iem.swan.features.code.soot.SourceFileLoader;
+import de.fraunhofer.iem.swan.io.dataset.SrmList;
 import de.fraunhofer.iem.swan.util.Util;
-import de.fraunhofer.iem.swan.io.dataset.Writer;
 import de.fraunhofer.iem.swan.data.Category;
 import de.fraunhofer.iem.swan.data.Method;
 import org.apache.commons.io.FileUtils;
@@ -44,23 +43,19 @@ public static void main(String[] args) throws IOException {
         uniqueMethod = new HashSet<>();
 
         //Load original training file methods
-        Parser parser = new Parser();
-        parser.parse(ORIGINAL_TRAINING_FILE);
+        SrmList srmListUtils = new SrmList();
+        //parser.parse(ORIGINAL_TRAINING_FILE);
 
-        for (Method m : parser.methods())
-            uniqueMethod.add(m.getClassName() + "." + m.getMethodName());
+        for (Method m : srmListUtils.getMethods())
+            uniqueMethod.add(m.getClassName() + "." + m.getName());
 
         //Add training methods to master list
-        Set<Method> trainingMethods = new HashSet<>(parser.getMethods());
+        Set<Method> trainingMethods = new HashSet<>(srmListUtils.getMethods());
 
         //Load methods from text file: extracted from thecodemaster.com and find-sec-bugs plugin
         loadClassesAndMethods("/swan/swan_core/src/main/resources/swandoc-new-training-data.txt");
 
         trainingMethods.addAll(extractMethodData(classes, method));
-
-        //Export new training file
-        Writer writer = new Writer();
-        writer.outputJSONFile(trainingMethods, NEW_TRAINING_FILE);
     }
 
     public static void extractManualList() throws IOException {
@@ -73,20 +68,16 @@ public static void extractManualList() throws IOException {
         uniqueMethod = new HashSet<>();
 
         //Load original training file methods
-        Parser parser = new Parser();
-        parser.parse(NEW_TRAINING_FILE);
+        SrmList srmListUtils = new SrmList();
+        //parser.parse(NEW_TRAINING_FILE);
 
         Set<Method> trainingMethods = new HashSet<>();
 
-        for (Method m : parser.methods()){
+        for (Method m : srmListUtils.getMethods()){
             if(m.getJavaSignature().contains("org.owasp.esapi.reference.DefaultEncoder") &&
             m.getDiscovery().contentEquals("find-sec-bugs"))
                 trainingMethods.add(m);
         }
-
-        //Export new training file
-        Writer writer = new Writer();
-        writer.outputJSONFile(trainingMethods, MANUAL_TRAINING_FILE);
     }
 
     /**