Merge pull request #53 from secure-software-engineering/develop

Fix/refactor feature extraction and classification modules

Author: oshando

doc-coverage-doclet/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>de.fraunhofer.iem</groupId>
     <artifactId>coverage-doclet</artifactId>
-    <version>1.0-SNAPSHOT</version>
+    <version>1.1</version>
 
     <properties>
         <maven.compiler.source>11</maven.compiler.source>

doc-xml-exporter-doclet/pom.xml

@@ -10,7 +10,7 @@
 
     <groupId>de.fraunhofer.iem.doclet</groupId>
     <artifactId>xml-doclet</artifactId>
-    <version>1.3</version>
+    <version>1.4</version>
 
     <properties>
         <maven.compiler.source>11</maven.compiler.source>

swan-pipeline/pom.xml

@@ -192,7 +192,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.13.2.1</version>
+            <version>2.13.2.2</version>
         </dependency>
         <dependency>
             <groupId>net.sf.meka</groupId>

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/SwanPipeline.java

@@ -4,12 +4,8 @@
 import de.fraunhofer.iem.swan.features.FeatureSetSelector;
 import de.fraunhofer.iem.swan.features.IFeatureSet;
 import de.fraunhofer.iem.swan.io.dataset.Dataset;
-import de.fraunhofer.iem.swan.io.dataset.SrmList;
-import de.fraunhofer.iem.swan.io.dataset.SrmListUtils;
-import de.fraunhofer.iem.swan.io.doc.JavadocProcessor;
+import de.fraunhofer.iem.swan.io.dataset.DatasetProcessor;
 import de.fraunhofer.iem.swan.model.ModelEvaluator;
-import de.fraunhofer.iem.swan.soot.Soot;
-import de.fraunhofer.iem.swan.util.Util;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -41,26 +37,9 @@ public void run() throws IOException, InterruptedException {
 
         long startAnalysisTime = System.currentTimeMillis();
 
-        //Run Soot
-        Soot soot = new Soot(options.getTrainDataDir(), options.getTestDataDir());
-
-        // Load methods in training dataset
-        Dataset dataset = new Dataset();
-        dataset.setTrain(SrmListUtils.importFile(options.getDatasetJson()));
-
-        if (!options.getTrainDataDir().isEmpty())
-            soot.cleanupList(dataset.getTrain());
-
-        logger.info("Loaded {} training methods, distribution={}", dataset.getTrainMethods().size(), Util.countCategories(dataset.getTrainMethods()));
-
-        //Load methods from the test set
-        dataset.setTest(new SrmList(options.getTestDataDir()));
-        dataset.getTest().setMethods(soot.loadMethods(dataset.getTest().getTestClasses()));
-
-        //Extract doc comments and add to test set, if option is selected
-        JavadocProcessor javadocProcessor = new JavadocProcessor(options.getTestDataSourceDir(), options.getOutputDir());
-        javadocProcessor.run(dataset.getTestMethods(), options.getFeatureSet());
-        logger.info("Loaded {} methods from {}", dataset.getTestMethods().size(), options.getTestDataDir());
+        //Create train and test datasets
+        DatasetProcessor datasetProcessor = new DatasetProcessor(options);
+        Dataset dataset = datasetProcessor.run();
 
         //Initialize and populate features
         FeatureSetSelector featureSetSelector = new FeatureSetSelector();

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/cli/CliRunner.java

@@ -32,10 +32,10 @@ public class CliRunner implements Callable<Integer> {
     @CommandLine.Option(names = {"-o", "--output"}, description = {"Directory to save output files"})
     private String outputDir = "";
 
-    @CommandLine.Option(names = {"-f", "--feature"}, description = {"Select one or more feature sets: all, code, doc-auto or doc-manual"})
+    @CommandLine.Option(names = {"-f", "--feature"}, arity = "1..*", description = {"Select one or more feature sets: all, code, doc-auto or doc-manual"})
     private List<String> featureSet =  Collections.singletonList("code");
 
-    @CommandLine.Option(names = {"-t", "--toolkit"}, description = {"ML toolkit: meka, weka, ml-plan"})
+    @CommandLine.Option(names = {"-t", "--toolkit"}, description = {"ML toolkit: meka, weka, autoweka, mlplan"})
     private String toolkit = "meka";
 
     @CommandLine.Option(names = {"-s", "--srm"}, description = {"SRM: all, source, sink, sanitizer, authentication, relevant"})
@@ -50,6 +50,9 @@ public class CliRunner implements Callable<Integer> {
     @CommandLine.Option(names = {"-doc", "--documented"}, description = {"Use only methods with Javadoc"})
     private boolean isDocumented = true;
 
+    @CommandLine.Option(names = {"-att", "--attribute-selection"}, description = {"Use attribute selection"})
+    private boolean reduceAttributes = false;
+
     @CommandLine.Option(names = {"-i", "--iterations"}, description = {"Number of iterations for training"})
     private int iterations = 10;
 
@@ -62,6 +65,12 @@ public class CliRunner implements Callable<Integer> {
     @CommandLine.Option(names = {"-pt", "--prediction-threshold"}, description = {"Threshold for predicting categories"})
     private double predictionThreshold = 0.5;
 
+    @CommandLine.Option(names = {"-ds", "--discovery"}, arity = "1..*", description = {"Select discovery for training set SRMs"})
+    private List<String> discovery =  Collections.singletonList("manual");
+
+    @CommandLine.Option(names = {"-tl", "--timelimit"}, description = {"Time (minutes) to execute operation "})
+    private int timeLimit = 1;
+
     public SwanOptions initializeOptions(){
 
         SwanOptions options = new SwanOptions(testDataDir,
@@ -81,6 +90,9 @@ public SwanOptions initializeOptions(){
         options.setInstances(arffInstancesFiles);
         options.setTrainDataSourceDir(trainDataDirSource);
         options.setTestDataSourceDir(testDataSourceDir);
+        options.setReduceAttributes(false);
+        options.setDiscovery(discovery);
+        options.setTimeLimit(timeLimit);
 
         return options;
     }