Move model evaluation code for MEKA, WEKA and ML-Plan

Author: oshando

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/model/ModelEvaluator.java

@@ -2,7 +2,9 @@
 
 import de.fraunhofer.iem.swan.cli.SwanOptions;
 import de.fraunhofer.iem.swan.data.Method;
-import de.fraunhofer.iem.swan.features.FeaturesHandler;
+import de.fraunhofer.iem.swan.features.IFeatureSet;
+import de.fraunhofer.iem.swan.features.MekaFeatureSet;
+import de.fraunhofer.iem.swan.features.WekaFeatureSet;
 import de.fraunhofer.iem.swan.model.engine.MLPlan;
 import de.fraunhofer.iem.swan.model.engine.Meka;
 import de.fraunhofer.iem.swan.model.engine.Weka;
@@ -29,12 +31,12 @@ public enum Phase {
         PREDICT
     }
 
-    private FeaturesHandler features;
+    private IFeatureSet features;
     private SwanOptions options;
     private Set<Method> methods;
     private static final Logger logger = LoggerFactory.getLogger(ModelEvaluator.class);
 
-    public ModelEvaluator(FeaturesHandler features, SwanOptions options, Set<Method> methods) {
+    public ModelEvaluator(IFeatureSet features, SwanOptions options, Set<Method> methods) {
         this.features = features;
         this.options = options;
         this.methods = methods;
@@ -51,18 +53,18 @@ public void trainModel() {
 
             case MEKA:
                 logger.info("Evaluating model with MEKA");
-                Meka meka = new Meka(features, options, methods);
+                Meka meka = new Meka((MekaFeatureSet)features, options, methods);
                 meka.trainModel();
                 break;
             case WEKA:
                 logger.info("Evaluating model with WEKA");
-                Weka weka = new Weka(features, options);
+                Weka weka = new Weka((WekaFeatureSet) features, options);
                 weka.trainModel();
                 break;
             case MLPLAN:
                 logger.info("Evaluating model with ML-PLAN");
                 MLPlan mlPlan = new MLPlan();
-                mlPlan.evaluateDataset(features.getInstances().get("train"));
+                mlPlan.evaluateDataset(((WekaFeatureSet)features).getInstances().get("train"));
                 break;
         }
     }

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/model/engine/MLPlan.java

@@ -108,7 +108,7 @@ public HashMap<String, ArrayList<Double>> evaluateDataset(Instances instances1)
                 Instances testInstances = testLoader.getDataSet();
                 testInstances.setClassIndex(testInstances.numAttributes() - 1);
 
-                monteCarloValidator.evaluate(optimizedClassifier, trainInstances, testInstances, iteration);
+                monteCarloValidator.evaluate(optimizedClassifier, trainInstances, testInstances);
 
 
                 /* evaluate solution produced by mlplan */

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/model/engine/Meka.java

@@ -3,7 +3,7 @@
 import de.fraunhofer.iem.swan.cli.SwanOptions;
 import de.fraunhofer.iem.swan.data.Category;
 import de.fraunhofer.iem.swan.data.Method;
-import de.fraunhofer.iem.swan.features.FeaturesHandler;
+import de.fraunhofer.iem.swan.features.MekaFeatureSet;
 import de.fraunhofer.iem.swan.io.dataset.SrmList;
 import de.fraunhofer.iem.swan.io.dataset.SrmListUtils;
 import de.fraunhofer.iem.swan.model.ModelEvaluator;
@@ -22,12 +22,12 @@
 
 public class Meka {
 
-    private FeaturesHandler features;
+    private MekaFeatureSet features;
     private SwanOptions options;
     private Set<Method> methods;
     private static final Logger logger = LoggerFactory.getLogger(ModelEvaluator.class);
 
-    public Meka(FeaturesHandler features, SwanOptions options, Set<Method> methods) {
+    public Meka(MekaFeatureSet features, SwanOptions options, Set<Method> methods) {
         this.features = features;
         this.options = options;
         this.methods = methods;

swan-pipeline/src/main/java/de/fraunhofer/iem/swan/model/engine/Weka.java

@@ -1,7 +1,7 @@
 package de.fraunhofer.iem.swan.model.engine;
 
 import de.fraunhofer.iem.swan.cli.SwanOptions;
-import de.fraunhofer.iem.swan.features.FeaturesHandler;
+import de.fraunhofer.iem.swan.features.WekaFeatureSet;
 import de.fraunhofer.iem.swan.model.ModelEvaluator;
 import de.fraunhofer.iem.swan.model.MonteCarloValidator;
 import org.slf4j.Logger;
@@ -31,11 +31,11 @@
  */
 public class Weka {
 
-    private FeaturesHandler features;
+    private WekaFeatureSet features;
     private SwanOptions options;
     private static final Logger logger = LoggerFactory.getLogger(ModelEvaluator.class);
 
-    public Weka(FeaturesHandler features, SwanOptions options) {
+    public Weka(WekaFeatureSet features, SwanOptions options) {
         this.features = features;
         this.options = options;
     }
@@ -47,14 +47,11 @@ public Weka(FeaturesHandler features, SwanOptions options) {
      */
     public HashMap<String, HashMap<String, String>> trainModel() {
 
-
         //Phase 1: classify SRM classes
         for (String srm : options.getSrmClasses())
             runManualEvaluation(features.getInstances().get(srm));
 
-        //Filter methods from CWE instances that were not classified
-        //into one of the SRM classes
-
+        //Filter methods from CWE instances that were not classified into one of the SRM classes
         //Phase 2: classify CWE classes
         for (String cwe : options.getCweClasses())
             runManualEvaluation(features.getInstances().get(cwe));
@@ -67,6 +64,9 @@ public HashMap<String, HashMap<String, String>> trainModel() {
      */
     public HashMap<String, HashMap<String, String>> runManualEvaluation(Instances instances) {
 
+        String category = instances.attribute(instances.numAttributes()-1).name();
+        instances.setClass(instances.attribute(instances.numAttributes()-1));
+
         LinkedHashMap<String, HashMap<String, String>> fMeasure = new LinkedHashMap<>();
 
         List<Classifier> classifiers = new ArrayList<>();
@@ -85,7 +85,7 @@ public HashMap<String, HashMap<String, String>> runManualEvaluation(Instances in
             evaluator.monteCarloValidate(instances, classifier, options.getTrainTestSplit(), options.getIterations());
 
             for (String key : evaluator.getFMeasure().keySet())
-                logger.info("F-measure for {} using {}: {}", key, classifier.getClass().getSimpleName(), evaluator.getFMeasure().get(key));
+                logger.info("F-measure for {}({}) using {}: {}", category, key , classifier.getClass().getSimpleName(), evaluator.getFMeasure().get(key));
         }
         return fMeasure;
     }