Add ability to control what appears in all index #179

Author: boyter

searchcode.properties.example

@@ -44,6 +44,7 @@ log_indexed=false
 follow_links=false
 deep_guess_files=false
 host_name=localhost:8080
+index_all_fields=content,filename,filenamereverse,path,interesting
 # The below will probably be moved into the settings page at some point in the
 # future.
 password=Adm1n234

src/main/java/com/searchcode/app/config/Values.java

@@ -111,6 +111,8 @@ public class Values {
     public static final String DEFAULT_HOST_NAME = "localhost:8080";
     public static final String DELETE_REPO_AFTER_PROCESS = "delete_repo_after_process";
     public static final String DEFAULT_DELETE_REPO_AFTER_PROCESS = "false";
+    public static final String INDEX_ALL_FIELDS = "index_all_fields";
+    public static final String DEFAULT_INDEX_ALL_FIELDS = "content,filename,filenamereverse,path,interesting";
 
     // Lucene stored values for indexing / searching
     public static final String CONTENTS = "contents";

src/main/java/com/searchcode/app/service/IndexService.java

@@ -86,6 +86,8 @@ public class IndexService implements IIndexService {
     private boolean repoJobExit = false;        // Controls if repo indexing jobs should exit instantly
     private int codeIndexLinesCount = 0;
 
+    private List<String> indexAllFields; // Contains the fields that should be added to the all portion of the index
+
     ///////////////////////////////////////////////////////////////////////
     // The below store state for when the reindexing + flip should occur
     //////////////////////////////////////////////////////////////////////
@@ -115,6 +117,8 @@ public IndexService(Data data, StatsService statsService, SearchCodeLib searchco
         this.MAX_INDEX_SIZE = this.helpers.tryParseInt(Properties.getProperties().getProperty(Values.MAXDOCUMENTQUEUESIZE, Values.DEFAULTMAXDOCUMENTQUEUESIZE), Values.DEFAULTMAXDOCUMENTQUEUESIZE);
         this.MAX_LINES_INDEX_SIZE = this.helpers.tryParseInt(Properties.getProperties().getProperty(Values.MAXDOCUMENTQUEUELINESIZE, Values.DEFAULTMAXDOCUMENTQUEUELINESIZE), Values.DEFAULTMAXDOCUMENTQUEUELINESIZE);
 
+        this.indexAllFields = Arrays.asList(Properties.getProperties().getProperty(Values.INDEX_ALL_FIELDS, Values.DEFAULT_INDEX_ALL_FIELDS).split(","));
+
         // Locations that should never change once class created
         this.INDEX_A_LOCATION = Paths.get(Properties.getProperties().getProperty(Values.INDEXLOCATION, Values.DEFAULTINDEXLOCATION) + "/" + Values.INDEX_A);
         this.INDEX_B_LOCATION = Paths.get(Properties.getProperties().getProperty(Values.INDEXLOCATION, Values.DEFAULTINDEXLOCATION) + "/" + Values.INDEX_B);
@@ -266,20 +270,33 @@ public Document buildDocument(CodeIndexDocument codeIndexDocument) {
             document.add(new SortedSetDocValuesFacetField(Values.SOURCE, codeIndexDocument.getSource()));
         }
 
-
         this.searchcodeLib.addToSpellingCorrector(codeIndexDocument.getContents());
 
         // This is the main pipeline for making code searchable and probably the most important
         // part of the indexer codebase
-        String indexContents = this.searchcodeLib.codeCleanPipeline(codeIndexDocument.getFileName()) + " " +
-                new StringBuilder(codeIndexDocument.getFileName()).reverse().toString() + " " +
-                this.searchcodeLib.splitKeywords(codeIndexDocument.getFileName(), true) + " " +
-                codeIndexDocument.getFileLocationFilename() + " " +
-                codeIndexDocument.getFileLocation() + " " +
-                this.searchcodeLib.splitKeywords(codeIndexDocument.getContents(), true) + " " +
-                this.searchcodeLib.codeCleanPipeline(codeIndexDocument.getContents()) + " " +
-                this.searchcodeLib.findInterestingKeywords(codeIndexDocument.getContents()) + " " +
-                this.searchcodeLib.findInterestingCharacters(codeIndexDocument.getContents()).toLowerCase();
+        StringBuilder indexBuilder = new StringBuilder();
+
+        if (this.indexAllFields.contains("filename")) {
+            indexBuilder.append(this.searchcodeLib.codeCleanPipeline(codeIndexDocument.getFileName())).append(" ");
+        }
+        if (this.indexAllFields.contains("filenamereverse")) {
+            indexBuilder.append(new StringBuilder(codeIndexDocument.getFileName()).reverse().toString()).append(" ");
+        }
+        if (this.indexAllFields.contains("path")) {
+            indexBuilder.append(this.searchcodeLib.splitKeywords(codeIndexDocument.getFileName(), true)).append(" ");
+            indexBuilder.append(codeIndexDocument.getFileLocationFilename()).append(" ");
+            indexBuilder.append(codeIndexDocument.getFileLocation()).append(" ");
+        }
+        if (this.indexAllFields.contains("content")) {
+            indexBuilder.append(this.searchcodeLib.splitKeywords(codeIndexDocument.getContents(), true)).append(" ");
+            indexBuilder.append( this.searchcodeLib.codeCleanPipeline(codeIndexDocument.getContents())).append(" ");
+        }
+        if (this.indexAllFields.contains("interesting")) {
+            indexBuilder.append(this.searchcodeLib.findInterestingKeywords(codeIndexDocument.getContents())).append(" ");
+            indexBuilder.append(this.searchcodeLib.findInterestingCharacters(codeIndexDocument.getContents())).append(" ");
+        }
+
+        String indexContents = this.searchcodeLib.codeCleanPipeline(indexBuilder.toString());
 
         document.add(new TextField(Values.REPONAME,                 codeIndexDocument.getRepoName().replace(" ", "_"), Field.Store.YES));
         document.add(new TextField(Values.REPO_NAME_LITERAL,        this.helpers.replaceForIndex(codeIndexDocument.getRepoName()).toLowerCase(), Field.Store.NO));