Merge pull request #182 from boyter/issue179

Issue179

Author: boyter

searchcode.properties.example

@@ -44,6 +44,7 @@ log_indexed=false
 follow_links=false
 deep_guess_files=false
 host_name=localhost:8080
+index_all_fields=content,filename,filenamereverse,path,interesting
 # The below will probably be moved into the settings page at some point in the
 # future.
 password=Adm1n234

src/main/java/com/searchcode/app/config/Values.java

@@ -111,6 +111,8 @@ public class Values {
     public static final String DEFAULT_HOST_NAME = "localhost:8080";
     public static final String DELETE_REPO_AFTER_PROCESS = "delete_repo_after_process";
     public static final String DEFAULT_DELETE_REPO_AFTER_PROCESS = "false";
+    public static final String INDEX_ALL_FIELDS = "index_all_fields";
+    public static final String DEFAULT_INDEX_ALL_FIELDS = "content,filename,filenamereverse,path,interesting";
 
     // Lucene stored values for indexing / searching
     public static final String CONTENTS = "contents";

src/main/java/com/searchcode/app/service/IndexService.java

@@ -86,6 +86,8 @@ public class IndexService implements IIndexService {
     private boolean repoJobExit = false;        // Controls if repo indexing jobs should exit instantly
     private int codeIndexLinesCount = 0;
 
+    private List<String> indexAllFields; // Contains the fields that should be added to the all portion of the index
+
     ///////////////////////////////////////////////////////////////////////
     // The below store state for when the reindexing + flip should occur
     //////////////////////////////////////////////////////////////////////
@@ -115,6 +117,8 @@ public IndexService(Data data, StatsService statsService, SearchCodeLib searchco
         this.MAX_INDEX_SIZE = this.helpers.tryParseInt(Properties.getProperties().getProperty(Values.MAXDOCUMENTQUEUESIZE, Values.DEFAULTMAXDOCUMENTQUEUESIZE), Values.DEFAULTMAXDOCUMENTQUEUESIZE);
         this.MAX_LINES_INDEX_SIZE = this.helpers.tryParseInt(Properties.getProperties().getProperty(Values.MAXDOCUMENTQUEUELINESIZE, Values.DEFAULTMAXDOCUMENTQUEUELINESIZE), Values.DEFAULTMAXDOCUMENTQUEUELINESIZE);
 
+        this.indexAllFields = Arrays.asList(Properties.getProperties().getProperty(Values.INDEX_ALL_FIELDS, Values.DEFAULT_INDEX_ALL_FIELDS).split(","));
+
         // Locations that should never change once class created
         this.INDEX_A_LOCATION = Paths.get(Properties.getProperties().getProperty(Values.INDEXLOCATION, Values.DEFAULTINDEXLOCATION) + "/" + Values.INDEX_A);
         this.INDEX_B_LOCATION = Paths.get(Properties.getProperties().getProperty(Values.INDEXLOCATION, Values.DEFAULTINDEXLOCATION) + "/" + Values.INDEX_B);
@@ -266,20 +270,33 @@ public Document buildDocument(CodeIndexDocument codeIndexDocument) {
             document.add(new SortedSetDocValuesFacetField(Values.SOURCE, codeIndexDocument.getSource()));
         }
 
-
         this.searchcodeLib.addToSpellingCorrector(codeIndexDocument.getContents());
 
         // This is the main pipeline for making code searchable and probably the most important
         // part of the indexer codebase
-        String indexContents = this.searchcodeLib.codeCleanPipeline(codeIndexDocument.getFileName()) + " " +
-                new StringBuilder(codeIndexDocument.getFileName()).reverse().toString() + " " +
-                this.searchcodeLib.splitKeywords(codeIndexDocument.getFileName(), true) + " " +
-                codeIndexDocument.getFileLocationFilename() + " " +
-                codeIndexDocument.getFileLocation() + " " +
-                this.searchcodeLib.splitKeywords(codeIndexDocument.getContents(), true) + " " +
-                this.searchcodeLib.codeCleanPipeline(codeIndexDocument.getContents()) + " " +
-                this.searchcodeLib.findInterestingKeywords(codeIndexDocument.getContents()) + " " +
-                this.searchcodeLib.findInterestingCharacters(codeIndexDocument.getContents()).toLowerCase();
+        StringBuilder indexBuilder = new StringBuilder();
+
+        if (this.indexAllFields.contains("filename")) {
+            indexBuilder.append(this.searchcodeLib.codeCleanPipeline(codeIndexDocument.getFileName())).append(" ");
+        }
+        if (this.indexAllFields.contains("filenamereverse")) {
+            indexBuilder.append(new StringBuilder(codeIndexDocument.getFileName()).reverse().toString()).append(" ");
+        }
+        if (this.indexAllFields.contains("path")) {
+            indexBuilder.append(this.searchcodeLib.splitKeywords(codeIndexDocument.getFileName(), true)).append(" ");
+            indexBuilder.append(codeIndexDocument.getFileLocationFilename()).append(" ");
+            indexBuilder.append(codeIndexDocument.getFileLocation()).append(" ");
+        }
+        if (this.indexAllFields.contains("content")) {
+            indexBuilder.append(this.searchcodeLib.splitKeywords(codeIndexDocument.getContents(), true)).append(" ");
+            indexBuilder.append( this.searchcodeLib.codeCleanPipeline(codeIndexDocument.getContents())).append(" ");
+        }
+        if (this.indexAllFields.contains("interesting")) {
+            indexBuilder.append(this.searchcodeLib.findInterestingKeywords(codeIndexDocument.getContents())).append(" ");
+            indexBuilder.append(this.searchcodeLib.findInterestingCharacters(codeIndexDocument.getContents()));
+        }
+
+        String indexContents = indexBuilder.toString();
 
         document.add(new TextField(Values.REPONAME,                 codeIndexDocument.getRepoName().replace(" ", "_"), Field.Store.YES));
         document.add(new TextField(Values.REPO_NAME_LITERAL,        this.helpers.replaceForIndex(codeIndexDocument.getRepoName()).toLowerCase(), Field.Store.NO));

src/main/java/com/searchcode/app/service/route/AdminRouteService.java

@@ -144,6 +144,7 @@ public Map<String, Object> adminPage(Request request, Response response) {
         map.put(Values.FOLLOW_LINKS, Properties.getProperties().getProperty(Values.FOLLOW_LINKS, Values.DEFAULT_FOLLOW_LINKS));
         map.put(Values.DEEP_GUESS_FILES, Properties.getProperties().getProperty(Values.DEEP_GUESS_FILES, Values.DEFAULT_DEEP_GUESS_FILES));
         map.put(Values.HOST_NAME, Properties.getProperties().getProperty(Values.HOST_NAME, Values.DEFAULT_HOST_NAME));
+        map.put(Values.INDEX_ALL_FIELDS, Properties.getProperties().getProperty(Values.INDEX_ALL_FIELDS, Values.DEFAULT_INDEX_ALL_FIELDS));
 
 
         map.put("repoCount", this.getStat("repoCount"));

src/main/resources/spark/template/freemarker/admin.ftl

@@ -190,7 +190,8 @@ default_and_match=${default_and_match}
 log_indexed=${log_indexed}
 follow_links=${follow_links}
 deep_guess_files=${deep_guess_files}
-host_name=${host_name}</textarea>
+host_name=${host_name}
+index_all_fields=${index_all_fields}</textarea>
         </div>
 
         <div style="width:100%; display: inline-block;">

src/main/resources/spark/template/freemarker/documentation.ftl

@@ -750,6 +750,8 @@ String myHmac = HmacUtils.hmacSha512Hex(MYPRIVATEKEY, PARAMSTOHMAC);</textarea>
               <dd>Boolean value true or false. If set to true when a file is encountered that cannot be classified though naming conventions its keywords will be analysed and a best guess made. This can be CPU heavy or incorrectly classify some files. Defaults to false.</dd>
               <dt>host_name</dt>
               <dd>String value. Set this to the expected DNS host name for your searchcode server instance. This will allow things like RSS links to work.</dd>
+              <dt>index_all_fields</dt>
+              <dd>A list of file content that will be added to the all portion of the index. You could use this to exclude filename or paths. Defaults to index_all_fields=content,filename,filenamereverse,path,interesting</dd>
             </dl>
         </p>