BOBO-287 Only fetch requested fields

bobo-browse/src/main/java/com/browseengine/bobo/api/BoboSubBrowser.java

@@ -306,9 +306,9 @@ public void browse(BrowseRequest req, Collector collector, Map<String, FacetAcce
 
   @Override
   public SortCollector getSortCollector(SortField[] sort, Query q, int offset, int count,
-      boolean fetchStoredFields, Set<String> termVectorsToFetch, String[] groupBy, int maxPerGroup,
-      boolean collectDocIdCache) {
-    return SortCollector.buildSortCollector(this, q, sort, offset, count, fetchStoredFields,
+      boolean fetchAllFields, Set<String> fieldsToFetch, Set<String> termVectorsToFetch, String[] groupBy,
+      int maxPerGroup, boolean collectDocIdCache) {
+    return SortCollector.buildSortCollector(this, q, sort, offset, count, fetchAllFields, fieldsToFetch,
       termVectorsToFetch, groupBy, maxPerGroup, collectDocIdCache);
   }
 

bobo-browse/src/main/java/com/browseengine/bobo/api/Browsable.java

@@ -40,8 +40,8 @@ void browse(BrowseRequest req, Collector hitCollector, Map<String, FacetAccessib
   int numDocs();
 
   SortCollector getSortCollector(SortField[] sort, Query q, int offset, int count,
-      boolean fetchStoredFields, Set<String> termVectorsToFetch, String[] groupBy, int maxPerGroup,
-      boolean collectDocIdCache);
+                                 boolean fetchAllFields, Set<String> fieldsToFetch, Set<String> termVectorsToFetch, String[] groupBy, int maxPerGroup,
+                                 boolean collectDocIdCache);
 
   void doClose() throws IOException;
 

bobo-browse/src/main/java/com/browseengine/bobo/api/BrowseRequest.java

@@ -29,6 +29,7 @@
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
@@ -70,7 +71,8 @@ public final void setTid(long tid) {
   private Query _query;
   private int _offset;
   private int _count;
-  private boolean _fetchStoredFields;
+  private boolean _fetchAllFields;
+  private Set<String> _fieldsToFetch;
   private Filter _filter;
   private boolean _showExplanation;
   private String _groupBy; // TODO: Leave here for backward compatible reason, will remove it later.
@@ -176,7 +178,8 @@ public BrowseRequest() {
     _facetSpecMap = new HashMap<String, FacetSpec>();
     _facetHandlerDataMap = new HashMap<String, FacetHandlerInitializerParam>();
     _filter = null;
-    _fetchStoredFields = false;
+    _fetchAllFields = false;
+    _fieldsToFetch = new HashSet<String>();
     _groupBy = null;
     _groupByMulti = null;
     _maxPerGroup = 0;
@@ -188,12 +191,31 @@ public BrowseRequest clearSort() {
     return this;
   }
 
-  public boolean isFetchStoredFields() {
-    return _fetchStoredFields;
+  public boolean isFetchAllFields() {
+    return _fetchAllFields;
   }
 
-  public BrowseRequest setFetchStoredFields(boolean fetchStoredFields) {
-    _fetchStoredFields = fetchStoredFields;
+  /**
+   * True if all fields should be fetched, in which case fieldsToFetch is ignored
+   * @param fetchAllFields
+   * @return
+   */
+  public BrowseRequest setFetchAllFields(boolean fetchAllFields) {
+    this._fetchAllFields = fetchAllFields;
+    return this;
+  }
+
+  public Set<String> getFieldsToFetch(){
+    return _fieldsToFetch;
+  }
+
+  /**
+   * Specifies fields to fetch.  Ignored if fetchAllFields is set to true
+   * @param fieldsToFetch
+   * @return
+   */
+  public BrowseRequest setFieldsToFetch(Set<String> fieldsToFetch) {
+    _fieldsToFetch = fieldsToFetch;
     return this;
   }
 
@@ -426,7 +448,7 @@ public String toString() {
     buf.append("sort spec: ").append(_sortSpecs).append('\n');
     buf.append("selections: ").append(_selections).append('\n');
     buf.append("facet spec: ").append(_facetSpecMap).append('\n');
-    buf.append("fetch stored fields: ").append(_fetchStoredFields).append('\n');
+    buf.append("fields to fetch: ").append(_fieldsToFetch).append('\n');
     return buf.toString();
   }
 }

bobo-browse/src/main/java/com/browseengine/bobo/api/MultiBoboBrowser.java

@@ -140,8 +140,8 @@ public BrowseResult browse(BrowseRequest req) throws BrowseException {
           + count);
     }
     SortCollector collector = getSortCollector(req.getSort(), req.getQuery(), offset, count,
-      req.isFetchStoredFields(), req.getTermVectorsToFetch(), req.getGroupBy(), req.getMaxPerGroup(),
-      req.getCollectDocIdCache());
+        req.isFetchAllFields(), req.getFieldsToFetch(), req.getTermVectorsToFetch(), req.getGroupBy(),
+        req.getMaxPerGroup(), req.getCollectDocIdCache());
 
     Map<String, FacetAccessible> facetCollectors = new HashMap<String, FacetAccessible>();
     browse(req, collector, facetCollectors, 0);
@@ -269,13 +269,13 @@ public void setFacetHandler(FacetHandler<?> facetHandler) throws IOException {
 
   @Override
   public SortCollector getSortCollector(SortField[] sort, Query q, int offset, int count,
-      boolean fetchStoredFields, Set<String> termVectorsToFetch, String[] groupBy, int maxPerGroup,
-      boolean collectDocIdCache) {
+                                        boolean fetchAllFields, Set<String> fieldsToFetch, Set<String> termVectorsToFetch, String[] groupBy, int maxPerGroup,
+                                        boolean collectDocIdCache) {
     if (_subBrowsers.length == 1) {
-      return _subBrowsers[0].getSortCollector(sort, q, offset, count, fetchStoredFields,
+      return _subBrowsers[0].getSortCollector(sort, q, offset, count, fetchAllFields, fieldsToFetch,
         termVectorsToFetch, groupBy, maxPerGroup, collectDocIdCache);
     }
-    return SortCollector.buildSortCollector(this, q, sort, offset, count, fetchStoredFields,
+    return SortCollector.buildSortCollector(this, q, sort, offset, count, fetchAllFields, fieldsToFetch,
       termVectorsToFetch, groupBy, maxPerGroup, collectDocIdCache);
   }
 

bobo-browse/src/main/java/com/browseengine/bobo/client/BrowseRequestBuilder.java

@@ -7,6 +7,8 @@
 import com.browseengine.bobo.api.FacetSpec;
 import com.browseengine.bobo.api.FacetSpec.FacetSortSpec;
 
+import java.util.HashSet;
+
 public class BrowseRequestBuilder {
   private BrowseRequest _req;
   private String _qString;
@@ -83,7 +85,8 @@ public BrowseRequestBuilder clear() {
     _req = new BrowseRequest();
     _req.setOffset(0);
     _req.setCount(5);
-    _req.setFetchStoredFields(true);
+    _req.setFetchAllFields(false);
+    _req.setFieldsToFetch(new HashSet<String>());
     _qString = null;
     return this;
   }

bobo-browse/src/main/java/com/browseengine/bobo/sort/SortCollector.java

@@ -133,12 +133,14 @@ public void clearRuntimeFacetData() {
 
   protected Collector _collector = null;
   protected final SortField[] _sortFields;
-  protected final boolean _fetchStoredFields;
+  protected final boolean _fetchAllFields;
+  protected final Set<String> _fieldsToFetch;
   protected boolean _closed = false;
 
-  protected SortCollector(SortField[] sortFields, boolean fetchStoredFields) {
+  protected SortCollector(SortField[] sortFields, boolean fetchAllFields, Set<String> fieldsToFetch) {
     _sortFields = sortFields;
-    _fetchStoredFields = fetchStoredFields;
+    _fetchAllFields = fetchAllFields;
+    _fieldsToFetch = fieldsToFetch;
   }
 
   abstract public BrowseHit[] topDocs() throws IOException;
@@ -233,7 +235,7 @@ private static SortField convert(Browsable browser, SortField sort) {
   }
 
   public static SortCollector buildSortCollector(Browsable browser, Query q, SortField[] sort,
-      int offset, int count, boolean fetchStoredFields, Set<String> termVectorsToFetch,
+      int offset, int count, boolean fetchAllFields, Set<String> fieldsToFetch, Set<String> termVectorsToFetch,
       String[] groupBy, int maxPerGroup, boolean collectDocIdCache) {
     if (sort == null || sort.length == 0) {
       if (q != null && !(q instanceof MatchAllDocsQuery)) {
@@ -263,7 +265,7 @@ public static SortCollector buildSortCollector(Browsable browser, Query q, SortF
       compSource = new MultiDocIdComparatorSource(compSources);
     }
     return new SortCollectorImpl(compSource, sort, browser, offset, count, doScoring,
-        fetchStoredFields, termVectorsToFetch, groupBy, maxPerGroup, collectDocIdCache);
+        fetchAllFields, fieldsToFetch, termVectorsToFetch, groupBy, maxPerGroup, collectDocIdCache);
   }
 
   public SortCollector setCollector(Collector collector) {

bobo-browse/src/main/java/com/browseengine/bobo/sort/SortCollectorImpl.java

@@ -16,6 +16,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.lucene.document.DocumentStoredFieldVisitor;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.DocsAndPositionsEnum;
@@ -125,9 +126,10 @@ Comparable<?> getValue() {
 
   @SuppressWarnings("unchecked")
   public SortCollectorImpl(DocComparatorSource compSource, SortField[] sortFields,
-      Browsable boboBrowser, int offset, int count, boolean doScoring, boolean fetchStoredFields,
-      Set<String> termVectorsToFetch, String[] groupBy, int maxPerGroup, boolean collectDocIdCache) {
-    super(sortFields, fetchStoredFields);
+      Browsable boboBrowser, int offset, int count, boolean doScoring, boolean fetchAllFields,
+      Set<String> fieldsToFetch, Set<String> termVectorsToFetch, String[] groupBy, int maxPerGroup,
+      boolean collectDocIdCache) {
+    super(sortFields, fetchAllFields, fieldsToFetch);
     assert (offset >= 0 && count >= 0);
     _boboBrowser = boboBrowser;
     _compSource = compSource;
@@ -412,22 +414,32 @@ public BrowseHit[] topDocs() throws IOException {
 
     Map<String, FacetHandler<?>> facetHandlerMap = _boboBrowser.getFacetHandlerMap();
     return buildHits(resList.toArray(new MyScoreDoc[resList.size()]), _sortFields, facetHandlerMap,
-      _fetchStoredFields, _termVectorsToFetch, groupBy, _groupAccessibles);
+       _fetchAllFields, _fieldsToFetch, _termVectorsToFetch, groupBy, _groupAccessibles);
   }
 
   protected static BrowseHit[] buildHits(MyScoreDoc[] scoreDocs, SortField[] sortFields,
-      Map<String, FacetHandler<?>> facetHandlerMap, boolean fetchStoredFields,
-      Set<String> termVectorsToFetch, FacetHandler<?> groupBy,
+      Map<String, FacetHandler<?>> facetHandlerMap, boolean fetchAllFields,
+      Set<String> fieldsToFetch, Set<String> termVectorsToFetch, FacetHandler<?> groupBy,
       CombinedFacetAccessible[] groupAccessibles) throws IOException {
     BrowseHit[] hits = new BrowseHit[scoreDocs.length];
     Collection<FacetHandler<?>> facetHandlers = facetHandlerMap.values();
+
+    if (fieldsToFetch != null && fieldsToFetch.isEmpty()) {
+      fieldsToFetch = null;
+    }
     for (int i = scoreDocs.length - 1; i >= 0; i--) {
       MyScoreDoc fdoc = scoreDocs[i];
       BoboSegmentReader reader = fdoc.reader;
       BrowseHit hit = new BrowseHit();
-      if (fetchStoredFields) {
+
+      if (fetchAllFields) {
         hit.setStoredFields(reader.document(fdoc.doc));
+      } else if (fieldsToFetch != null) {
+        DocumentStoredFieldVisitor fieldVisitor = new DocumentStoredFieldVisitor(fieldsToFetch);
+        reader.document(fdoc.doc, fieldVisitor);
+        hit.setStoredFields(fieldVisitor.getDocument());
       }
+
       if (termVectorsToFetch != null && termVectorsToFetch.size() > 0) {
         Map<String, List<BoboTerm>> tvMap = new HashMap<String, List<BoboTerm>>();
         hit.setTermVectorMap(tvMap);

bobo-browse/src/test/java/com/browseengine/bobo/test/BoboTestCase.java

@@ -36,6 +36,7 @@
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
@@ -756,8 +757,7 @@ public void testStoredFacetField() throws Exception {
 
     BrowseSelection colorSel = new BrowseSelection("testStored");
     colorSel.addValue("stored");
-    br.addSelection(colorSel);
-    br.setFetchStoredFields(true);
+    br.addSelection(colorSel);    
 
     BrowseResult result = null;
     BoboBrowser boboBrowser = null;
@@ -768,6 +768,13 @@ public void testStoredFacetField() throws Exception {
       assertEquals(1, result.getNumHits());
       BrowseHit hit = result.getHits()[0];
       List<SerializableField> storedFields = hit.getStoredFields();
+      assertNull(storedFields);
+
+      br.setFieldsToFetch(Collections.singleton("testStored"));
+      result = boboBrowser.browse(br);
+      assertEquals(1, result.getNumHits());
+      hit = result.getHits()[0];
+      storedFields = hit.getStoredFields();
       assertNotNull(storedFields);
 
       List<String> fieldValues = new ArrayList<String>();
@@ -782,6 +789,26 @@ public void testStoredFacetField() throws Exception {
       assertEquals(1, values.length);
       assertTrue("stored".equals(values[0]));
 
+      br.setFetchAllFields(true);
+      br.setFieldsToFetch(null);
+      result = boboBrowser.browse(br);
+      assertEquals(1, result.getNumHits());
+      hit = result.getHits()[0];
+      storedFields = hit.getStoredFields();
+      assertNotNull(storedFields);
+
+      fieldValues = new ArrayList<String>();
+      for (SerializableField field : storedFields) {
+        if (field.name().equals("testStored") && field.stringValue() != null) {
+          fieldValues.add(field.stringValue());
+        }
+      }
+      values = fieldValues.toArray(new String[fieldValues.size()]);
+
+      assertNotNull(values);
+      assertEquals(1, values.length);
+      assertTrue("stored".equals(values[0]));
+
     } catch (BrowseException e) {
       e.printStackTrace();
       fail(e.getMessage());
@@ -827,12 +854,20 @@ public void testStoredField() throws Exception {
       BrowseHit hit = result.getHits()[0];
       assertNull(hit.getStoredFields());
 
-      br.setFetchStoredFields(true);
+      br.setFieldsToFetch(Collections.singleton("testStored"));
       result = boboBrowser.browse(br);
       assertEquals(1, result.getNumHits());
       hit = result.getHits()[0];
       String stored = hit.getFieldStringValue("testStored");
       assertTrue("stored".equals(stored));
+
+      br.setFetchAllFields(true);
+      br.setFieldsToFetch(null);
+      result = boboBrowser.browse(br);
+      assertEquals(1, result.getNumHits());
+      hit = result.getHits()[0];
+      stored = hit.getFieldStringValue("testStored");
+      assertTrue("stored".equals(stored));
     } catch (BrowseException e) {
       e.printStackTrace();
       fail(e.getMessage());
@@ -881,7 +916,7 @@ public void testRetrieveTermVector() throws Exception {
       BrowseHit hit = result.getHits()[0];
       assertNull(hit.getStoredFields());
 
-      br.setFetchStoredFields(true);
+      br.setFieldsToFetch(Collections.<String>emptySet());
       result = boboBrowser.browse(br);
       assertEquals(1, result.getNumHits());
       hit = result.getHits()[0];