LUCENE-7671 - Enhance UpgradeIndexMergePolicy with additional options

lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java

@@ -260,7 +260,7 @@ public void testCreateIndexWithDocValuesUpdates() throws Exception {
         writer.commit(); // flush every 10 docs
       }
     }
-
+      
     // first segment: no updates
 
     // second segment: update two fields, same gen
@@ -435,6 +435,8 @@ public void testCreateIndexWithDocValuesUpdates() throws Exception {
 
   // TODO: on 6.0.0 release, gen the single segment indices and add here:
   final static String[] oldSingleSegmentNames = {
+    "6.0.0.singlesegment-cfs",
+    "6.0.0.singlesegment-nocfs"
   };
 
   static Map<String,Directory> oldIndexDirs;
@@ -462,8 +464,9 @@ public static void beforeClass() throws Exception {
     oldIndexDirs = new HashMap<>();
     for (String name : names) {
       Path dir = createTempDir(name);
-      InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream("index." + name + ".zip");
-      assertNotNull("Index name " + name + " not found", resource);
+      String nameOnDisk = "index." + name + ".zip";
+      InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(nameOnDisk);
+      assertNotNull("Index name " + nameOnDisk + " not found", resource);
       TestUtil.unzip(resource, dir);
       oldIndexDirs.put(name, newFSDirectory(dir));
     }
@@ -1315,14 +1318,19 @@ public void testNumericFields() throws Exception {
   }
 
   private int checkAllSegmentsUpgraded(Directory dir, Version indexCreatedVersion) throws IOException {
+    return this.checkAllSegmentsUpgraded(dir, Version.LATEST, indexCreatedVersion);
+  }
+
+  private int checkAllSegmentsUpgraded(Directory dir, Version upgradedVersion, Version indexCreatedVersion) throws IOException {
     final SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
     if (VERBOSE) {
-      System.out.println("checkAllSegmentsUpgraded: " + infos);
+      System.out.println("checkAllSegmentsUpgraded: " + indexCreatedVersion + "-" + infos);
     }
     for (SegmentCommitInfo si : infos) {
-      assertEquals(Version.LATEST, si.info.getVersion());
+      assertEquals(upgradedVersion, si.info.getVersion());
     }
-    assertEquals(Version.LATEST, infos.getCommitLuceneVersion());
+
+    assertEquals(upgradedVersion, infos.getCommitLuceneVersion());
     assertEquals(indexCreatedVersion, infos.getIndexCreatedVersion());
     return infos.size();
   }
@@ -1342,14 +1350,64 @@ public void testUpgradeOldIndex() throws Exception {
       }
       Directory dir = newDirectory(oldIndexDirs.get(name));
       Version indexCreatedVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersion();
+
+      int numSegmentsBefore = SegmentInfos.readLatestCommit(dir).size();
+
+      newIndexUpgrader(dir).upgrade(Integer.MAX_VALUE);
 
-      newIndexUpgrader(dir).upgrade();
+      assertEquals(numSegmentsBefore, checkAllSegmentsUpgraded(dir, indexCreatedVersion));
+
+      dir.close();
+    }
+  }
+
+  public void testUpgradeWithExcplicitUpgrades() throws Exception {
+    List<String> names = new ArrayList<>(oldNames.length + oldSingleSegmentNames.length);
+    names.addAll(Arrays.asList(oldNames));
+    names.addAll(Arrays.asList(oldSingleSegmentNames));
+    for(String name : names) {
+      if (VERBOSE) {
+        System.out.println("testUpgradeWithExcplicitUpgrades: index=" +name);
+      }
+      Directory dir = newDirectory(oldIndexDirs.get(name));
+
+      SegmentInfos infosBefore = SegmentInfos.readLatestCommit(dir);
+      int numSegmentsBefore = infosBefore.size();
+      Version versionBefore = infosBefore.getCommitLuceneVersion();
+      Version createdVersionBefore = infosBefore.getIndexCreatedVersion();
+
+      assertFalse("Excpected these segments to be an old version", versionBefore.equals(Version.LATEST));
+
+      UpgradeIndexMergePolicy uimp = new UpgradeIndexMergePolicy(NoMergePolicy.INSTANCE);
+
+      uimp.setRequireExplicitUpgrades(true);
+      uimp.setIgnoreNewSegments(true);
+
+      assertEquals(numSegmentsBefore, checkAllSegmentsUpgraded(dir, versionBefore, createdVersionBefore));
+
+      try (IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(uimp))) {
+        w.forceMerge(numSegmentsBefore); // Don't optimize just upgrade
+      }
+
+      // Upgrade should not have happened yet
+      assertEquals(numSegmentsBefore, checkAllSegmentsUpgraded(dir, versionBefore, createdVersionBefore));
+
+      uimp.setUpgradeInProgress(true); // Turn on upgrades
+
+      try (IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(uimp))) {
+        w.forceMerge(numSegmentsBefore); // Don't optimize just upgrade
+      }
 
-      checkAllSegmentsUpgraded(dir, indexCreatedVersion);
+      // Upgrade should now have happened.
+      Version indexCreatedVersionAfter = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersion();
 
+      assertEquals(numSegmentsBefore, checkAllSegmentsUpgraded(dir, indexCreatedVersionAfter));
+
       dir.close();
     }
   }
+
+  // Write a test that checks that the underlying policy gets delegated to??
 
   public void testCommandLineArgs() throws Exception {
 

lucene/core/src/java/org/apache/lucene/index/IndexUpgrader.java

@@ -147,9 +147,13 @@ public IndexUpgrader(Directory dir, IndexWriterConfig iwc, boolean deletePriorCo
     this.iwc = iwc;
     this.deletePriorCommits = deletePriorCommits;
   }
+
+  public void upgrade() throws IOException {
+    this.upgrade(1);
+  }
 
   /** Perform the upgrade. */
-  public void upgrade() throws IOException {
+  public void upgrade(int maxSegements) throws IOException {
     if (!DirectoryReader.indexExists(dir)) {
       throw new IndexNotFoundException(dir.toString());
     }
@@ -161,15 +165,17 @@ public void upgrade() throws IOException {
       }
     }
 
-    iwc.setMergePolicy(new UpgradeIndexMergePolicy(iwc.getMergePolicy()));
+    UpgradeIndexMergePolicy uimp = new UpgradeIndexMergePolicy(iwc.getMergePolicy());
+    uimp.setIgnoreNewSegments(true);
+    iwc.setMergePolicy(uimp);
     iwc.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
 
     try (final IndexWriter w = new IndexWriter(dir, iwc)) {
       InfoStream infoStream = iwc.getInfoStream();
       if (infoStream.isEnabled(LOG_PREFIX)) {
         infoStream.message(LOG_PREFIX, "Upgrading all pre-" + Version.LATEST + " segments of index directory '" + dir + "' to version " + Version.LATEST + "...");
       }
-      w.forceMerge(1);
+      w.forceMerge(maxSegements);
       if (infoStream.isEnabled(LOG_PREFIX)) {
         infoStream.message(LOG_PREFIX, "All segments upgraded to version " + Version.LATEST);
         infoStream.message(LOG_PREFIX, "Enforcing commit to rewrite all index metadata...");