From 36f24e909e0c1bec928c04e41c838e91da6d61bf Mon Sep 17 00:00:00 2001
From: "Brian S. O'Neill" <bronee@gmail.com>
Date: Sun, 9 Sep 2012 15:43:20 +0000
Subject: Index repair improvements.

---
 .../carbonado/repo/indexed/ManagedIndex.java       | 71 ++++++++++++++--------
 1 file changed, 46 insertions(+), 25 deletions(-)

(limited to 'src/main/java/com/amazon/carbonado')

diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java b/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java
index 701aa37..60a21d5 100644
--- a/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java
+++ b/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java
@@ -69,7 +69,7 @@ import com.amazon.carbonado.util.Throttle;
 class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
     private static final int BUILD_SORT_BUFFER_SIZE = 65536;
     private static final int BUILD_INFO_DELAY_MILLIS = 5000;
-    static final int BUILD_BATCH_SIZE = 128;
+    static final int BUILD_BATCH_SIZE = 1000;
     static final int BUILD_THROTTLE_WINDOW = BUILD_BATCH_SIZE * 10;
     static final int BUILD_THROTTLE_SLEEP_PRECISION = 10;
 
@@ -458,6 +458,8 @@ class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
             Storable indexEntry = null;
             Storable lastIndexEntry = null;
 
+            long nextReportTime = System.currentTimeMillis() + BUILD_INFO_DELAY_MILLIS;
+
             Iterator it = buffer.iterator();
             bufferIterate: while (true) {
                 if (!retry) {
@@ -518,28 +520,42 @@ class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
                             // Existing index entry is ahead so check later.
                             break;
                         } else {
-                            // Existing index entry is bogus.
-                            existingIndexEntry.tryDelete();
-
-                            totalDeleted++;
+                            // Existing index entry might be bogus. Check again
+                            // in case master record changed.
+                            doDelete: {
+                                S master = mMasterStorage.prepare();
+                                copyToMasterPrimaryKey(existingIndexEntry, master);
+                                if (master.tryLoad()) {
+                                    Storable temp = makeIndexEntry(master);
+                                    existingIndexEntry.copyVersionProperty(temp);
+                                    if (existingIndexEntry.equalProperties(temp)) {
+                                        break doDelete;
+                                    }
+                                }
 
-                            if (totalDeleted % BUILD_BATCH_SIZE == 0) {
-                                txn.commit();
-                                txn.exit();
+                                existingIndexEntry.tryDelete();
+                                totalDeleted++;
 
-                                logProgress(log, totalProgress, bufferSize,
-                                            totalInserted, totalUpdated, totalDeleted);
+                                if (totalDeleted % BUILD_BATCH_SIZE == 0) {
+                                    txn.commit();
+                                    txn.exit();
 
-                                txn = enterBuildTxn();
+                                    nextReportTime = logProgress
+                                        (nextReportTime, log, totalProgress, bufferSize,
+                                         totalInserted, totalUpdated, totalDeleted);
 
-                                indexEntryCursor.close();
-                                indexEntryCursor = indexEntryQuery.fetchAfter(existingIndexEntry);
+                                    txn = enterBuildTxn();
 
-                                if (!indexEntryCursor.hasNext()) {
                                     indexEntryCursor.close();
-                                    // Don't try opening again.
-                                    indexEntryCursor = null;
-                                    break;
+                                    indexEntryCursor = indexEntryQuery
+                                        .fetchAfter(existingIndexEntry);
+
+                                    if (!indexEntryCursor.hasNext()) {
+                                        indexEntryCursor.close();
+                                        // Don't try opening again.
+                                        indexEntryCursor = null;
+                                        break;
+                                    }
                                 }
                             }
 
@@ -573,8 +589,8 @@ class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
                     txn.commit();
                     txn.exit();
 
-                    logProgress(log, totalProgress, bufferSize,
-                                totalInserted, totalUpdated, totalDeleted);
+                    nextReportTime = logProgress(nextReportTime, log, totalProgress, bufferSize,
+                                                 totalInserted, totalUpdated, totalDeleted);
 
                     txn = enterBuildTxn();
 
@@ -627,16 +643,21 @@ class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
         }
     }
 
-    private void logProgress(Log log,
+    private long logProgress(long nextReportTime, Log log,
                              long totalProgress, int bufferSize,
                              long totalInserted, long totalUpdated, long totalDeleted)
     {
-        if (log.isInfoEnabled()) {
-            String format = "Index build progress: %.3f%% " +
-                progressSubMessgage(totalInserted, totalUpdated, totalDeleted);
-            double percent = 100.0 * totalProgress / bufferSize;
-            log.info(String.format(format, percent));
+        long now = System.currentTimeMillis();
+        if (now >= nextReportTime) {
+            if (log.isInfoEnabled()) {
+                String format = "Index build progress: %.3f%% " +
+                    progressSubMessgage(totalInserted, totalUpdated, totalDeleted);
+                double percent = 100.0 * totalProgress / bufferSize;
+                log.info(String.format(format, percent));
+            }
+            nextReportTime = now + BUILD_INFO_DELAY_MILLIS;
         }
+        return nextReportTime;
     }
 
     private String progressSubMessgage(long totalInserted, long totalUpdated, long totalDeleted) {
-- 
cgit v1.2.3