Cope with errors during resync.

2 files changed, 119 insertions, 75 deletions

diff --git a/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java b/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java
index 89e139d..6deb3ee 100644
--- a/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java
+++ b/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java
@@ -33,6 +33,9 @@ import com.amazon.carbonado.RepositoryException;
 import com.amazon.carbonado.Storable;

 import com.amazon.carbonado.Storage;

 import com.amazon.carbonado.Transaction;

+import com.amazon.carbonado.UniqueConstraintException;

+

+import com.amazon.carbonado.capability.ResyncCapability;

 

 import com.amazon.carbonado.info.StorableInfo;

 import com.amazon.carbonado.info.StorableIntrospector;

@@ -86,73 +89,102 @@ public class LayoutFactory implements LayoutCapability {
 

         StorableInfo<?> info = StorableIntrospector.examine(type);

 

-        Transaction txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED);

-        txn.setForUpdate(true);

-        try {

-            // If type represents a new generation, then a new layout needs to

-            // be inserted.

-            Layout newLayout = null;

-

-            for (int i=0; i<HASH_MULTIPLIERS.length; i++) {

-                // Generate an identifier which has a high likelyhood of being unique.

-                long layoutID = mixInHash(0L, info, HASH_MULTIPLIERS[i]);

-

-                // Initially use for comparison purposes.

-                newLayout = new Layout(this, info, layoutID);

+        Layout layout;

+        ResyncCapability resyncCap = null;

 

-                StoredLayout storedLayout = mLayoutStorage.prepare();

-                storedLayout.setLayoutID(layoutID);

-

-                if (!storedLayout.tryLoad()) {

-                    // Not found, so break out and insert.

-                    break;

-                }

-

-                Layout knownLayout = new Layout(this, storedLayout);

-                if (knownLayout.equalLayouts(newLayout)) {

-                    // Type does not represent a new generation. Return

-                    // existing layout.

-                    return knownLayout;

+        // Try to insert metadata up to three times.

+        loadLayout: for (int retryCount = 3;;) {

+            try {

+                Transaction txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED);

+                txn.setForUpdate(true);

+                try {

+                    // If type represents a new generation, then a new layout needs to

+                    // be inserted.

+                    Layout newLayout = null;

+

+                    for (int i=0; i<HASH_MULTIPLIERS.length; i++) {

+                        // Generate an identifier which has a high likelyhood of being unique.

+                        long layoutID = mixInHash(0L, info, HASH_MULTIPLIERS[i]);

+

+                        // Initially use for comparison purposes.

+                        newLayout = new Layout(this, info, layoutID);

+

+                        StoredLayout storedLayout = mLayoutStorage.prepare();

+                        storedLayout.setLayoutID(layoutID);

+

+                        if (!storedLayout.tryLoad()) {

+                            // Not found, so break out and insert.

+                            break;

+                        }

+

+                        Layout knownLayout = new Layout(this, storedLayout);

+                        if (knownLayout.equalLayouts(newLayout)) {

+                            // Type does not represent a new generation. Return

+                            // existing layout.

+                            layout = knownLayout;

+                            break loadLayout;

+                        }

+

+                        // If this point is reached, then there was a hash collision in

+                        // the generated layout ID. This should be extremely rare.

+                        // Rehash and try again.

+

+                        if (i >= HASH_MULTIPLIERS.length - 1) {

+                            // No more rehashes to attempt. This should be extremely,

+                            // extremely rare, unless there is a bug somewhere.

+                            throw new FetchException

+                                ("Unable to generate unique layout identifier");

+                        }

+                    }

+

+                    // If this point is reached, then type represents a new

+                    // generation. Calculate next generation value and insert.

+

+                    assert(newLayout != null);

+                    int generation = 0;

+

+                    Cursor<StoredLayout> cursor = mLayoutStorage

+                        .query("storableTypeName = ?")

+                        .with(info.getStorableType().getName())

+                        .orderBy("-generation")

+                        .fetch();

+

+                    try {

+                        if (cursor.hasNext()) {

+                            generation = cursor.next().getGeneration() + 1;

+                        }

+                    } finally {

+                        cursor.close();

+                    }

+

+                    newLayout.insert(generation);

+                    layout = newLayout;

+

+                    txn.commit();

+                } finally {

+                    txn.exit();

                 }

 

-                // If this point is reached, then there was a hash collision in

-                // the generated layout ID. This should be extremely rare.

-                // Rehash and try again.

-

-                if (i >= HASH_MULTIPLIERS.length - 1) {

-                    // No more rehashes to attempt. This should be extremely,

-                    // extremely rare, unless there is a bug somewhere.

-                    throw new FetchException("Unable to generate unique layout identifier");

-                }

+                break;

+            } catch (UniqueConstraintException e) {

+                // This might be caused by a transient replication error. Retry

+                // a few times before throwing exception. Wait up to a second

+                // before each retry.

+                retryCount = e.backoff(e, retryCount, 1000);

+                resyncCap = mRepository.getCapability(ResyncCapability.class);

             }

+        }

 

-            // If this point is reached, then type represents a new

-            // generation. Calculate next generation value and insert.

-

-            assert(newLayout != null);

-            int generation = 0;

-

-            Cursor<StoredLayout> cursor = mLayoutStorage

-                .query("storableTypeName = ?")

-                .with(info.getStorableType().getName())

-                .orderBy("-generation")

-                .fetch();

-

+        if (resyncCap != null) {

+            // Make sure that all layout records are sync'd.

             try {

-                if (cursor.hasNext()) {

-                    generation = cursor.next().getGeneration() + 1;

-                }

-            } finally {

-                cursor.close();

+                resyncCap.resync(StoredLayoutProperty.class, 1.0, null);

+            } catch (RepositoryException e) {

+                throw e.toPersistException();

             }

-

-            newLayout.insert(generation);

-            txn.commit();

-

-            return newLayout;

-        } finally {

-            txn.exit();

         }

+

+        return layout;

     }

 

     /**

diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java
index 331c7b0..d8a3066 100644
--- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java
+++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java
@@ -36,6 +36,7 @@ import com.amazon.carbonado.Storage;
 import com.amazon.carbonado.SupportException;

 import com.amazon.carbonado.Transaction;

 import com.amazon.carbonado.Trigger;

+import com.amazon.carbonado.UniqueConstraintException;

 

 import com.amazon.carbonado.capability.IndexInfo;

 

@@ -721,24 +722,35 @@ abstract class BDBStorage<Txn, S extends Storable> implements Storage<S>, Storag
         }

         info.setDatabaseName(getStorableType().getName());

 

-        Transaction txn = repo.enterTopTransaction(IsolationLevel.READ_COMMITTED);

-        txn.setForUpdate(true);

-        try {

-            if (!info.tryLoad()) {

-                if (layout == null) {

-                    info.setEvolutionStrategy(StoredDatabaseInfo.EVOLUTION_NONE);

-                } else {

-                    info.setEvolutionStrategy(StoredDatabaseInfo.EVOLUTION_STANDARD);

-                }

-                info.setCreationTimestamp(System.currentTimeMillis());

-                info.setVersionNumber(0);

-                if (!readOnly) {

-                    info.insert();

+        // Try to insert metadata up to three times.

+        for (int retryCount = 3;;) {

+            try {

+                Transaction txn = repo.enterTopTransaction(IsolationLevel.READ_COMMITTED);

+                txn.setForUpdate(true);

+                try {

+                    if (!info.tryLoad()) {

+                        if (layout == null) {

+                            info.setEvolutionStrategy(StoredDatabaseInfo.EVOLUTION_NONE);

+                        } else {

+                            info.setEvolutionStrategy(StoredDatabaseInfo.EVOLUTION_STANDARD);

+                        }

+                        info.setCreationTimestamp(System.currentTimeMillis());

+                        info.setVersionNumber(0);

+                        if (!readOnly) {

+                            info.insert();

+                        }

+                    }

+                    txn.commit();

+                } finally {

+                    txn.exit();

                 }

+                break;

+            } catch (UniqueConstraintException e) {

+                // This might be caused by a transient replication error. Retry

+                // a few times before throwing exception. Wait up to a second

+                // before each retry.

+                retryCount = e.backoff(e, retryCount, 1000);

             }

-            txn.commit();

-        } finally {

-            txn.exit();

         }

 

         return info;