From b98349fd0cadf2f0c17dad9cbcf9fea312ec2684 Mon Sep 17 00:00:00 2001 From: "Brian S. O'Neill" Date: Wed, 30 Apr 2008 18:08:10 +0000 Subject: Recover from potential deadlocks against repositories with coarse locks. --- .../com/amazon/carbonado/layout/LayoutFactory.java | 35 ++++++- .../carbonado/repo/indexed/IndexedStorage.java | 115 ++++++++++++++++----- .../carbonado/repo/indexed/ManagedIndex.java | 19 +++- .../carbonado/repo/sleepycat/BDBStorage.java | 65 ++++++++++-- 4 files changed, 198 insertions(+), 36 deletions(-) (limited to 'src/main/java/com/amazon') diff --git a/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java b/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java index d7bf3da..1b2f0fc 100644 --- a/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java +++ b/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java @@ -29,10 +29,14 @@ import java.util.Map; import org.cojen.util.SoftValuedHashMap; import com.amazon.carbonado.Cursor; +import com.amazon.carbonado.FetchDeadlockException; import com.amazon.carbonado.FetchException; import com.amazon.carbonado.FetchNoneException; +import com.amazon.carbonado.FetchTimeoutException; import com.amazon.carbonado.IsolationLevel; +import com.amazon.carbonado.PersistDeadlockException; import com.amazon.carbonado.PersistException; +import com.amazon.carbonado.PersistTimeoutException; import com.amazon.carbonado.Repository; import com.amazon.carbonado.RepositoryException; import com.amazon.carbonado.Storable; @@ -98,9 +102,16 @@ public class LayoutFactory implements LayoutCapability { ResyncCapability resyncCap = null; // Try to insert metadata up to three times. + boolean top = true; loadLayout: for (int retryCount = 3;;) { try { - Transaction txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED); + Transaction txn; + if (top) { + txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED); + } else { + txn = mRepository.enterTransaction(IsolationLevel.READ_COMMITTED); + } + txn.setForUpdate(true); try { // If type represents a new generation, then a new layout needs to @@ -185,6 +196,28 @@ public class LayoutFactory implements LayoutCapability { // before each retry. retryCount = e.backoff(e, retryCount, 1000); resyncCap = mRepository.getCapability(ResyncCapability.class); + } catch (FetchException e) { + if (e instanceof FetchDeadlockException || e instanceof FetchTimeoutException) { + // Might be caused by coarse locks. Switch to nested + // transaction to share the locks. + if (top) { + top = false; + retryCount = e.backoff(e, retryCount, 100); + continue; + } + } + throw e; + } catch (PersistException e) { + if (e instanceof PersistDeadlockException || e instanceof PersistTimeoutException){ + // Might be caused by coarse locks. Switch to nested + // transaction to share the locks. + if (top) { + top = false; + retryCount = e.backoff(e, retryCount, 100); + continue; + } + } + throw e; } } diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java index dbd6e4f..cefb031 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java @@ -29,9 +29,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import com.amazon.carbonado.Cursor; +import com.amazon.carbonado.FetchDeadlockException; import com.amazon.carbonado.FetchException; +import com.amazon.carbonado.FetchTimeoutException; import com.amazon.carbonado.IsolationLevel; +import com.amazon.carbonado.PersistDeadlockException; import com.amazon.carbonado.PersistException; +import com.amazon.carbonado.PersistTimeoutException; import com.amazon.carbonado.Query; import com.amazon.carbonado.RepositoryException; import com.amazon.carbonado.Storable; @@ -118,12 +122,28 @@ class IndexedStorage implements Storage, StorageAccess .with(getStorableType().getName() + '~' + '\uffff'); List storedInfos; - Transaction txn = repository.getWrappedRepository() - .enterTopTransaction(IsolationLevel.READ_COMMITTED); try { - storedInfos = query.fetch().toList(); - } finally { - txn.exit(); + Transaction txn = repository.getWrappedRepository() + .enterTopTransaction(IsolationLevel.READ_COMMITTED); + try { + storedInfos = query.fetch().toList(); + } finally { + txn.exit(); + } + } catch (FetchException e) { + if (e instanceof FetchDeadlockException || e instanceof FetchTimeoutException) { + // Might be caused by coarse locks. Switch to nested + // transaction to share the locks. + Transaction txn = repository.getWrappedRepository() + .enterTransaction(IsolationLevel.READ_COMMITTED); + try { + storedInfos = query.fetch().toList(); + } finally { + txn.exit(); + } + } else { + throw e; + } } for (StoredIndexInfo indexInfo : storedInfos) { @@ -460,34 +480,81 @@ class IndexedStorage implements Storage, StorageAccess .storageFor(StoredIndexInfo.class).prepare(); info.setIndexName(index.getNameDescriptor()); - Transaction txn = mRepository.getWrappedRepository() - .enterTopTransaction(IsolationLevel.READ_COMMITTED); try { - if (info.tryLoad()) { - // Index already exists and is registered. - return; + Transaction txn = mRepository.getWrappedRepository() + .enterTopTransaction(IsolationLevel.READ_COMMITTED); + try { + if (info.tryLoad()) { + // Index already exists and is registered. + return; + } + } finally { + txn.exit(); + } + } catch (FetchException e) { + if (e instanceof FetchDeadlockException || e instanceof FetchTimeoutException) { + // Might be caused by coarse locks. Switch to nested + // transaction to share the locks. + Transaction txn = mRepository.getWrappedRepository() + .enterTransaction(IsolationLevel.READ_COMMITTED); + try { + if (info.tryLoad()) { + // Index already exists and is registered. + return; + } + } finally { + txn.exit(); + } + } else { + throw e; } - } finally { - txn.exit(); } // New index, so populate it. managedIndex.populateIndex(mRepository, mMasterStorage, mRepository.getIndexRepairThrottle()); - txn = mRepository.getWrappedRepository() - .enterTopTransaction(IsolationLevel.READ_COMMITTED); - txn.setForUpdate(true); - try { - if (!info.tryLoad()) { - info.setIndexTypeDescriptor(index.getTypeDescriptor()); - info.setCreationTimestamp(System.currentTimeMillis()); - info.setVersionNumber(0); - info.insert(); - txn.commit(); + boolean top = true; + while (true) { + try { + Transaction txn; + if (top) { + txn = mRepository.getWrappedRepository() + .enterTopTransaction(IsolationLevel.READ_COMMITTED); + } else { + txn = mRepository.getWrappedRepository() + .enterTransaction(IsolationLevel.READ_COMMITTED); + } + + txn.setForUpdate(true); + try { + if (!info.tryLoad()) { + info.setIndexTypeDescriptor(index.getTypeDescriptor()); + info.setCreationTimestamp(System.currentTimeMillis()); + info.setVersionNumber(0); + info.insert(); + txn.commit(); + } + } finally { + txn.exit(); + } + + break; + } catch (RepositoryException e) { + if (e instanceof FetchDeadlockException || + e instanceof FetchTimeoutException || + e instanceof PersistDeadlockException || + e instanceof PersistTimeoutException) + { + // Might be caused by coarse locks. Switch to nested + // transaction to share the locks. + if (top) { + top = false; + continue; + } + } + throw e; } - } finally { - txn.exit(); } } diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java b/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java index 507a09f..ea3bbb3 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java @@ -242,17 +242,26 @@ class ManagedIndex implements IndexEntryAccessor { masterQuery = masterStorage.query().orderBy(naturalOrdering); } - // Enter top transaction with isolation level of none to make sure - // preload operation does not run in a long nested transaction. - Transaction txn = repo.enterTopTransaction(IsolationLevel.NONE); - try { - Cursor cursor = masterQuery.fetch(); + // Quick check to see if any records exist in master. + { + Transaction txn = repo.enterTransaction(IsolationLevel.READ_COMMITTED); try { + Cursor cursor = masterQuery.fetch(); if (!cursor.hasNext()) { // Nothing exists in master, so nothing to populate. return; } + } finally { + txn.exit(); + } + } + // Enter top transaction with isolation level of none to make sure + // preload operation does not run in a long nested transaction. + Transaction txn = repo.enterTopTransaction(IsolationLevel.NONE); + try { + Cursor cursor = masterQuery.fetch(); + try { if (log.isInfoEnabled()) { StringBuilder b = new StringBuilder(); b.append("Populating index on "); diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java index 6c4d4e0..7332a61 100644 --- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java +++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java @@ -27,9 +27,13 @@ import org.apache.commons.logging.LogFactory; import org.cojen.classfile.TypeDesc; import com.amazon.carbonado.Cursor; +import com.amazon.carbonado.FetchDeadlockException; import com.amazon.carbonado.FetchException; +import com.amazon.carbonado.FetchTimeoutException; import com.amazon.carbonado.IsolationLevel; +import com.amazon.carbonado.PersistDeadlockException; import com.amazon.carbonado.PersistException; +import com.amazon.carbonado.PersistTimeoutException; import com.amazon.carbonado.Query; import com.amazon.carbonado.Repository; import com.amazon.carbonado.RepositoryException; @@ -491,12 +495,32 @@ abstract class BDBStorage implements Storage, Storag if (!readOnly) { Repository repo = mRepository.getRootRepository(); - Transaction txn = repo.enterTopTransaction(IsolationLevel.READ_COMMITTED); try { - primaryInfo.update(); - txn.commit(); - } finally { - txn.exit(); + Transaction txn = + repo.enterTopTransaction(IsolationLevel.READ_COMMITTED); + try { + primaryInfo.update(); + txn.commit(); + } finally { + txn.exit(); + } + } catch (PersistException e) { + if (e instanceof PersistDeadlockException || + e instanceof PersistTimeoutException) + { + // Might be caused by coarse locks. Switch to + // nested transaction to share the locks. + Transaction txn = + repo.enterTransaction(IsolationLevel.READ_COMMITTED); + try { + primaryInfo.update(); + txn.commit(); + } finally { + txn.exit(); + } + } else { + throw e; + } } } } @@ -788,9 +812,16 @@ abstract class BDBStorage implements Storage, Storag info.setDatabaseName(getStorableType().getName()); // Try to insert metadata up to three times. + boolean top = true; for (int retryCount = 3;;) { try { - Transaction txn = repo.enterTopTransaction(IsolationLevel.READ_COMMITTED); + Transaction txn; + if (top) { + txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED); + } else { + txn = mRepository.enterTransaction(IsolationLevel.READ_COMMITTED); + } + txn.setForUpdate(true); try { if (!info.tryLoad()) { @@ -815,6 +846,28 @@ abstract class BDBStorage implements Storage, Storag // a few times before throwing exception. Wait up to a second // before each retry. retryCount = e.backoff(e, retryCount, 1000); + } catch (FetchException e) { + if (e instanceof FetchDeadlockException || e instanceof FetchTimeoutException) { + // Might be caused by coarse locks. Switch to nested + // transaction to share the locks. + if (top) { + top = false; + retryCount = e.backoff(e, retryCount, 100); + continue; + } + } + throw e; + } catch (PersistException e) { + if (e instanceof PersistDeadlockException || e instanceof PersistTimeoutException){ + // Might be caused by coarse locks. Switch to nested + // transaction to share the locks. + if (top) { + top = false; + retryCount = e.backoff(e, retryCount, 100); + continue; + } + } + throw e; } } -- cgit v1.2.3