summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorBrian S. O'Neill <bronee@gmail.com>2008-04-30 18:08:10 +0000
committerBrian S. O'Neill <bronee@gmail.com>2008-04-30 18:08:10 +0000
commitb98349fd0cadf2f0c17dad9cbcf9fea312ec2684 (patch)
treee9803ecf207863a6fb3b27bada812c777fe09c27 /src
parent6404fde31389a510d82d456336b7933f968f2b81 (diff)
Recover from potential deadlocks against repositories with coarse locks.
Diffstat (limited to 'src')
-rw-r--r--src/main/java/com/amazon/carbonado/layout/LayoutFactory.java35
-rw-r--r--src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java115
-rw-r--r--src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java19
-rw-r--r--src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java65
4 files changed, 198 insertions, 36 deletions
diff --git a/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java b/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java
index d7bf3da..1b2f0fc 100644
--- a/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java
+++ b/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java
@@ -29,10 +29,14 @@ import java.util.Map;
import org.cojen.util.SoftValuedHashMap;
import com.amazon.carbonado.Cursor;
+import com.amazon.carbonado.FetchDeadlockException;
import com.amazon.carbonado.FetchException;
import com.amazon.carbonado.FetchNoneException;
+import com.amazon.carbonado.FetchTimeoutException;
import com.amazon.carbonado.IsolationLevel;
+import com.amazon.carbonado.PersistDeadlockException;
import com.amazon.carbonado.PersistException;
+import com.amazon.carbonado.PersistTimeoutException;
import com.amazon.carbonado.Repository;
import com.amazon.carbonado.RepositoryException;
import com.amazon.carbonado.Storable;
@@ -98,9 +102,16 @@ public class LayoutFactory implements LayoutCapability {
ResyncCapability resyncCap = null;
// Try to insert metadata up to three times.
+ boolean top = true;
loadLayout: for (int retryCount = 3;;) {
try {
- Transaction txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED);
+ Transaction txn;
+ if (top) {
+ txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED);
+ } else {
+ txn = mRepository.enterTransaction(IsolationLevel.READ_COMMITTED);
+ }
+
txn.setForUpdate(true);
try {
// If type represents a new generation, then a new layout needs to
@@ -185,6 +196,28 @@ public class LayoutFactory implements LayoutCapability {
// before each retry.
retryCount = e.backoff(e, retryCount, 1000);
resyncCap = mRepository.getCapability(ResyncCapability.class);
+ } catch (FetchException e) {
+ if (e instanceof FetchDeadlockException || e instanceof FetchTimeoutException) {
+ // Might be caused by coarse locks. Switch to nested
+ // transaction to share the locks.
+ if (top) {
+ top = false;
+ retryCount = e.backoff(e, retryCount, 100);
+ continue;
+ }
+ }
+ throw e;
+ } catch (PersistException e) {
+ if (e instanceof PersistDeadlockException || e instanceof PersistTimeoutException){
+ // Might be caused by coarse locks. Switch to nested
+ // transaction to share the locks.
+ if (top) {
+ top = false;
+ retryCount = e.backoff(e, retryCount, 100);
+ continue;
+ }
+ }
+ throw e;
}
}
diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java
index dbd6e4f..cefb031 100644
--- a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java
+++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java
@@ -29,9 +29,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.amazon.carbonado.Cursor;
+import com.amazon.carbonado.FetchDeadlockException;
import com.amazon.carbonado.FetchException;
+import com.amazon.carbonado.FetchTimeoutException;
import com.amazon.carbonado.IsolationLevel;
+import com.amazon.carbonado.PersistDeadlockException;
import com.amazon.carbonado.PersistException;
+import com.amazon.carbonado.PersistTimeoutException;
import com.amazon.carbonado.Query;
import com.amazon.carbonado.RepositoryException;
import com.amazon.carbonado.Storable;
@@ -118,12 +122,28 @@ class IndexedStorage<S extends Storable> implements Storage<S>, StorageAccess<S>
.with(getStorableType().getName() + '~' + '\uffff');
List<StoredIndexInfo> storedInfos;
- Transaction txn = repository.getWrappedRepository()
- .enterTopTransaction(IsolationLevel.READ_COMMITTED);
try {
- storedInfos = query.fetch().toList();
- } finally {
- txn.exit();
+ Transaction txn = repository.getWrappedRepository()
+ .enterTopTransaction(IsolationLevel.READ_COMMITTED);
+ try {
+ storedInfos = query.fetch().toList();
+ } finally {
+ txn.exit();
+ }
+ } catch (FetchException e) {
+ if (e instanceof FetchDeadlockException || e instanceof FetchTimeoutException) {
+ // Might be caused by coarse locks. Switch to nested
+ // transaction to share the locks.
+ Transaction txn = repository.getWrappedRepository()
+ .enterTransaction(IsolationLevel.READ_COMMITTED);
+ try {
+ storedInfos = query.fetch().toList();
+ } finally {
+ txn.exit();
+ }
+ } else {
+ throw e;
+ }
}
for (StoredIndexInfo indexInfo : storedInfos) {
@@ -460,34 +480,81 @@ class IndexedStorage<S extends Storable> implements Storage<S>, StorageAccess<S>
.storageFor(StoredIndexInfo.class).prepare();
info.setIndexName(index.getNameDescriptor());
- Transaction txn = mRepository.getWrappedRepository()
- .enterTopTransaction(IsolationLevel.READ_COMMITTED);
try {
- if (info.tryLoad()) {
- // Index already exists and is registered.
- return;
+ Transaction txn = mRepository.getWrappedRepository()
+ .enterTopTransaction(IsolationLevel.READ_COMMITTED);
+ try {
+ if (info.tryLoad()) {
+ // Index already exists and is registered.
+ return;
+ }
+ } finally {
+ txn.exit();
+ }
+ } catch (FetchException e) {
+ if (e instanceof FetchDeadlockException || e instanceof FetchTimeoutException) {
+ // Might be caused by coarse locks. Switch to nested
+ // transaction to share the locks.
+ Transaction txn = mRepository.getWrappedRepository()
+ .enterTransaction(IsolationLevel.READ_COMMITTED);
+ try {
+ if (info.tryLoad()) {
+ // Index already exists and is registered.
+ return;
+ }
+ } finally {
+ txn.exit();
+ }
+ } else {
+ throw e;
}
- } finally {
- txn.exit();
}
// New index, so populate it.
managedIndex.populateIndex(mRepository, mMasterStorage,
mRepository.getIndexRepairThrottle());
- txn = mRepository.getWrappedRepository()
- .enterTopTransaction(IsolationLevel.READ_COMMITTED);
- txn.setForUpdate(true);
- try {
- if (!info.tryLoad()) {
- info.setIndexTypeDescriptor(index.getTypeDescriptor());
- info.setCreationTimestamp(System.currentTimeMillis());
- info.setVersionNumber(0);
- info.insert();
- txn.commit();
+ boolean top = true;
+ while (true) {
+ try {
+ Transaction txn;
+ if (top) {
+ txn = mRepository.getWrappedRepository()
+ .enterTopTransaction(IsolationLevel.READ_COMMITTED);
+ } else {
+ txn = mRepository.getWrappedRepository()
+ .enterTransaction(IsolationLevel.READ_COMMITTED);
+ }
+
+ txn.setForUpdate(true);
+ try {
+ if (!info.tryLoad()) {
+ info.setIndexTypeDescriptor(index.getTypeDescriptor());
+ info.setCreationTimestamp(System.currentTimeMillis());
+ info.setVersionNumber(0);
+ info.insert();
+ txn.commit();
+ }
+ } finally {
+ txn.exit();
+ }
+
+ break;
+ } catch (RepositoryException e) {
+ if (e instanceof FetchDeadlockException ||
+ e instanceof FetchTimeoutException ||
+ e instanceof PersistDeadlockException ||
+ e instanceof PersistTimeoutException)
+ {
+ // Might be caused by coarse locks. Switch to nested
+ // transaction to share the locks.
+ if (top) {
+ top = false;
+ continue;
+ }
+ }
+ throw e;
}
- } finally {
- txn.exit();
}
}
diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java b/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java
index 507a09f..ea3bbb3 100644
--- a/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java
+++ b/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java
@@ -242,17 +242,26 @@ class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
masterQuery = masterStorage.query().orderBy(naturalOrdering);
}
- // Enter top transaction with isolation level of none to make sure
- // preload operation does not run in a long nested transaction.
- Transaction txn = repo.enterTopTransaction(IsolationLevel.NONE);
- try {
- Cursor<S> cursor = masterQuery.fetch();
+ // Quick check to see if any records exist in master.
+ {
+ Transaction txn = repo.enterTransaction(IsolationLevel.READ_COMMITTED);
try {
+ Cursor<S> cursor = masterQuery.fetch();
if (!cursor.hasNext()) {
// Nothing exists in master, so nothing to populate.
return;
}
+ } finally {
+ txn.exit();
+ }
+ }
+ // Enter top transaction with isolation level of none to make sure
+ // preload operation does not run in a long nested transaction.
+ Transaction txn = repo.enterTopTransaction(IsolationLevel.NONE);
+ try {
+ Cursor<S> cursor = masterQuery.fetch();
+ try {
if (log.isInfoEnabled()) {
StringBuilder b = new StringBuilder();
b.append("Populating index on ");
diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java
index 6c4d4e0..7332a61 100644
--- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java
+++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java
@@ -27,9 +27,13 @@ import org.apache.commons.logging.LogFactory;
import org.cojen.classfile.TypeDesc;
import com.amazon.carbonado.Cursor;
+import com.amazon.carbonado.FetchDeadlockException;
import com.amazon.carbonado.FetchException;
+import com.amazon.carbonado.FetchTimeoutException;
import com.amazon.carbonado.IsolationLevel;
+import com.amazon.carbonado.PersistDeadlockException;
import com.amazon.carbonado.PersistException;
+import com.amazon.carbonado.PersistTimeoutException;
import com.amazon.carbonado.Query;
import com.amazon.carbonado.Repository;
import com.amazon.carbonado.RepositoryException;
@@ -491,12 +495,32 @@ abstract class BDBStorage<Txn, S extends Storable> implements Storage<S>, Storag
if (!readOnly) {
Repository repo = mRepository.getRootRepository();
- Transaction txn = repo.enterTopTransaction(IsolationLevel.READ_COMMITTED);
try {
- primaryInfo.update();
- txn.commit();
- } finally {
- txn.exit();
+ Transaction txn =
+ repo.enterTopTransaction(IsolationLevel.READ_COMMITTED);
+ try {
+ primaryInfo.update();
+ txn.commit();
+ } finally {
+ txn.exit();
+ }
+ } catch (PersistException e) {
+ if (e instanceof PersistDeadlockException ||
+ e instanceof PersistTimeoutException)
+ {
+ // Might be caused by coarse locks. Switch to
+ // nested transaction to share the locks.
+ Transaction txn =
+ repo.enterTransaction(IsolationLevel.READ_COMMITTED);
+ try {
+ primaryInfo.update();
+ txn.commit();
+ } finally {
+ txn.exit();
+ }
+ } else {
+ throw e;
+ }
}
}
}
@@ -788,9 +812,16 @@ abstract class BDBStorage<Txn, S extends Storable> implements Storage<S>, Storag
info.setDatabaseName(getStorableType().getName());
// Try to insert metadata up to three times.
+ boolean top = true;
for (int retryCount = 3;;) {
try {
- Transaction txn = repo.enterTopTransaction(IsolationLevel.READ_COMMITTED);
+ Transaction txn;
+ if (top) {
+ txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED);
+ } else {
+ txn = mRepository.enterTransaction(IsolationLevel.READ_COMMITTED);
+ }
+
txn.setForUpdate(true);
try {
if (!info.tryLoad()) {
@@ -815,6 +846,28 @@ abstract class BDBStorage<Txn, S extends Storable> implements Storage<S>, Storag
// a few times before throwing exception. Wait up to a second
// before each retry.
retryCount = e.backoff(e, retryCount, 1000);
+ } catch (FetchException e) {
+ if (e instanceof FetchDeadlockException || e instanceof FetchTimeoutException) {
+ // Might be caused by coarse locks. Switch to nested
+ // transaction to share the locks.
+ if (top) {
+ top = false;
+ retryCount = e.backoff(e, retryCount, 100);
+ continue;
+ }
+ }
+ throw e;
+ } catch (PersistException e) {
+ if (e instanceof PersistDeadlockException || e instanceof PersistTimeoutException){
+ // Might be caused by coarse locks. Switch to nested
+ // transaction to share the locks.
+ if (top) {
+ top = false;
+ retryCount = e.backoff(e, retryCount, 100);
+ continue;
+ }
+ }
+ throw e;
}
}