From 6cee40de307fa88391573b94af60fc0942a99faa Mon Sep 17 00:00:00 2001
From: "Brian S. O'Neill" <bronee@gmail.com>
Date: Sun, 25 May 2008 19:02:37 +0000
Subject: Fixed cyclic dependency with indexes against complex derived
properties.
---
.../repo/indexed/DependentStorableFetcher.java | 38 ++-
.../repo/indexed/DerivedIndexesTrigger.java | 2 +-
.../carbonado/repo/indexed/IndexAnalysis.java | 262 ++++++++++++++++++++-
.../carbonado/repo/indexed/IndexAnalysisPool.java | 41 ++++
.../carbonado/repo/indexed/IndexedRepository.java | 48 +++-
.../carbonado/repo/indexed/IndexedStorage.java | 261 ++------------------
.../carbonado/repo/indexed/ManagedIndex.java | 30 +--
7 files changed, 386 insertions(+), 296 deletions(-)
create mode 100644 src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysisPool.java
(limited to 'src/main')
diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/DependentStorableFetcher.java b/src/main/java/com/amazon/carbonado/repo/indexed/DependentStorableFetcher.java
index 98454f1..d9e9f32 100644
--- a/src/main/java/com/amazon/carbonado/repo/indexed/DependentStorableFetcher.java
+++ b/src/main/java/com/amazon/carbonado/repo/indexed/DependentStorableFetcher.java
@@ -45,9 +45,11 @@ import com.amazon.carbonado.info.StorableProperty;
class DependentStorableFetcher<S extends Storable, D extends Storable> {
private final IndexedRepository mRepository;
private final IndexEntryAccessor<D>[] mIndexEntryAccessors;
- private final Query<D> mQuery;
+ private final Filter<D> mFilter;
private final String[] mJoinProperties;
+ private Query<D> mQuery;
+
/**
* @param derivedTo special chained property from StorableProperty.getDerivedToProperties
*/
@@ -81,7 +83,7 @@ class DependentStorableFetcher<S extends Storable, D extends Storable> {
if (accessorList.size() == 0) {
throw new SupportException
("Unable to find index accessors for derived-to property: " + derivedTo +
- ", enclosing type: " + dType);
+ ", enclosing type: " + dType.getName() + ", source type: " + sType.getName());
}
// Build a query on D joined to S.
@@ -100,18 +102,18 @@ class DependentStorableFetcher<S extends Storable, D extends Storable> {
Filter<D> dFilter = Filter.getOpenFilter(dType);
for (int i=0; i<joinElementCount; i++) {
- StorableProperty<S> element = join.getInternalJoinElement(i);
- joinProperties[i] = element.getName();
+ joinProperties[i] = join.getExternalJoinElement(i).getName();
+ StorableProperty<S> internal = join.getInternalJoinElement(i);
if (base == null) {
- dFilter = dFilter.and(element.getName(), RelOp.EQ);
+ dFilter = dFilter.and(internal.getName(), RelOp.EQ);
} else {
- dFilter = dFilter.and(base.append(element).toString(), RelOp.EQ);
+ dFilter = dFilter.and(base.append(internal).toString(), RelOp.EQ);
}
}
mRepository = repository;
mIndexEntryAccessors = accessorList.toArray(new IndexEntryAccessor[accessorList.size()]);
- mQuery = repository.storageFor(dType).query(dFilter);
+ mFilter = dFilter;
mJoinProperties = joinProperties;
}
@@ -120,7 +122,7 @@ class DependentStorableFetcher<S extends Storable, D extends Storable> {
}
public Cursor<D> fetchDependenentStorables(S storable) throws FetchException {
- Query<D> query = mQuery;
+ Query<D> query = query();
for (String property : mJoinProperties) {
query = query.with(storable.getPropertyValue(property));
}
@@ -144,7 +146,7 @@ class DependentStorableFetcher<S extends Storable, D extends Storable> {
@Override
public int hashCode() {
- return mQuery.getFilter().hashCode();
+ return mFilter.hashCode();
}
@Override
@@ -154,7 +156,7 @@ class DependentStorableFetcher<S extends Storable, D extends Storable> {
}
if (obj instanceof DependentStorableFetcher) {
DependentStorableFetcher other = (DependentStorableFetcher) obj;
- return mQuery.getFilter().equals(other.mQuery.getFilter())
+ return mFilter.equals(other.mFilter)
&& Arrays.equals(mJoinProperties, other.mJoinProperties)
&& Arrays.equals(mIndexEntryAccessors, other.mIndexEntryAccessors);
}
@@ -164,7 +166,21 @@ class DependentStorableFetcher<S extends Storable, D extends Storable> {
@Override
public String toString() {
return "DependentStorableFetcher: {indexes=" + Arrays.toString(mIndexEntryAccessors) +
- ", query=" + mQuery +
+ ", filter=" + mFilter +
", join properties=" + Arrays.toString(mJoinProperties) + '}';
}
+
+ private Query<D> query() throws FetchException {
+ // Query is lazily created to avoid stack overflow due to cyclic
+ // dependencies.
+ Query<D> query = mQuery;
+ if (query == null) {
+ try {
+ mQuery = query = mRepository.storageFor(mFilter.getStorableType()).query(mFilter);
+ } catch (RepositoryException e) {
+ throw e.toFetchException();
+ }
+ }
+ return query;
+ }
}
diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/DerivedIndexesTrigger.java b/src/main/java/com/amazon/carbonado/repo/indexed/DerivedIndexesTrigger.java
index 9e69935..2cc3d8f 100644
--- a/src/main/java/com/amazon/carbonado/repo/indexed/DerivedIndexesTrigger.java
+++ b/src/main/java/com/amazon/carbonado/repo/indexed/DerivedIndexesTrigger.java
@@ -50,7 +50,7 @@ class DerivedIndexesTrigger<S extends Storable, D extends Storable> extends Trig
this(new DependentStorableFetcher(repository, sType, derivedTo));
}
- DerivedIndexesTrigger(DependentStorableFetcher<S, D> fetcher) {
+ private DerivedIndexesTrigger(DependentStorableFetcher<S, D> fetcher) {
mFetcher = fetcher;
}
diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysis.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysis.java
index d65e32e..241f297 100644
--- a/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysis.java
+++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysis.java
@@ -20,9 +20,23 @@ package com.amazon.carbonado.repo.indexed;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
import java.util.Set;
+import com.amazon.carbonado.FetchException;
+import com.amazon.carbonado.FetchDeadlockException;
+import com.amazon.carbonado.FetchTimeoutException;
+import com.amazon.carbonado.IsolationLevel;
+import com.amazon.carbonado.Query;
+import com.amazon.carbonado.RepositoryException;
import com.amazon.carbonado.Storable;
+import com.amazon.carbonado.Storage;
+import com.amazon.carbonado.Transaction;
+
+import com.amazon.carbonado.capability.IndexInfo;
+import com.amazon.carbonado.capability.IndexInfoCapability;
import com.amazon.carbonado.filter.Filter;
import com.amazon.carbonado.filter.RelOp;
@@ -38,12 +52,245 @@ import com.amazon.carbonado.qe.FilteringScore;
import com.amazon.carbonado.qe.StorableIndexSet;
/**
- * Collection of static methods which perform index analysis.
+ * Builds various sets of indexes for a Storable type.
*
* @author Brian S O'Neill
* @since 1.2
*/
-class IndexAnalysis {
+class IndexAnalysis<S extends Storable> {
+
+ final IndexedRepository repository;
+ final Storage<S> masterStorage;
+
+ // The set of indexes that can actually be used for querying. If index
+ // repair is enabled, this set will be the same as desiredIndexSet.
+ // Otherwise, it will be the intersection of existingIndexSet and
+ // desiredIndexSet. In both cases, "free" indexes are added to the set too.
+ final StorableIndexSet<S> queryableIndexSet;
+
+ // The set of indexes that should be removed and no longer managed. If
+ // index repair is enabled, this set will be the existingIndexSet minus
+ // desiredIndexSet minus freeIndexSet plus bogusIndexSet. Otherwise, it
+ // will be empty.
+ final StorableIndexSet<S> removeIndexSet;
+
+ // The set of indexes that should be freshly populated. If index repair is
+ // enabled, this set will be the desiredIndexSet minus existingIndexSet
+ // minus freeIndexSet. Otherwise, it will be empty.
+ final StorableIndexSet<S> addIndexSet;
+
+ // Maps free and managed indexes to IndexInfo and ManagedIndex objects.
+ final Map<StorableIndex<S>, IndexInfo> allIndexInfoMap;
+
+ // Trigger which must be installed to keep managed indexes up to date. Is
+ // null if there are no managed indexes.
+ final IndexesTrigger<S> indexesTrigger;
+
+ // The set of derived-to properties in external storables that are used by
+ // indexes. Is null if none.
+ final Set<ChainedProperty<?>> derivedToDependencies;
+
+ public IndexAnalysis(IndexedRepository repository, Storage<S> masterStorage)
+ throws RepositoryException
+ {
+ this.repository = repository;
+ this.masterStorage = masterStorage;
+
+ StorableInfo<S> info = StorableIntrospector.examine(masterStorage.getStorableType());
+
+ // The set of indexes that the Storable defines, reduced.
+ final StorableIndexSet<S> desiredIndexSet;
+ {
+ desiredIndexSet = gatherDesiredIndexes(info);
+ desiredIndexSet.reduce(Direction.ASCENDING);
+ if (repository.isAllClustered()) {
+ desiredIndexSet.markClustered(true);
+ }
+ }
+
+ // The set of indexes that are populated and available for use. This is
+ // determined by examining index metadata. If the Storable has not
+ // changed, it will be the same as desiredIndexSet. If any existing
+ // indexes use a property whose type has changed, it is added to
+ // bogusIndexSet. Bogus indexes are removed if repair is enabled.
+ final StorableIndexSet<S> existingIndexSet;
+ final StorableIndexSet<S> bogusIndexSet;
+ {
+ existingIndexSet = new StorableIndexSet<S>();
+ bogusIndexSet = new StorableIndexSet<S>();
+
+ Query<StoredIndexInfo> query = repository.getWrappedRepository()
+ .storageFor(StoredIndexInfo.class)
+ // Primary key of StoredIndexInfo is an index descriptor, which
+ // starts with the storable type name. This emulates a
+ // "wildcard at the end" search.
+ .query("indexName >= ? & indexName < ?")
+ .with(info.getStorableType().getName() + '~')
+ .with(info.getStorableType().getName() + '~' + '\uffff');
+
+ List<StoredIndexInfo> storedInfos;
+ try {
+ Transaction txn = repository.getWrappedRepository()
+ .enterTopTransaction(IsolationLevel.READ_COMMITTED);
+ try {
+ storedInfos = query.fetch().toList();
+ } finally {
+ txn.exit();
+ }
+ } catch (FetchException e) {
+ if (e instanceof FetchDeadlockException || e instanceof FetchTimeoutException) {
+ // Might be caused by coarse locks. Switch to nested
+ // transaction to share the locks.
+ Transaction txn = repository.getWrappedRepository()
+ .enterTransaction(IsolationLevel.READ_COMMITTED);
+ try {
+ storedInfos = query.fetch().toList();
+ } finally {
+ txn.exit();
+ }
+ } else {
+ throw e;
+ }
+ }
+
+ for (StoredIndexInfo indexInfo : storedInfos) {
+ String name = indexInfo.getIndexName();
+ StorableIndex index;
+ try {
+ index = StorableIndex.parseNameDescriptor(name, info);
+ } catch (IllegalArgumentException e) {
+ // Skip unrecognized descriptors.
+ continue;
+ }
+ if (index.getTypeDescriptor().equals(indexInfo.getIndexTypeDescriptor())) {
+ existingIndexSet.add(index);
+ } else {
+ bogusIndexSet.add(index);
+ }
+ }
+ }
+
+ nonUniqueSearch: {
+ // If any existing indexes are non-unique, then indexes are for an
+ // older version. For compatibility, don't uniquify the
+ // indexes. Otherwise, these indexes would need to be rebuilt.
+ for (StorableIndex<S> index : existingIndexSet) {
+ if (!index.isUnique()) {
+ break nonUniqueSearch;
+ }
+ }
+
+ // The index implementation includes all primary key properties
+ // anyhow, so adding them here allows query analyzer to see these
+ // properties. As a side-effect of uniquify, all indexes are
+ // unique, and thus have 'U' in the descriptor. Each time
+ // nonUniqueSearch is run, it will not find any non-unique indexes.
+ desiredIndexSet.uniquify(info);
+ }
+
+ // The set of free indexes, which are already provided by the underlying
+ // storage. They can be used for querying, but we should not manage them.
+ final StorableIndexSet<S> freeIndexSet;
+ {
+ freeIndexSet = new StorableIndexSet<S>();
+ allIndexInfoMap = new IdentityHashMap<StorableIndex<S>, IndexInfo>();
+
+ IndexInfoCapability cap = repository.getWrappedRepository()
+ .getCapability(IndexInfoCapability.class);
+
+ if (cap != null) {
+ for (IndexInfo ii : cap.getIndexInfo(info.getStorableType())) {
+ StorableIndex<S> freeIndex;
+ try {
+ freeIndex = new StorableIndex<S>(info.getStorableType(), ii);
+ } catch (IllegalArgumentException e) {
+ // Assume index is malformed, so ignore it.
+ continue;
+ }
+ if (repository.isAllClustered()) {
+ freeIndex = freeIndex.clustered(true);
+ }
+ freeIndexSet.add(freeIndex);
+ allIndexInfoMap.put(freeIndex, ii);
+ }
+ }
+ }
+
+ {
+ queryableIndexSet = new StorableIndexSet<S>(desiredIndexSet);
+
+ if (!repository.isIndexRepairEnabled()) {
+ // Can only query the intersection.
+ queryableIndexSet.retainAll(existingIndexSet);
+ }
+
+ // Add the indexes we get for free.
+ queryableIndexSet.addAll(freeIndexSet);
+ }
+
+ // The set of indexes that should be kept up-to-date. If index repair
+ // is enabled, this set will be the same as desiredIndexSet. Otherwise,
+ // it will be the union of existingIndexSet and desiredIndexSet. In
+ // both cases, "free" indexes are removed from the set too. By doing a
+ // union, no harm is caused by changing the index set and then
+ // reverting.
+ final StorableIndexSet<S> managedIndexSet;
+ {
+ managedIndexSet = new StorableIndexSet<S>(desiredIndexSet);
+
+ if (repository.isIndexRepairEnabled()) {
+ // Must manage the union.
+ managedIndexSet.addAll(existingIndexSet);
+ }
+
+ // Remove the indexes we get for free.
+ managedIndexSet.removeAll(freeIndexSet);
+ }
+
+ {
+ removeIndexSet = new StorableIndexSet<S>();
+
+ if (repository.isIndexRepairEnabled()) {
+ removeIndexSet.addAll(existingIndexSet);
+ removeIndexSet.removeAll(desiredIndexSet);
+ removeIndexSet.removeAll(freeIndexSet);
+ removeIndexSet.addAll(bogusIndexSet);
+ }
+ }
+
+ {
+ addIndexSet = new StorableIndexSet<S>();
+
+ if (repository.isIndexRepairEnabled()) {
+ addIndexSet.addAll(desiredIndexSet);
+ addIndexSet.removeAll(existingIndexSet);
+ addIndexSet.removeAll(freeIndexSet);
+ }
+ }
+
+ // Support for managed indexes...
+ if (managedIndexSet.size() <= 0) {
+ indexesTrigger = null;
+ } else {
+ ManagedIndex<S>[] managedIndexes = new ManagedIndex[managedIndexSet.size()];
+ int i = 0;
+ for (StorableIndex<S> index : managedIndexSet) {
+ IndexEntryGenerator<S> builder = IndexEntryGenerator.getInstance(index);
+ Class<? extends Storable> indexEntryClass = builder.getIndexEntryClass();
+ Storage<?> indexEntryStorage = repository.getIndexEntryStorageFor(indexEntryClass);
+ ManagedIndex managedIndex = new ManagedIndex<S>
+ (repository, masterStorage, index, builder, indexEntryStorage);
+
+ allIndexInfoMap.put(index, managedIndex);
+ managedIndexes[i++] = managedIndex;
+ }
+
+ indexesTrigger = new IndexesTrigger<S>(managedIndexes);
+ }
+
+ derivedToDependencies = gatherDerivedToDependencies(info);
+ }
+
static <S extends Storable> StorableIndexSet<S> gatherDesiredIndexes(StorableInfo<S> info) {
StorableIndexSet<S> indexSet = new StorableIndexSet<S>();
indexSet.addIndexes(info);
@@ -85,7 +332,7 @@ class IndexAnalysis {
return indexSet;
}
- static boolean isUsedByIndex(StorableProperty<?> property) {
+ private static boolean isUsedByIndex(StorableProperty<?> property) {
StorableInfo<?> info = StorableIntrospector.examine(property.getEnclosingType());
for (int i=info.getIndexCount(); --i>=0; ) {
StorableIndex<?> index = info.getIndex(i);
@@ -99,7 +346,7 @@ class IndexAnalysis {
return false;
}
- static boolean isJoinAndUsedByIndexedDerivedProperty(StorableProperty<?> property) {
+ private static boolean isJoinAndUsedByIndexedDerivedProperty(StorableProperty<?> property) {
if (property.isJoin()) {
for (ChainedProperty<?> derivedTo : property.getDerivedToProperties()) {
if (isUsedByIndex(derivedTo.getPrimeProperty())) {
@@ -110,12 +357,7 @@ class IndexAnalysis {
return false;
}
- /**
- * Returns derived-to properties in external storables that are used by indexes.
- *
- * @return null if none
- */
- static Set<ChainedProperty<?>> gatherDerivedToDependencies(StorableInfo<?> info) {
+ private static Set<ChainedProperty<?>> gatherDerivedToDependencies(StorableInfo<?> info) {
Set<ChainedProperty<?>> set = null;
for (StorableProperty<?> property : info.getAllProperties().values()) {
for (ChainedProperty<?> derivedTo : property.getDerivedToProperties()) {
diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysisPool.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysisPool.java
new file mode 100644
index 0000000..1380a78
--- /dev/null
+++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysisPool.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2008 Amazon Technologies, Inc. or its affiliates.
+ * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks
+ * of Amazon Technologies, Inc. or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.amazon.carbonado.repo.indexed;
+
+import com.amazon.carbonado.RepositoryException;
+import com.amazon.carbonado.Storage;
+
+import com.amazon.carbonado.util.AbstractPool;
+
+/**
+ *
+ *
+ * @author Brian S O'Neill
+ */
+class IndexAnalysisPool extends AbstractPool<Storage, IndexAnalysis, RepositoryException> {
+ private final IndexedRepository mRepository;
+
+ public IndexAnalysisPool(IndexedRepository repository) {
+ mRepository = repository;
+ }
+
+ protected IndexAnalysis create(Storage masterStorage) throws RepositoryException {
+ return new IndexAnalysis(mRepository, masterStorage);
+ }
+}
diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedRepository.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedRepository.java
index 7c28ee8..44cc5d1 100644
--- a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedRepository.java
+++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedRepository.java
@@ -20,6 +20,7 @@ package com.amazon.carbonado.repo.indexed;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
@@ -64,6 +65,7 @@ class IndexedRepository implements Repository,
private final double mIndexThrottle;
private final boolean mAllClustered;
private final StoragePool mStoragePool;
+ private final IndexAnalysisPool mIndexAnalysisPool;
IndexedRepository(AtomicReference<Repository> rootRef, String name,
Repository repository,
@@ -71,12 +73,19 @@ class IndexedRepository implements Repository,
double indexThrottle,
boolean allClustered)
{
+ if (repository.getCapability(IndexInfoCapability.class) == null) {
+ throw new UnsupportedOperationException
+ ("Wrapped repository doesn't support being indexed -- " +
+ "it must support IndexInfoCapability.");
+ }
+
mRootRef = rootRef;
mRepository = repository;
mName = name;
mIndexRepairEnabled = indexRepairEnabled;
mIndexThrottle = indexThrottle;
mAllClustered = allClustered;
+ mIndexAnalysisPool = new IndexAnalysisPool(this);
mStoragePool = new StoragePool() {
@Override
@@ -97,15 +106,11 @@ class IndexedRepository implements Repository,
return masterStorage;
}
- return new IndexedStorage<S>(IndexedRepository.this, masterStorage);
+ IndexAnalysis<S> analysis = mIndexAnalysisPool.get(masterStorage);
+
+ return new IndexedStorage<S>(analysis);
}
};
-
- if (repository.getCapability(IndexInfoCapability.class) == null) {
- throw new UnsupportedOperationException
- ("Wrapped repository doesn't support being indexed -- " +
- "it must support IndexInfoCapability.");
- }
}
public String getName() {
@@ -143,22 +148,41 @@ class IndexedRepository implements Repository,
return mRepository.getCapability(capabilityType);
}
+ // Required by IndexInfoCapability.
public <S extends Storable> IndexInfo[] getIndexInfo(Class<S> storableType)
throws RepositoryException
{
- return ((IndexedStorage) storageFor(storableType)).getIndexInfo();
+ if (Unindexed.class.isAssignableFrom(storableType)) {
+ return new IndexInfo[0];
+ }
+
+ Storage<S> masterStorage = mRepository.storageFor(storableType);
+ IndexAnalysis<S> analysis = mIndexAnalysisPool.get(masterStorage);
+
+ IndexInfo[] infos = new IndexInfo[analysis.allIndexInfoMap.size()];
+ return analysis.allIndexInfoMap.values().toArray(infos);
}
+ // Required by IndexEntryAccessCapability.
public <S extends Storable> IndexEntryAccessor<S>[]
getIndexEntryAccessors(Class<S> storableType)
throws RepositoryException
{
- Storage<S> storage = storageFor(storableType);
- if (storage instanceof IndexedStorage) {
- return ((IndexedStorage<S>) storage).getIndexEntryAccessors();
- } else {
+ if (Unindexed.class.isAssignableFrom(storableType)) {
return new IndexEntryAccessor[0];
}
+
+ Storage<S> masterStorage = mRepository.storageFor(storableType);
+ IndexAnalysis<S> analysis = mIndexAnalysisPool.get(masterStorage);
+
+ List<IndexEntryAccessor<S>> accessors =
+ new ArrayList<IndexEntryAccessor<S>>(analysis.allIndexInfoMap.size());
+ for (IndexInfo info : analysis.allIndexInfoMap.values()) {
+ if (info instanceof IndexEntryAccessor) {
+ accessors.add((IndexEntryAccessor<S>) info);
+ }
+ }
+ return accessors.toArray(new IndexEntryAccessor[accessors.size()]);
}
public String[] getUserStorableTypeNames() throws RepositoryException {
diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java
index 9ff204f..1abb064 100644
--- a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java
+++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java
@@ -18,12 +18,8 @@
package com.amazon.carbonado.repo.indexed;
-import java.util.ArrayList;
import java.util.Collection;
-import java.util.IdentityHashMap;
-import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -43,16 +39,12 @@ import com.amazon.carbonado.Storage;
import com.amazon.carbonado.Transaction;
import com.amazon.carbonado.Trigger;
import com.amazon.carbonado.capability.IndexInfo;
-import com.amazon.carbonado.capability.IndexInfoCapability;
import com.amazon.carbonado.cursor.MergeSortBuffer;
import com.amazon.carbonado.filter.Filter;
import com.amazon.carbonado.info.ChainedProperty;
-import com.amazon.carbonado.info.Direction;
-import com.amazon.carbonado.info.StorableInfo;
-import com.amazon.carbonado.info.StorableIntrospector;
import com.amazon.carbonado.info.StorableIndex;
import com.amazon.carbonado.cursor.SortBuffer;
@@ -84,241 +76,35 @@ class IndexedStorage<S extends Storable> implements Storage<S>, StorageAccess<S>
private final QueryEngine<S> mQueryEngine;
- @SuppressWarnings("unchecked")
- IndexedStorage(IndexedRepository repository, Storage<S> masterStorage)
- throws RepositoryException
- {
- mRepository = repository;
- mMasterStorage = masterStorage;
- mAllIndexInfoMap = new IdentityHashMap<StorableIndex<S>, IndexInfo>();
-
- StorableInfo<S> info = StorableIntrospector.examine(masterStorage.getStorableType());
-
- // The set of indexes that the Storable defines, reduced.
- final StorableIndexSet<S> desiredIndexSet;
- {
- desiredIndexSet = IndexAnalysis.gatherDesiredIndexes(info);
- desiredIndexSet.reduce(Direction.ASCENDING);
- if (mRepository.isAllClustered()) {
- desiredIndexSet.markClustered(true);
- }
- }
-
- // The set of indexes that are populated and available for use. This is
- // determined by examining index metadata. If the Storable has not
- // changed, it will be the same as desiredIndexSet. If any existing
- // indexes use a property whose type has changed, it is added to
- // bogusIndexSet. Bogus indexes are removed if repair is enabled.
- final StorableIndexSet<S> existingIndexSet;
- final StorableIndexSet<S> bogusIndexSet;
- {
- existingIndexSet = new StorableIndexSet<S>();
- bogusIndexSet = new StorableIndexSet<S>();
-
- Query<StoredIndexInfo> query = repository.getWrappedRepository()
- .storageFor(StoredIndexInfo.class)
- // Primary key of StoredIndexInfo is an index descriptor, which
- // starts with the storable type name. This emulates a
- // "wildcard at the end" search.
- .query("indexName >= ? & indexName < ?")
- .with(getStorableType().getName() + '~')
- .with(getStorableType().getName() + '~' + '\uffff');
-
- List<StoredIndexInfo> storedInfos;
- try {
- Transaction txn = repository.getWrappedRepository()
- .enterTopTransaction(IsolationLevel.READ_COMMITTED);
- try {
- storedInfos = query.fetch().toList();
- } finally {
- txn.exit();
- }
- } catch (FetchException e) {
- if (e instanceof FetchDeadlockException || e instanceof FetchTimeoutException) {
- // Might be caused by coarse locks. Switch to nested
- // transaction to share the locks.
- Transaction txn = repository.getWrappedRepository()
- .enterTransaction(IsolationLevel.READ_COMMITTED);
- try {
- storedInfos = query.fetch().toList();
- } finally {
- txn.exit();
- }
- } else {
- throw e;
- }
- }
-
- for (StoredIndexInfo indexInfo : storedInfos) {
- String name = indexInfo.getIndexName();
- StorableIndex index;
- try {
- index = StorableIndex.parseNameDescriptor(name, info);
- } catch (IllegalArgumentException e) {
- // Remove unrecognized descriptors.
- unregisterIndex(name);
- continue;
- }
- if (index.getTypeDescriptor().equals(indexInfo.getIndexTypeDescriptor())) {
- existingIndexSet.add(index);
- } else {
- bogusIndexSet.add(index);
- }
- }
- }
-
- nonUniqueSearch: {
- // If any existing indexes are non-unique, then indexes are for an
- // older version. For compatibility, don't uniquify the
- // indexes. Otherwise, these indexes would need to be rebuilt.
- for (StorableIndex<S> index : existingIndexSet) {
- if (!index.isUnique()) {
- break nonUniqueSearch;
- }
- }
-
- // The index implementation includes all primary key properties
- // anyhow, so adding them here allows query analyzer to see these
- // properties. As a side-effect of uniquify, all indexes are
- // unique, and thus have 'U' in the descriptor. Each time
- // nonUniqueSearch is run, it will not find any non-unique indexes.
- desiredIndexSet.uniquify(info);
- }
-
- // Gather free indexes, which are already provided by the underlying
- // storage. They can be used for querying, but we should not manage them.
- final StorableIndexSet<S> freeIndexSet;
- {
- freeIndexSet = new StorableIndexSet<S>();
-
- IndexInfoCapability cap = repository.getWrappedRepository()
- .getCapability(IndexInfoCapability.class);
-
- if (cap != null) {
- for (IndexInfo ii : cap.getIndexInfo(masterStorage.getStorableType())) {
- StorableIndex<S> freeIndex;
- try {
- freeIndex = new StorableIndex<S>(masterStorage.getStorableType(), ii);
- } catch (IllegalArgumentException e) {
- // Assume index is malformed, so ignore it.
- continue;
- }
- if (mRepository.isAllClustered()) {
- freeIndex = freeIndex.clustered(true);
- }
- mAllIndexInfoMap.put(freeIndex, ii);
- freeIndexSet.add(freeIndex);
- }
- }
- }
-
- // The set of indexes that can actually be used for querying. If index
- // repair is enabled, this set will be the same as
- // desiredIndexSet. Otherwise, it will be the intersection of
- // existingIndexSet and desiredIndexSet. In both cases, "free" indexes
- // are added to the set too.
- final StorableIndexSet<S> queryableIndexSet;
- {
- queryableIndexSet = new StorableIndexSet<S>(desiredIndexSet);
-
- if (!mRepository.isIndexRepairEnabled()) {
- // Can only query the intersection.
- queryableIndexSet.retainAll(existingIndexSet);
- }
-
- // Add the indexes we get for free.
- queryableIndexSet.addAll(freeIndexSet);
- }
-
- // The set of indexes that should be kept up-to-date. If index repair
- // is enabled, this set will be the same as desiredIndexSet. Otherwise,
- // it will be the union of existingIndexSet and desiredIndexSet. In
- // both cases, "free" indexes are removed from the set too. By doing a
- // union, no harm is caused by changing the index set and then reverting.
- final StorableIndexSet<S> managedIndexSet;
- {
- managedIndexSet = new StorableIndexSet<S>(desiredIndexSet);
-
- if (!mRepository.isIndexRepairEnabled()) {
- // Must manage the union.
- managedIndexSet.addAll(existingIndexSet);
- }
-
- // Remove the indexes we get for free.
- managedIndexSet.removeAll(freeIndexSet);
- }
-
- // The set of indexes that should be removed and no longer managed. If
- // index repair is enabled, this set will be the existingIndexSet minus
- // desiredIndexSet minus freeIndexSet plus bogusIndexSet. Otherwise, it
- // will be empty.
- final StorableIndexSet<S> removeIndexSet;
- {
- removeIndexSet = new StorableIndexSet<S>();
-
- if (mRepository.isIndexRepairEnabled()) {
- removeIndexSet.addAll(existingIndexSet);
- removeIndexSet.removeAll(desiredIndexSet);
- removeIndexSet.removeAll(freeIndexSet);
- removeIndexSet.addAll(bogusIndexSet);
- }
- }
-
- // The set of indexes that should be freshly populated. If index repair
- // is enabled, this set will be the desiredIndexSet minus
- // existingIndexSet minus freeIndexSet. Otherwise, it will be empty.
- final StorableIndexSet<S> addIndexSet;
- {
- addIndexSet = new StorableIndexSet<S>();
-
- if (mRepository.isIndexRepairEnabled()) {
- addIndexSet.addAll(desiredIndexSet);
- addIndexSet.removeAll(existingIndexSet);
- addIndexSet.removeAll(freeIndexSet);
- }
- }
-
- // Support for managed indexes...
- if (managedIndexSet.size() > 0) {
- ManagedIndex<S>[] managedIndexes = new ManagedIndex[managedIndexSet.size()];
- int i = 0;
- for (StorableIndex<S> index : managedIndexSet) {
- IndexEntryGenerator<S> builder = IndexEntryGenerator.getInstance(index);
- Class<? extends Storable> indexEntryClass = builder.getIndexEntryClass();
- Storage<?> indexEntryStorage = repository.getIndexEntryStorageFor(indexEntryClass);
- ManagedIndex managedIndex =
- new ManagedIndex<S>(this, index, builder, indexEntryStorage);
-
- mAllIndexInfoMap.put(index, managedIndex);
- managedIndexes[i++] = managedIndex;
- }
+ IndexedStorage(IndexAnalysis<S> analysis) throws RepositoryException {
+ mRepository = analysis.repository;
+ mMasterStorage = analysis.masterStorage;
+ mAllIndexInfoMap = analysis.allIndexInfoMap;
+ mQueryableIndexSet = analysis.queryableIndexSet;
- if (!addTrigger(new IndexesTrigger<S>(managedIndexes))) {
+ if (analysis.indexesTrigger != null) {
+ if (!addTrigger(analysis.indexesTrigger)) {
+ // This might be caused by this storage being created again recursively.
throw new RepositoryException("Unable to add trigger for managing indexes");
}
}
// Okay, now start doing some damage. First, remove unnecessary indexes.
- for (StorableIndex<S> index : removeIndexSet) {
+ for (StorableIndex<S> index : analysis.removeIndexSet) {
removeIndex(index);
}
// Now add new indexes.
- for (StorableIndex<S> index : addIndexSet) {
+ for (StorableIndex<S> index : analysis.addIndexSet) {
registerIndex((ManagedIndex) mAllIndexInfoMap.get(index));
}
- mQueryableIndexSet = queryableIndexSet;
- mQueryEngine = new QueryEngine<S>(masterStorage.getStorableType(), repository);
+ mQueryEngine = new QueryEngine<S>(mMasterStorage.getStorableType(), mRepository);
// Install triggers to manage derived properties in external Storables.
-
- Set<ChainedProperty<?>> derivedToDependencies =
- IndexAnalysis.gatherDerivedToDependencies(info);
-
- if (derivedToDependencies != null) {
- for (ChainedProperty<?> derivedTo : derivedToDependencies) {
- addTrigger(new DerivedIndexesTrigger(repository, getStorableType(), derivedTo));
+ if (analysis.derivedToDependencies != null) {
+ for (ChainedProperty<?> derivedTo : analysis.derivedToDependencies) {
+ addTrigger(new DerivedIndexesTrigger(mRepository, getStorableType(), derivedTo));
}
}
}
@@ -381,25 +167,6 @@ class IndexedStorage<S extends Storable> implements Storage<S>, StorageAccess<S>
return mMasterStorage.removeTrigger(trigger);
}
- // Required by IndexInfoCapability.
- public IndexInfo[] getIndexInfo() {
- IndexInfo[] infos = new IndexInfo[mAllIndexInfoMap.size()];
- return mAllIndexInfoMap.values().toArray(infos);
- }
-
- // Required by IndexEntryAccessCapability.
- @SuppressWarnings("unchecked")
- public IndexEntryAccessor<S>[] getIndexEntryAccessors() {
- List<IndexEntryAccessor<S>> accessors =
- new ArrayList<IndexEntryAccessor<S>>(mAllIndexInfoMap.size());
- for (IndexInfo info : mAllIndexInfoMap.values()) {
- if (info instanceof IndexEntryAccessor) {
- accessors.add((IndexEntryAccessor<S>) info);
- }
- }
- return accessors.toArray(new IndexEntryAccessor[accessors.size()]);
- }
-
// Required by StorageAccess.
public QueryExecutorFactory<S> getQueryExecutorFactory() {
return mQueryEngine;
diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java b/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java
index a8f5788..2d0617e 100644
--- a/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java
+++ b/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java
@@ -82,20 +82,23 @@ class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
return naturalOrdering;
}
- private final IndexedStorage<S> mIndexedStorage;
+ private final IndexedRepository mRepository;
+ private final Storage<S> mMasterStorage;
private final StorableIndex mIndex;
private final IndexEntryGenerator<S> mGenerator;
private final Storage<?> mIndexEntryStorage;
private Query<?> mSingleMatchQuery;
- ManagedIndex(IndexedStorage<S> indexedStorage,
+ ManagedIndex(IndexedRepository repository,
+ Storage<S> masterStorage,
StorableIndex<S> index,
IndexEntryGenerator<S> generator,
Storage<?> indexEntryStorage)
throws SupportException
{
- mIndexedStorage = indexedStorage;
+ mRepository = repository;
+ mMasterStorage = masterStorage;
mIndex = index;
mGenerator = generator;
mIndexEntryStorage = indexEntryStorage;
@@ -236,9 +239,6 @@ class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
* @param repo used to enter transactions
*/
void buildIndex(double desiredSpeed) throws RepositoryException {
- final Repository repo = mIndexedStorage.mRepository;
- final Storage<S> masterStorage = mIndexedStorage.mMasterStorage;
-
final MergeSortBuffer buffer;
final Comparator c;
@@ -249,13 +249,13 @@ class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
// Need to explicitly order master query by primary key in order
// for fetchAfter to work correctly in case corrupt records are
// encountered.
- masterQuery = masterStorage.query()
- .orderBy(naturalOrdering(masterStorage.getStorableType()));
+ masterQuery = mMasterStorage.query()
+ .orderBy(naturalOrdering(mMasterStorage.getStorableType()));
}
// Quick check to see if any records exist in master.
{
- Transaction txn = repo.enterTransaction(IsolationLevel.READ_COMMITTED);
+ Transaction txn = mRepository.enterTransaction(IsolationLevel.READ_COMMITTED);
try {
Cursor<S> cursor = masterQuery.fetch();
if (!cursor.hasNext()) {
@@ -269,14 +269,14 @@ class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
// Enter top transaction with isolation level of none to make sure
// preload operation does not run in a long nested transaction.
- Transaction txn = repo.enterTopTransaction(IsolationLevel.NONE);
+ Transaction txn = mRepository.enterTopTransaction(IsolationLevel.NONE);
try {
Cursor<S> cursor = masterQuery.fetch();
try {
if (log.isInfoEnabled()) {
StringBuilder b = new StringBuilder();
- b.append("Building index on ");
- b.append(masterStorage.getStorableType().getName());
+ b.append("Preparing index on ");
+ b.append(mMasterStorage.getStorableType().getName());
b.append(": ");
try {
mIndex.appendTo(b);
@@ -350,7 +350,7 @@ class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
// fail, since unique index cannot be built.
if (log.isInfoEnabled()) {
- log.info("Verifying unique index");
+ log.info("Verifying index");
}
Object last = null;
@@ -384,7 +384,7 @@ class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
long totalDeleted = 0;
long totalProgress = 0;
- txn = repo.enterTopTransaction(IsolationLevel.READ_COMMITTED);
+ txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED);
try {
txn.setForUpdate(true);
@@ -463,7 +463,7 @@ class ManagedIndex<S extends Storable> implements IndexEntryAccessor<S> {
log.info(String.format(format, percent));
}
- txn = repo.enterTopTransaction(IsolationLevel.READ_COMMITTED);
+ txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED);
if (indexEntryCursor != null) {
indexEntryCursor.close();
--
cgit v1.2.3