From 6cee40de307fa88391573b94af60fc0942a99faa Mon Sep 17 00:00:00 2001 From: "Brian S. O'Neill" Date: Sun, 25 May 2008 19:02:37 +0000 Subject: Fixed cyclic dependency with indexes against complex derived properties. --- .../repo/indexed/DependentStorableFetcher.java | 38 ++- .../repo/indexed/DerivedIndexesTrigger.java | 2 +- .../carbonado/repo/indexed/IndexAnalysis.java | 262 ++++++++++++++++++++- .../carbonado/repo/indexed/IndexAnalysisPool.java | 41 ++++ .../carbonado/repo/indexed/IndexedRepository.java | 48 +++- .../carbonado/repo/indexed/IndexedStorage.java | 261 ++------------------ .../carbonado/repo/indexed/ManagedIndex.java | 30 +-- 7 files changed, 386 insertions(+), 296 deletions(-) create mode 100644 src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysisPool.java (limited to 'src/main/java/com/amazon/carbonado/repo/indexed') diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/DependentStorableFetcher.java b/src/main/java/com/amazon/carbonado/repo/indexed/DependentStorableFetcher.java index 98454f1..d9e9f32 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/DependentStorableFetcher.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/DependentStorableFetcher.java @@ -45,9 +45,11 @@ import com.amazon.carbonado.info.StorableProperty; class DependentStorableFetcher { private final IndexedRepository mRepository; private final IndexEntryAccessor[] mIndexEntryAccessors; - private final Query mQuery; + private final Filter mFilter; private final String[] mJoinProperties; + private Query mQuery; + /** * @param derivedTo special chained property from StorableProperty.getDerivedToProperties */ @@ -81,7 +83,7 @@ class DependentStorableFetcher { if (accessorList.size() == 0) { throw new SupportException ("Unable to find index accessors for derived-to property: " + derivedTo + - ", enclosing type: " + dType); + ", enclosing type: " + dType.getName() + ", source type: " + sType.getName()); } // Build a query on D joined to S. @@ -100,18 +102,18 @@ class DependentStorableFetcher { Filter dFilter = Filter.getOpenFilter(dType); for (int i=0; i element = join.getInternalJoinElement(i); - joinProperties[i] = element.getName(); + joinProperties[i] = join.getExternalJoinElement(i).getName(); + StorableProperty internal = join.getInternalJoinElement(i); if (base == null) { - dFilter = dFilter.and(element.getName(), RelOp.EQ); + dFilter = dFilter.and(internal.getName(), RelOp.EQ); } else { - dFilter = dFilter.and(base.append(element).toString(), RelOp.EQ); + dFilter = dFilter.and(base.append(internal).toString(), RelOp.EQ); } } mRepository = repository; mIndexEntryAccessors = accessorList.toArray(new IndexEntryAccessor[accessorList.size()]); - mQuery = repository.storageFor(dType).query(dFilter); + mFilter = dFilter; mJoinProperties = joinProperties; } @@ -120,7 +122,7 @@ class DependentStorableFetcher { } public Cursor fetchDependenentStorables(S storable) throws FetchException { - Query query = mQuery; + Query query = query(); for (String property : mJoinProperties) { query = query.with(storable.getPropertyValue(property)); } @@ -144,7 +146,7 @@ class DependentStorableFetcher { @Override public int hashCode() { - return mQuery.getFilter().hashCode(); + return mFilter.hashCode(); } @Override @@ -154,7 +156,7 @@ class DependentStorableFetcher { } if (obj instanceof DependentStorableFetcher) { DependentStorableFetcher other = (DependentStorableFetcher) obj; - return mQuery.getFilter().equals(other.mQuery.getFilter()) + return mFilter.equals(other.mFilter) && Arrays.equals(mJoinProperties, other.mJoinProperties) && Arrays.equals(mIndexEntryAccessors, other.mIndexEntryAccessors); } @@ -164,7 +166,21 @@ class DependentStorableFetcher { @Override public String toString() { return "DependentStorableFetcher: {indexes=" + Arrays.toString(mIndexEntryAccessors) + - ", query=" + mQuery + + ", filter=" + mFilter + ", join properties=" + Arrays.toString(mJoinProperties) + '}'; } + + private Query query() throws FetchException { + // Query is lazily created to avoid stack overflow due to cyclic + // dependencies. + Query query = mQuery; + if (query == null) { + try { + mQuery = query = mRepository.storageFor(mFilter.getStorableType()).query(mFilter); + } catch (RepositoryException e) { + throw e.toFetchException(); + } + } + return query; + } } diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/DerivedIndexesTrigger.java b/src/main/java/com/amazon/carbonado/repo/indexed/DerivedIndexesTrigger.java index 9e69935..2cc3d8f 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/DerivedIndexesTrigger.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/DerivedIndexesTrigger.java @@ -50,7 +50,7 @@ class DerivedIndexesTrigger extends Trig this(new DependentStorableFetcher(repository, sType, derivedTo)); } - DerivedIndexesTrigger(DependentStorableFetcher fetcher) { + private DerivedIndexesTrigger(DependentStorableFetcher fetcher) { mFetcher = fetcher; } diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysis.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysis.java index d65e32e..241f297 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysis.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysis.java @@ -20,9 +20,23 @@ package com.amazon.carbonado.repo.indexed; import java.util.Arrays; import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; import java.util.Set; +import com.amazon.carbonado.FetchException; +import com.amazon.carbonado.FetchDeadlockException; +import com.amazon.carbonado.FetchTimeoutException; +import com.amazon.carbonado.IsolationLevel; +import com.amazon.carbonado.Query; +import com.amazon.carbonado.RepositoryException; import com.amazon.carbonado.Storable; +import com.amazon.carbonado.Storage; +import com.amazon.carbonado.Transaction; + +import com.amazon.carbonado.capability.IndexInfo; +import com.amazon.carbonado.capability.IndexInfoCapability; import com.amazon.carbonado.filter.Filter; import com.amazon.carbonado.filter.RelOp; @@ -38,12 +52,245 @@ import com.amazon.carbonado.qe.FilteringScore; import com.amazon.carbonado.qe.StorableIndexSet; /** - * Collection of static methods which perform index analysis. + * Builds various sets of indexes for a Storable type. * * @author Brian S O'Neill * @since 1.2 */ -class IndexAnalysis { +class IndexAnalysis { + + final IndexedRepository repository; + final Storage masterStorage; + + // The set of indexes that can actually be used for querying. If index + // repair is enabled, this set will be the same as desiredIndexSet. + // Otherwise, it will be the intersection of existingIndexSet and + // desiredIndexSet. In both cases, "free" indexes are added to the set too. + final StorableIndexSet queryableIndexSet; + + // The set of indexes that should be removed and no longer managed. If + // index repair is enabled, this set will be the existingIndexSet minus + // desiredIndexSet minus freeIndexSet plus bogusIndexSet. Otherwise, it + // will be empty. + final StorableIndexSet removeIndexSet; + + // The set of indexes that should be freshly populated. If index repair is + // enabled, this set will be the desiredIndexSet minus existingIndexSet + // minus freeIndexSet. Otherwise, it will be empty. + final StorableIndexSet addIndexSet; + + // Maps free and managed indexes to IndexInfo and ManagedIndex objects. + final Map, IndexInfo> allIndexInfoMap; + + // Trigger which must be installed to keep managed indexes up to date. Is + // null if there are no managed indexes. + final IndexesTrigger indexesTrigger; + + // The set of derived-to properties in external storables that are used by + // indexes. Is null if none. + final Set> derivedToDependencies; + + public IndexAnalysis(IndexedRepository repository, Storage masterStorage) + throws RepositoryException + { + this.repository = repository; + this.masterStorage = masterStorage; + + StorableInfo info = StorableIntrospector.examine(masterStorage.getStorableType()); + + // The set of indexes that the Storable defines, reduced. + final StorableIndexSet desiredIndexSet; + { + desiredIndexSet = gatherDesiredIndexes(info); + desiredIndexSet.reduce(Direction.ASCENDING); + if (repository.isAllClustered()) { + desiredIndexSet.markClustered(true); + } + } + + // The set of indexes that are populated and available for use. This is + // determined by examining index metadata. If the Storable has not + // changed, it will be the same as desiredIndexSet. If any existing + // indexes use a property whose type has changed, it is added to + // bogusIndexSet. Bogus indexes are removed if repair is enabled. + final StorableIndexSet existingIndexSet; + final StorableIndexSet bogusIndexSet; + { + existingIndexSet = new StorableIndexSet(); + bogusIndexSet = new StorableIndexSet(); + + Query query = repository.getWrappedRepository() + .storageFor(StoredIndexInfo.class) + // Primary key of StoredIndexInfo is an index descriptor, which + // starts with the storable type name. This emulates a + // "wildcard at the end" search. + .query("indexName >= ? & indexName < ?") + .with(info.getStorableType().getName() + '~') + .with(info.getStorableType().getName() + '~' + '\uffff'); + + List storedInfos; + try { + Transaction txn = repository.getWrappedRepository() + .enterTopTransaction(IsolationLevel.READ_COMMITTED); + try { + storedInfos = query.fetch().toList(); + } finally { + txn.exit(); + } + } catch (FetchException e) { + if (e instanceof FetchDeadlockException || e instanceof FetchTimeoutException) { + // Might be caused by coarse locks. Switch to nested + // transaction to share the locks. + Transaction txn = repository.getWrappedRepository() + .enterTransaction(IsolationLevel.READ_COMMITTED); + try { + storedInfos = query.fetch().toList(); + } finally { + txn.exit(); + } + } else { + throw e; + } + } + + for (StoredIndexInfo indexInfo : storedInfos) { + String name = indexInfo.getIndexName(); + StorableIndex index; + try { + index = StorableIndex.parseNameDescriptor(name, info); + } catch (IllegalArgumentException e) { + // Skip unrecognized descriptors. + continue; + } + if (index.getTypeDescriptor().equals(indexInfo.getIndexTypeDescriptor())) { + existingIndexSet.add(index); + } else { + bogusIndexSet.add(index); + } + } + } + + nonUniqueSearch: { + // If any existing indexes are non-unique, then indexes are for an + // older version. For compatibility, don't uniquify the + // indexes. Otherwise, these indexes would need to be rebuilt. + for (StorableIndex index : existingIndexSet) { + if (!index.isUnique()) { + break nonUniqueSearch; + } + } + + // The index implementation includes all primary key properties + // anyhow, so adding them here allows query analyzer to see these + // properties. As a side-effect of uniquify, all indexes are + // unique, and thus have 'U' in the descriptor. Each time + // nonUniqueSearch is run, it will not find any non-unique indexes. + desiredIndexSet.uniquify(info); + } + + // The set of free indexes, which are already provided by the underlying + // storage. They can be used for querying, but we should not manage them. + final StorableIndexSet freeIndexSet; + { + freeIndexSet = new StorableIndexSet(); + allIndexInfoMap = new IdentityHashMap, IndexInfo>(); + + IndexInfoCapability cap = repository.getWrappedRepository() + .getCapability(IndexInfoCapability.class); + + if (cap != null) { + for (IndexInfo ii : cap.getIndexInfo(info.getStorableType())) { + StorableIndex freeIndex; + try { + freeIndex = new StorableIndex(info.getStorableType(), ii); + } catch (IllegalArgumentException e) { + // Assume index is malformed, so ignore it. + continue; + } + if (repository.isAllClustered()) { + freeIndex = freeIndex.clustered(true); + } + freeIndexSet.add(freeIndex); + allIndexInfoMap.put(freeIndex, ii); + } + } + } + + { + queryableIndexSet = new StorableIndexSet(desiredIndexSet); + + if (!repository.isIndexRepairEnabled()) { + // Can only query the intersection. + queryableIndexSet.retainAll(existingIndexSet); + } + + // Add the indexes we get for free. + queryableIndexSet.addAll(freeIndexSet); + } + + // The set of indexes that should be kept up-to-date. If index repair + // is enabled, this set will be the same as desiredIndexSet. Otherwise, + // it will be the union of existingIndexSet and desiredIndexSet. In + // both cases, "free" indexes are removed from the set too. By doing a + // union, no harm is caused by changing the index set and then + // reverting. + final StorableIndexSet managedIndexSet; + { + managedIndexSet = new StorableIndexSet(desiredIndexSet); + + if (repository.isIndexRepairEnabled()) { + // Must manage the union. + managedIndexSet.addAll(existingIndexSet); + } + + // Remove the indexes we get for free. + managedIndexSet.removeAll(freeIndexSet); + } + + { + removeIndexSet = new StorableIndexSet(); + + if (repository.isIndexRepairEnabled()) { + removeIndexSet.addAll(existingIndexSet); + removeIndexSet.removeAll(desiredIndexSet); + removeIndexSet.removeAll(freeIndexSet); + removeIndexSet.addAll(bogusIndexSet); + } + } + + { + addIndexSet = new StorableIndexSet(); + + if (repository.isIndexRepairEnabled()) { + addIndexSet.addAll(desiredIndexSet); + addIndexSet.removeAll(existingIndexSet); + addIndexSet.removeAll(freeIndexSet); + } + } + + // Support for managed indexes... + if (managedIndexSet.size() <= 0) { + indexesTrigger = null; + } else { + ManagedIndex[] managedIndexes = new ManagedIndex[managedIndexSet.size()]; + int i = 0; + for (StorableIndex index : managedIndexSet) { + IndexEntryGenerator builder = IndexEntryGenerator.getInstance(index); + Class indexEntryClass = builder.getIndexEntryClass(); + Storage indexEntryStorage = repository.getIndexEntryStorageFor(indexEntryClass); + ManagedIndex managedIndex = new ManagedIndex + (repository, masterStorage, index, builder, indexEntryStorage); + + allIndexInfoMap.put(index, managedIndex); + managedIndexes[i++] = managedIndex; + } + + indexesTrigger = new IndexesTrigger(managedIndexes); + } + + derivedToDependencies = gatherDerivedToDependencies(info); + } + static StorableIndexSet gatherDesiredIndexes(StorableInfo info) { StorableIndexSet indexSet = new StorableIndexSet(); indexSet.addIndexes(info); @@ -85,7 +332,7 @@ class IndexAnalysis { return indexSet; } - static boolean isUsedByIndex(StorableProperty property) { + private static boolean isUsedByIndex(StorableProperty property) { StorableInfo info = StorableIntrospector.examine(property.getEnclosingType()); for (int i=info.getIndexCount(); --i>=0; ) { StorableIndex index = info.getIndex(i); @@ -99,7 +346,7 @@ class IndexAnalysis { return false; } - static boolean isJoinAndUsedByIndexedDerivedProperty(StorableProperty property) { + private static boolean isJoinAndUsedByIndexedDerivedProperty(StorableProperty property) { if (property.isJoin()) { for (ChainedProperty derivedTo : property.getDerivedToProperties()) { if (isUsedByIndex(derivedTo.getPrimeProperty())) { @@ -110,12 +357,7 @@ class IndexAnalysis { return false; } - /** - * Returns derived-to properties in external storables that are used by indexes. - * - * @return null if none - */ - static Set> gatherDerivedToDependencies(StorableInfo info) { + private static Set> gatherDerivedToDependencies(StorableInfo info) { Set> set = null; for (StorableProperty property : info.getAllProperties().values()) { for (ChainedProperty derivedTo : property.getDerivedToProperties()) { diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysisPool.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysisPool.java new file mode 100644 index 0000000..1380a78 --- /dev/null +++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysisPool.java @@ -0,0 +1,41 @@ +/* + * Copyright 2008 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.repo.indexed; + +import com.amazon.carbonado.RepositoryException; +import com.amazon.carbonado.Storage; + +import com.amazon.carbonado.util.AbstractPool; + +/** + * + * + * @author Brian S O'Neill + */ +class IndexAnalysisPool extends AbstractPool { + private final IndexedRepository mRepository; + + public IndexAnalysisPool(IndexedRepository repository) { + mRepository = repository; + } + + protected IndexAnalysis create(Storage masterStorage) throws RepositoryException { + return new IndexAnalysis(mRepository, masterStorage); + } +} diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedRepository.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedRepository.java index 7c28ee8..44cc5d1 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedRepository.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedRepository.java @@ -20,6 +20,7 @@ package com.amazon.carbonado.repo.indexed; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.concurrent.atomic.AtomicReference; @@ -64,6 +65,7 @@ class IndexedRepository implements Repository, private final double mIndexThrottle; private final boolean mAllClustered; private final StoragePool mStoragePool; + private final IndexAnalysisPool mIndexAnalysisPool; IndexedRepository(AtomicReference rootRef, String name, Repository repository, @@ -71,12 +73,19 @@ class IndexedRepository implements Repository, double indexThrottle, boolean allClustered) { + if (repository.getCapability(IndexInfoCapability.class) == null) { + throw new UnsupportedOperationException + ("Wrapped repository doesn't support being indexed -- " + + "it must support IndexInfoCapability."); + } + mRootRef = rootRef; mRepository = repository; mName = name; mIndexRepairEnabled = indexRepairEnabled; mIndexThrottle = indexThrottle; mAllClustered = allClustered; + mIndexAnalysisPool = new IndexAnalysisPool(this); mStoragePool = new StoragePool() { @Override @@ -97,15 +106,11 @@ class IndexedRepository implements Repository, return masterStorage; } - return new IndexedStorage(IndexedRepository.this, masterStorage); + IndexAnalysis analysis = mIndexAnalysisPool.get(masterStorage); + + return new IndexedStorage(analysis); } }; - - if (repository.getCapability(IndexInfoCapability.class) == null) { - throw new UnsupportedOperationException - ("Wrapped repository doesn't support being indexed -- " + - "it must support IndexInfoCapability."); - } } public String getName() { @@ -143,22 +148,41 @@ class IndexedRepository implements Repository, return mRepository.getCapability(capabilityType); } + // Required by IndexInfoCapability. public IndexInfo[] getIndexInfo(Class storableType) throws RepositoryException { - return ((IndexedStorage) storageFor(storableType)).getIndexInfo(); + if (Unindexed.class.isAssignableFrom(storableType)) { + return new IndexInfo[0]; + } + + Storage masterStorage = mRepository.storageFor(storableType); + IndexAnalysis analysis = mIndexAnalysisPool.get(masterStorage); + + IndexInfo[] infos = new IndexInfo[analysis.allIndexInfoMap.size()]; + return analysis.allIndexInfoMap.values().toArray(infos); } + // Required by IndexEntryAccessCapability. public IndexEntryAccessor[] getIndexEntryAccessors(Class storableType) throws RepositoryException { - Storage storage = storageFor(storableType); - if (storage instanceof IndexedStorage) { - return ((IndexedStorage) storage).getIndexEntryAccessors(); - } else { + if (Unindexed.class.isAssignableFrom(storableType)) { return new IndexEntryAccessor[0]; } + + Storage masterStorage = mRepository.storageFor(storableType); + IndexAnalysis analysis = mIndexAnalysisPool.get(masterStorage); + + List> accessors = + new ArrayList>(analysis.allIndexInfoMap.size()); + for (IndexInfo info : analysis.allIndexInfoMap.values()) { + if (info instanceof IndexEntryAccessor) { + accessors.add((IndexEntryAccessor) info); + } + } + return accessors.toArray(new IndexEntryAccessor[accessors.size()]); } public String[] getUserStorableTypeNames() throws RepositoryException { diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java index 9ff204f..1abb064 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java @@ -18,12 +18,8 @@ package com.amazon.carbonado.repo.indexed; -import java.util.ArrayList; import java.util.Collection; -import java.util.IdentityHashMap; -import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -43,16 +39,12 @@ import com.amazon.carbonado.Storage; import com.amazon.carbonado.Transaction; import com.amazon.carbonado.Trigger; import com.amazon.carbonado.capability.IndexInfo; -import com.amazon.carbonado.capability.IndexInfoCapability; import com.amazon.carbonado.cursor.MergeSortBuffer; import com.amazon.carbonado.filter.Filter; import com.amazon.carbonado.info.ChainedProperty; -import com.amazon.carbonado.info.Direction; -import com.amazon.carbonado.info.StorableInfo; -import com.amazon.carbonado.info.StorableIntrospector; import com.amazon.carbonado.info.StorableIndex; import com.amazon.carbonado.cursor.SortBuffer; @@ -84,241 +76,35 @@ class IndexedStorage implements Storage, StorageAccess private final QueryEngine mQueryEngine; - @SuppressWarnings("unchecked") - IndexedStorage(IndexedRepository repository, Storage masterStorage) - throws RepositoryException - { - mRepository = repository; - mMasterStorage = masterStorage; - mAllIndexInfoMap = new IdentityHashMap, IndexInfo>(); - - StorableInfo info = StorableIntrospector.examine(masterStorage.getStorableType()); - - // The set of indexes that the Storable defines, reduced. - final StorableIndexSet desiredIndexSet; - { - desiredIndexSet = IndexAnalysis.gatherDesiredIndexes(info); - desiredIndexSet.reduce(Direction.ASCENDING); - if (mRepository.isAllClustered()) { - desiredIndexSet.markClustered(true); - } - } - - // The set of indexes that are populated and available for use. This is - // determined by examining index metadata. If the Storable has not - // changed, it will be the same as desiredIndexSet. If any existing - // indexes use a property whose type has changed, it is added to - // bogusIndexSet. Bogus indexes are removed if repair is enabled. - final StorableIndexSet existingIndexSet; - final StorableIndexSet bogusIndexSet; - { - existingIndexSet = new StorableIndexSet(); - bogusIndexSet = new StorableIndexSet(); - - Query query = repository.getWrappedRepository() - .storageFor(StoredIndexInfo.class) - // Primary key of StoredIndexInfo is an index descriptor, which - // starts with the storable type name. This emulates a - // "wildcard at the end" search. - .query("indexName >= ? & indexName < ?") - .with(getStorableType().getName() + '~') - .with(getStorableType().getName() + '~' + '\uffff'); - - List storedInfos; - try { - Transaction txn = repository.getWrappedRepository() - .enterTopTransaction(IsolationLevel.READ_COMMITTED); - try { - storedInfos = query.fetch().toList(); - } finally { - txn.exit(); - } - } catch (FetchException e) { - if (e instanceof FetchDeadlockException || e instanceof FetchTimeoutException) { - // Might be caused by coarse locks. Switch to nested - // transaction to share the locks. - Transaction txn = repository.getWrappedRepository() - .enterTransaction(IsolationLevel.READ_COMMITTED); - try { - storedInfos = query.fetch().toList(); - } finally { - txn.exit(); - } - } else { - throw e; - } - } - - for (StoredIndexInfo indexInfo : storedInfos) { - String name = indexInfo.getIndexName(); - StorableIndex index; - try { - index = StorableIndex.parseNameDescriptor(name, info); - } catch (IllegalArgumentException e) { - // Remove unrecognized descriptors. - unregisterIndex(name); - continue; - } - if (index.getTypeDescriptor().equals(indexInfo.getIndexTypeDescriptor())) { - existingIndexSet.add(index); - } else { - bogusIndexSet.add(index); - } - } - } - - nonUniqueSearch: { - // If any existing indexes are non-unique, then indexes are for an - // older version. For compatibility, don't uniquify the - // indexes. Otherwise, these indexes would need to be rebuilt. - for (StorableIndex index : existingIndexSet) { - if (!index.isUnique()) { - break nonUniqueSearch; - } - } - - // The index implementation includes all primary key properties - // anyhow, so adding them here allows query analyzer to see these - // properties. As a side-effect of uniquify, all indexes are - // unique, and thus have 'U' in the descriptor. Each time - // nonUniqueSearch is run, it will not find any non-unique indexes. - desiredIndexSet.uniquify(info); - } - - // Gather free indexes, which are already provided by the underlying - // storage. They can be used for querying, but we should not manage them. - final StorableIndexSet freeIndexSet; - { - freeIndexSet = new StorableIndexSet(); - - IndexInfoCapability cap = repository.getWrappedRepository() - .getCapability(IndexInfoCapability.class); - - if (cap != null) { - for (IndexInfo ii : cap.getIndexInfo(masterStorage.getStorableType())) { - StorableIndex freeIndex; - try { - freeIndex = new StorableIndex(masterStorage.getStorableType(), ii); - } catch (IllegalArgumentException e) { - // Assume index is malformed, so ignore it. - continue; - } - if (mRepository.isAllClustered()) { - freeIndex = freeIndex.clustered(true); - } - mAllIndexInfoMap.put(freeIndex, ii); - freeIndexSet.add(freeIndex); - } - } - } - - // The set of indexes that can actually be used for querying. If index - // repair is enabled, this set will be the same as - // desiredIndexSet. Otherwise, it will be the intersection of - // existingIndexSet and desiredIndexSet. In both cases, "free" indexes - // are added to the set too. - final StorableIndexSet queryableIndexSet; - { - queryableIndexSet = new StorableIndexSet(desiredIndexSet); - - if (!mRepository.isIndexRepairEnabled()) { - // Can only query the intersection. - queryableIndexSet.retainAll(existingIndexSet); - } - - // Add the indexes we get for free. - queryableIndexSet.addAll(freeIndexSet); - } - - // The set of indexes that should be kept up-to-date. If index repair - // is enabled, this set will be the same as desiredIndexSet. Otherwise, - // it will be the union of existingIndexSet and desiredIndexSet. In - // both cases, "free" indexes are removed from the set too. By doing a - // union, no harm is caused by changing the index set and then reverting. - final StorableIndexSet managedIndexSet; - { - managedIndexSet = new StorableIndexSet(desiredIndexSet); - - if (!mRepository.isIndexRepairEnabled()) { - // Must manage the union. - managedIndexSet.addAll(existingIndexSet); - } - - // Remove the indexes we get for free. - managedIndexSet.removeAll(freeIndexSet); - } - - // The set of indexes that should be removed and no longer managed. If - // index repair is enabled, this set will be the existingIndexSet minus - // desiredIndexSet minus freeIndexSet plus bogusIndexSet. Otherwise, it - // will be empty. - final StorableIndexSet removeIndexSet; - { - removeIndexSet = new StorableIndexSet(); - - if (mRepository.isIndexRepairEnabled()) { - removeIndexSet.addAll(existingIndexSet); - removeIndexSet.removeAll(desiredIndexSet); - removeIndexSet.removeAll(freeIndexSet); - removeIndexSet.addAll(bogusIndexSet); - } - } - - // The set of indexes that should be freshly populated. If index repair - // is enabled, this set will be the desiredIndexSet minus - // existingIndexSet minus freeIndexSet. Otherwise, it will be empty. - final StorableIndexSet addIndexSet; - { - addIndexSet = new StorableIndexSet(); - - if (mRepository.isIndexRepairEnabled()) { - addIndexSet.addAll(desiredIndexSet); - addIndexSet.removeAll(existingIndexSet); - addIndexSet.removeAll(freeIndexSet); - } - } - - // Support for managed indexes... - if (managedIndexSet.size() > 0) { - ManagedIndex[] managedIndexes = new ManagedIndex[managedIndexSet.size()]; - int i = 0; - for (StorableIndex index : managedIndexSet) { - IndexEntryGenerator builder = IndexEntryGenerator.getInstance(index); - Class indexEntryClass = builder.getIndexEntryClass(); - Storage indexEntryStorage = repository.getIndexEntryStorageFor(indexEntryClass); - ManagedIndex managedIndex = - new ManagedIndex(this, index, builder, indexEntryStorage); - - mAllIndexInfoMap.put(index, managedIndex); - managedIndexes[i++] = managedIndex; - } + IndexedStorage(IndexAnalysis analysis) throws RepositoryException { + mRepository = analysis.repository; + mMasterStorage = analysis.masterStorage; + mAllIndexInfoMap = analysis.allIndexInfoMap; + mQueryableIndexSet = analysis.queryableIndexSet; - if (!addTrigger(new IndexesTrigger(managedIndexes))) { + if (analysis.indexesTrigger != null) { + if (!addTrigger(analysis.indexesTrigger)) { + // This might be caused by this storage being created again recursively. throw new RepositoryException("Unable to add trigger for managing indexes"); } } // Okay, now start doing some damage. First, remove unnecessary indexes. - for (StorableIndex index : removeIndexSet) { + for (StorableIndex index : analysis.removeIndexSet) { removeIndex(index); } // Now add new indexes. - for (StorableIndex index : addIndexSet) { + for (StorableIndex index : analysis.addIndexSet) { registerIndex((ManagedIndex) mAllIndexInfoMap.get(index)); } - mQueryableIndexSet = queryableIndexSet; - mQueryEngine = new QueryEngine(masterStorage.getStorableType(), repository); + mQueryEngine = new QueryEngine(mMasterStorage.getStorableType(), mRepository); // Install triggers to manage derived properties in external Storables. - - Set> derivedToDependencies = - IndexAnalysis.gatherDerivedToDependencies(info); - - if (derivedToDependencies != null) { - for (ChainedProperty derivedTo : derivedToDependencies) { - addTrigger(new DerivedIndexesTrigger(repository, getStorableType(), derivedTo)); + if (analysis.derivedToDependencies != null) { + for (ChainedProperty derivedTo : analysis.derivedToDependencies) { + addTrigger(new DerivedIndexesTrigger(mRepository, getStorableType(), derivedTo)); } } } @@ -381,25 +167,6 @@ class IndexedStorage implements Storage, StorageAccess return mMasterStorage.removeTrigger(trigger); } - // Required by IndexInfoCapability. - public IndexInfo[] getIndexInfo() { - IndexInfo[] infos = new IndexInfo[mAllIndexInfoMap.size()]; - return mAllIndexInfoMap.values().toArray(infos); - } - - // Required by IndexEntryAccessCapability. - @SuppressWarnings("unchecked") - public IndexEntryAccessor[] getIndexEntryAccessors() { - List> accessors = - new ArrayList>(mAllIndexInfoMap.size()); - for (IndexInfo info : mAllIndexInfoMap.values()) { - if (info instanceof IndexEntryAccessor) { - accessors.add((IndexEntryAccessor) info); - } - } - return accessors.toArray(new IndexEntryAccessor[accessors.size()]); - } - // Required by StorageAccess. public QueryExecutorFactory getQueryExecutorFactory() { return mQueryEngine; diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java b/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java index a8f5788..2d0617e 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java @@ -82,20 +82,23 @@ class ManagedIndex implements IndexEntryAccessor { return naturalOrdering; } - private final IndexedStorage mIndexedStorage; + private final IndexedRepository mRepository; + private final Storage mMasterStorage; private final StorableIndex mIndex; private final IndexEntryGenerator mGenerator; private final Storage mIndexEntryStorage; private Query mSingleMatchQuery; - ManagedIndex(IndexedStorage indexedStorage, + ManagedIndex(IndexedRepository repository, + Storage masterStorage, StorableIndex index, IndexEntryGenerator generator, Storage indexEntryStorage) throws SupportException { - mIndexedStorage = indexedStorage; + mRepository = repository; + mMasterStorage = masterStorage; mIndex = index; mGenerator = generator; mIndexEntryStorage = indexEntryStorage; @@ -236,9 +239,6 @@ class ManagedIndex implements IndexEntryAccessor { * @param repo used to enter transactions */ void buildIndex(double desiredSpeed) throws RepositoryException { - final Repository repo = mIndexedStorage.mRepository; - final Storage masterStorage = mIndexedStorage.mMasterStorage; - final MergeSortBuffer buffer; final Comparator c; @@ -249,13 +249,13 @@ class ManagedIndex implements IndexEntryAccessor { // Need to explicitly order master query by primary key in order // for fetchAfter to work correctly in case corrupt records are // encountered. - masterQuery = masterStorage.query() - .orderBy(naturalOrdering(masterStorage.getStorableType())); + masterQuery = mMasterStorage.query() + .orderBy(naturalOrdering(mMasterStorage.getStorableType())); } // Quick check to see if any records exist in master. { - Transaction txn = repo.enterTransaction(IsolationLevel.READ_COMMITTED); + Transaction txn = mRepository.enterTransaction(IsolationLevel.READ_COMMITTED); try { Cursor cursor = masterQuery.fetch(); if (!cursor.hasNext()) { @@ -269,14 +269,14 @@ class ManagedIndex implements IndexEntryAccessor { // Enter top transaction with isolation level of none to make sure // preload operation does not run in a long nested transaction. - Transaction txn = repo.enterTopTransaction(IsolationLevel.NONE); + Transaction txn = mRepository.enterTopTransaction(IsolationLevel.NONE); try { Cursor cursor = masterQuery.fetch(); try { if (log.isInfoEnabled()) { StringBuilder b = new StringBuilder(); - b.append("Building index on "); - b.append(masterStorage.getStorableType().getName()); + b.append("Preparing index on "); + b.append(mMasterStorage.getStorableType().getName()); b.append(": "); try { mIndex.appendTo(b); @@ -350,7 +350,7 @@ class ManagedIndex implements IndexEntryAccessor { // fail, since unique index cannot be built. if (log.isInfoEnabled()) { - log.info("Verifying unique index"); + log.info("Verifying index"); } Object last = null; @@ -384,7 +384,7 @@ class ManagedIndex implements IndexEntryAccessor { long totalDeleted = 0; long totalProgress = 0; - txn = repo.enterTopTransaction(IsolationLevel.READ_COMMITTED); + txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED); try { txn.setForUpdate(true); @@ -463,7 +463,7 @@ class ManagedIndex implements IndexEntryAccessor { log.info(String.format(format, percent)); } - txn = repo.enterTopTransaction(IsolationLevel.READ_COMMITTED); + txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED); if (indexEntryCursor != null) { indexEntryCursor.close(); -- cgit v1.2.3