From 97af4be638e371a2f693bde2798fc233a143f3f9 Mon Sep 17 00:00:00 2001 From: "Brian S. O'Neill" Date: Sun, 29 Apr 2007 17:47:50 +0000 Subject: Merged in support for derived properties. --- .../repo/indexed/DependentStorableFetcher.java | 174 +++++++++++++++++++++ .../repo/indexed/DerivedIndexesTrigger.java | 164 +++++++++++++++++++ .../carbonado/repo/indexed/IndexAnalysis.java | 132 ++++++++++++++++ .../carbonado/repo/indexed/IndexEntryAccessor.java | 2 +- .../carbonado/repo/indexed/IndexedRepository.java | 9 +- .../carbonado/repo/indexed/IndexedStorage.java | 25 ++- .../carbonado/repo/indexed/ManagedIndex.java | 23 ++- 7 files changed, 514 insertions(+), 15 deletions(-) create mode 100644 src/main/java/com/amazon/carbonado/repo/indexed/DependentStorableFetcher.java create mode 100644 src/main/java/com/amazon/carbonado/repo/indexed/DerivedIndexesTrigger.java create mode 100644 src/main/java/com/amazon/carbonado/repo/indexed/IndexAnalysis.java (limited to 'src/main/java/com/amazon/carbonado/repo/indexed') diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/DependentStorableFetcher.java b/src/main/java/com/amazon/carbonado/repo/indexed/DependentStorableFetcher.java new file mode 100644 index 0000000..4c24c04 --- /dev/null +++ b/src/main/java/com/amazon/carbonado/repo/indexed/DependentStorableFetcher.java @@ -0,0 +1,174 @@ +/* + * Copyright 2007 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.repo.indexed; + +import java.util.Arrays; +import java.util.ArrayList; +import java.util.List; + +import org.cojen.util.BeanPropertyAccessor; + +import com.amazon.carbonado.Cursor; +import com.amazon.carbonado.FetchException; +import com.amazon.carbonado.Query; +import com.amazon.carbonado.RepositoryException; +import com.amazon.carbonado.Storable; +import com.amazon.carbonado.Storage; +import com.amazon.carbonado.SupportException; +import com.amazon.carbonado.Transaction; + +import com.amazon.carbonado.filter.Filter; +import com.amazon.carbonado.filter.RelOp; + +import com.amazon.carbonado.info.ChainedProperty; +import com.amazon.carbonado.info.StorableProperty; + +/** + * Fetches Storables that have indexed derived-to properties which depend on S. + * + * @author Brian S O'Neill + */ +class DependentStorableFetcher { + private final IndexedRepository mRepository; + private final IndexEntryAccessor[] mIndexEntryAccessors; + private final Query mQuery; + private final String[] mJoinProperties; + private final BeanPropertyAccessor mPropertyAccessor; + + /** + * @param derivedTo special chained property from StorableProperty.getDerivedToProperties + */ + DependentStorableFetcher(IndexedRepository repository, + Class sType, ChainedProperty derivedTo) + throws RepositoryException + { + if (derivedTo.getChainCount() == 0) { + throw new IllegalArgumentException(); + } + if (derivedTo.getLastProperty().getType() != sType) { + throw new IllegalArgumentException(); + } + if (!derivedTo.getLastProperty().isJoin()) { + throw new IllegalArgumentException(); + } + + Class dType = derivedTo.getPrimeProperty().getEnclosingType(); + + // Find the indexes that contain the prime derivedTo property. + List> accessorList = new ArrayList>(); + for (IndexEntryAccessor acc : repository.getIndexEntryAccessors(dType)) { + for (String indexPropName : acc.getPropertyNames()) { + if (indexPropName.equals(derivedTo.getPrimeProperty().getName())) { + accessorList.add(acc); + break; + } + } + } + + if (accessorList.size() == 0) { + throw new SupportException + ("Unable to find index accessors for derived-to property: " + derivedTo + + ", enclosing type: " + dType); + } + + // Build a query on D joined to S. + + StorableProperty join = (StorableProperty) derivedTo.getLastProperty(); + + ChainedProperty base; + if (derivedTo.getChainCount() <= 1) { + base = null; + } else { + base = derivedTo.tail().trim(); + } + + int joinElementCount = join.getJoinElementCount(); + String[] joinProperties = new String[joinElementCount]; + + Filter dFilter = Filter.getOpenFilter(dType); + for (int i=0; i element = join.getInternalJoinElement(i); + joinProperties[i] = element.getName(); + if (base == null) { + dFilter = dFilter.and(element.getName(), RelOp.EQ); + } else { + dFilter = dFilter.and(base.append(element).toString(), RelOp.EQ); + } + } + + mRepository = repository; + mIndexEntryAccessors = accessorList.toArray(new IndexEntryAccessor[accessorList.size()]); + mQuery = repository.storageFor(dType).query(dFilter); + mJoinProperties = joinProperties; + mPropertyAccessor = BeanPropertyAccessor.forClass(sType); + } + + public Transaction enterTransaction() { + return mRepository.enterTransaction(); + } + + public Cursor fetchDependenentStorables(S storable) throws FetchException { + Query query = mQuery; + for (String property : mJoinProperties) { + query = query.with(mPropertyAccessor.getPropertyValue(storable, property)); + } + return query.fetch(); + } + + /** + * @return amount added to list + */ + public int createIndexEntries(D master, List indexEntries) { + IndexEntryAccessor[] accessors = mIndexEntryAccessors; + int length = accessors.length; + for (int i=0; i extends Trigger { + private final DependentStorableFetcher mFetcher; + + /** + * @param derivedTo special chained property from StorableProperty.getDerivedToProperties + */ + DerivedIndexesTrigger(IndexedRepository repository, + Class sType, ChainedProperty derivedTo) + throws RepositoryException + { + this(new DependentStorableFetcher(repository, sType, derivedTo)); + } + + DerivedIndexesTrigger(DependentStorableFetcher fetcher) { + mFetcher = fetcher; + } + + @Override + public Object beforeInsert(S storable) throws PersistException { + return createDependentIndexEntries(storable); + } + + @Override + public void afterInsert(S storable, Object state) throws PersistException { + updateValues(storable, state); + } + + @Override + public Object beforeUpdate(S storable) throws PersistException { + return createDependentIndexEntries(storable); + } + + @Override + public void afterUpdate(S storable, Object state) throws PersistException { + updateValues(storable, state); + } + + @Override + public Object beforeDelete(S storable) throws PersistException { + try { + if (storable.copy().tryLoad()) { + return createDependentIndexEntries(storable); + } + } catch (FetchException e) { + throw e.toPersistException(); + } + return null; + } + + @Override + public void afterDelete(S storable, Object state) throws PersistException { + updateValues(storable, state); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj instanceof DerivedIndexesTrigger) { + DerivedIndexesTrigger other = (DerivedIndexesTrigger) obj; + return mFetcher.equals(other.mFetcher); + } + return false; + } + + private List createDependentIndexEntries(S storable) throws PersistException { + List dependentIndexEntries = new ArrayList(); + createDependentIndexEntries(storable, dependentIndexEntries); + return dependentIndexEntries; + } + + private void createDependentIndexEntries(S storable, List dependentIndexEntries) + throws PersistException + { + try { + Transaction txn = mFetcher.enterTransaction(); + try { + // Make sure write lock is acquired when reading dependencies + // since they might be updated later. Locks are held after this + // transaction exits since it is nested in the trigger's transaction. + txn.setForUpdate(true); + + Cursor dependencies = mFetcher.fetchDependenentStorables(storable); + try { + while (dependencies.hasNext()) { + mFetcher.createIndexEntries(dependencies.next(), dependentIndexEntries); + } + } finally { + dependencies.close(); + } + } finally { + txn.exit(); + } + } catch (FetchException e) { + throw e.toPersistException(); + } + } + + private void updateValues(S storable, Object state) throws PersistException { + if (state == null) { + return; + } + + List oldIndexEntries = (List) state; + int size = oldIndexEntries.size(); + + List newIndexEntries = new ArrayList(size); + createDependentIndexEntries(storable, newIndexEntries); + + if (size != newIndexEntries.size()) { + // This is not expected to happen. + throw new PersistException("Amount of affected dependent indexes changed: " + + size + " != " + newIndexEntries.size()); + } + + for (int i=0; i StorableIndexSet gatherDesiredIndexes(StorableInfo info) { + StorableIndexSet indexSet = new StorableIndexSet(); + indexSet.addIndexes(info); + indexSet.addAlternateKeys(info); + + // If any join properties are used by indexed derived properties, make + // sure join internal properties are indexed. + + for (StorableProperty property : info.getAllProperties().values()) { + if (!isJoinAndUsedByIndexedDerivedProperty(property)) { + continue; + } + + // Internal properties of join need to be indexed. Check if a + // suitable index exists before defining a new one. + + Filter filter = Filter.getOpenFilter(info.getStorableType()); + for (int i=property.getJoinElementCount(); --i>=0; ) { + filter = filter.and(property.getInternalJoinElement(i).getName(), RelOp.EQ); + } + + for (int i=info.getIndexCount(); --i>=0; ) { + FilteringScore score = FilteringScore.evaluate(info.getIndex(i), filter); + if (score.getIdentityCount() == property.getJoinElementCount()) { + // Suitable index already exists. + continue; + } + } + + Direction[] directions = new Direction[property.getJoinElementCount()]; + Arrays.fill(directions, Direction.UNSPECIFIED); + + StorableIndex index = + new StorableIndex(property.getInternalJoinElements(), directions); + + indexSet.add(index); + } + + return indexSet; + } + + static boolean isUsedByIndex(StorableProperty property) { + StorableInfo info = StorableIntrospector.examine(property.getEnclosingType()); + for (int i=info.getIndexCount(); --i>=0; ) { + StorableIndex index = info.getIndex(i); + int propertyCount = index.getPropertyCount(); + for (int j=0; j property) { + if (property.isJoin()) { + for (ChainedProperty derivedTo : property.getDerivedToProperties()) { + if (isUsedByIndex(derivedTo.getPrimeProperty())) { + return true; + } + } + } + return false; + } + + /** + * Returns derived-to properties in external storables that are used by indexes. + * + * @return null if none + */ + static Set> gatherDerivedToDependencies(StorableInfo info) { + Set> set = null; + for (StorableProperty property : info.getAllProperties().values()) { + for (ChainedProperty derivedTo : property.getDerivedToProperties()) { + if (derivedTo.getChainCount() > 0 && isUsedByIndex(derivedTo.getPrimeProperty())) { + if (set == null) { + set = new HashSet>(); + } + set.add(derivedTo); + } + } + } + return set; + } +} diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexEntryAccessor.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexEntryAccessor.java index 0c1e6b8..5f27aa8 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/IndexEntryAccessor.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexEntryAccessor.java @@ -59,7 +59,7 @@ public interface IndexEntryAccessor extends IndexInfo { /** * Returns true if the properties of the given index entry match those * contained in the master, exluding any version property. This will always - * return true after a call to setAllProperties. + * return true after a call to copyFromMaster. * * @param indexEntry index entry whose properties will be tested * @param master source of property values diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedRepository.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedRepository.java index 097185a..6bea049 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedRepository.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedRepository.java @@ -86,7 +86,7 @@ class IndexedRepository implements Repository, if (Unindexed.class.isAssignableFrom(type)) { // Verify no indexes. - int indexCount = IndexedStorage + int indexCount = IndexAnalysis .gatherDesiredIndexes(StorableIntrospector.examine(type)).size(); if (indexCount > 0) { throw new MalformedTypeException @@ -152,7 +152,12 @@ class IndexedRepository implements Repository, getIndexEntryAccessors(Class storableType) throws RepositoryException { - return ((IndexedStorage) storageFor(storableType)).getIndexEntryAccessors(); + Storage storage = storageFor(storableType); + if (storage instanceof IndexedStorage) { + return ((IndexedStorage) storage).getIndexEntryAccessors(); + } else { + return new IndexEntryAccessor[0]; + } } public String[] getUserStorableTypeNames() throws RepositoryException { diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java index fe0cfe8..28a9d35 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexedStorage.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -32,9 +33,11 @@ import com.amazon.carbonado.FetchException; import com.amazon.carbonado.IsolationLevel; import com.amazon.carbonado.PersistException; import com.amazon.carbonado.Query; +import com.amazon.carbonado.Repository; import com.amazon.carbonado.RepositoryException; import com.amazon.carbonado.Storable; import com.amazon.carbonado.Storage; +import com.amazon.carbonado.SupportException; import com.amazon.carbonado.Transaction; import com.amazon.carbonado.Trigger; import com.amazon.carbonado.capability.IndexInfo; @@ -45,6 +48,7 @@ import com.amazon.carbonado.cursor.MergeSortBuffer; import com.amazon.carbonado.filter.Filter; +import com.amazon.carbonado.info.ChainedProperty; import com.amazon.carbonado.info.Direction; import com.amazon.carbonado.info.StorableInfo; import com.amazon.carbonado.info.StorableIntrospector; @@ -53,6 +57,7 @@ import com.amazon.carbonado.info.StorableIndex; import com.amazon.carbonado.cursor.SortBuffer; import com.amazon.carbonado.qe.BoundaryType; +import com.amazon.carbonado.qe.FilteringScore; import com.amazon.carbonado.qe.QueryEngine; import com.amazon.carbonado.qe.QueryExecutorFactory; import com.amazon.carbonado.qe.StorageAccess; @@ -69,13 +74,6 @@ import static com.amazon.carbonado.repo.indexed.ManagedIndex.*; * @author Brian S O'Neill */ class IndexedStorage implements Storage, StorageAccess { - static StorableIndexSet gatherDesiredIndexes(StorableInfo info) { - StorableIndexSet indexSet = new StorableIndexSet(); - indexSet.addIndexes(info); - indexSet.addAlternateKeys(info); - return indexSet; - } - final IndexedRepository mRepository; final Storage mMasterStorage; @@ -102,7 +100,7 @@ class IndexedStorage implements Storage, StorageAccess // The set of indexes that the Storable defines, reduced. final StorableIndexSet desiredIndexSet; { - desiredIndexSet = gatherDesiredIndexes(info); + desiredIndexSet = IndexAnalysis.gatherDesiredIndexes(info); desiredIndexSet.reduce(Direction.ASCENDING); } @@ -299,6 +297,17 @@ class IndexedStorage implements Storage, StorageAccess mQueryableIndexSet = queryableIndexSet; mQueryEngine = new QueryEngine(masterStorage.getStorableType(), repository); + + // Install triggers to manage derived properties in external Storables. + + Set> derivedToDependencies = + IndexAnalysis.gatherDerivedToDependencies(info); + + if (derivedToDependencies != null) { + for (ChainedProperty derivedTo : derivedToDependencies) { + addTrigger(new DerivedIndexesTrigger(repository, getStorableType(), derivedTo)); + } + } } public Class getStorableType() { diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java b/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java index 2e4cea3..b1e76cc 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/ManagedIndex.java @@ -18,6 +18,8 @@ package com.amazon.carbonado.repo.indexed; +import java.lang.reflect.UndeclaredThrowableException; + import java.util.Comparator; import org.apache.commons.logging.Log; @@ -451,10 +453,23 @@ class ManagedIndex implements IndexEntryAccessor { } } - private Storable makeIndexEntry(S userStorable) { - Storable indexEntry = mIndexEntryStorage.prepare(); - mGenerator.copyFromMaster(indexEntry, userStorable); - return indexEntry; + private Storable makeIndexEntry(S userStorable) throws PersistException { + try { + Storable indexEntry = mIndexEntryStorage.prepare(); + mGenerator.copyFromMaster(indexEntry, userStorable); + return indexEntry; + } catch (UndeclaredThrowableException e) { + Throwable cause = e.getCause(); + if (cause instanceof PersistException) { + throw (PersistException) cause; + } + throw new PersistException(cause); + } catch (Exception e) { + if (e instanceof PersistException) { + throw (PersistException) e; + } + throw new PersistException(e); + } } /** Assumes caller is in a transaction */ -- cgit v1.2.3