From b66fb3db6951b2b6d3ace72ca3e197c7c2048e86 Mon Sep 17 00:00:00 2001 From: "Brian S. O'Neill" Date: Thu, 18 Dec 2008 20:58:00 +0000 Subject: Fixes for excessive class generation and memory usage when opening multiple repositories. --- .../java/com/amazon/carbonado/layout/Layout.java | 43 ++++++ .../amazon/carbonado/raw/GenericStorableCodec.java | 122 ++++++++--------- .../java/com/amazon/carbonado/raw/RawSupport.java | 14 ++ .../com/amazon/carbonado/raw/StorableCodec.java | 14 ++ .../repo/indexed/IndexEntryGenerator.java | 23 ++-- .../carbonado/repo/sleepycat/BDBStorage.java | 11 +- .../SyntheticStorableReferenceAccess.java | 152 +++++++++++++++++++++ .../SyntheticStorableReferenceBuilder.java | 110 +++++---------- 8 files changed, 338 insertions(+), 151 deletions(-) create mode 100644 src/main/java/com/amazon/carbonado/synthetic/SyntheticStorableReferenceAccess.java diff --git a/src/main/java/com/amazon/carbonado/layout/Layout.java b/src/main/java/com/amazon/carbonado/layout/Layout.java index ca50172..b6c6c0d 100644 --- a/src/main/java/com/amazon/carbonado/layout/Layout.java +++ b/src/main/java/com/amazon/carbonado/layout/Layout.java @@ -394,6 +394,49 @@ public class Layout { return reconstructed; } + @Override + public int hashCode() { + long id = getLayoutID(); + return ((int) id) ^ (int) (id >> 32); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj instanceof Layout) { + Layout other = (Layout) obj; + try { + return mStoredLayout.equals(other.mStoredLayout) && equalLayouts(other); + } catch (FetchException e) { + return false; + } + } + return false; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append("Layout {type=").append(getStorableTypeName()); + b.append(", generation=").append(getGeneration()); + b.append(", properties={"); + try { + List props = getAllProperties(); + for (int i=0; i 0) { + b.append(", "); + } + b.append(props.get(i)); + } + } catch (FetchException e) { + b.append(e.toString()); + } + b.append("}}"); + return b.toString(); + } + /** * Returns true if the given layout matches this one. Layout ID, * generation, and creation info is not considered in the comparison. diff --git a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java index dd43515..4506bc1 100644 --- a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java +++ b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java @@ -18,7 +18,6 @@ package com.amazon.carbonado.raw; -import java.lang.ref.WeakReference; import java.lang.reflect.Method; import java.lang.reflect.UndeclaredThrowableException; import java.util.Map; @@ -51,6 +50,8 @@ import com.amazon.carbonado.info.StorableProperty; import com.amazon.carbonado.layout.Layout; import com.amazon.carbonado.gen.CodeBuilderUtil; +import com.amazon.carbonado.gen.StorableGenerator; +import com.amazon.carbonado.gen.TriggerSupport; import com.amazon.carbonado.util.ThrowUnchecked; import com.amazon.carbonado.util.QuickConstructorGenerator; @@ -64,10 +65,8 @@ import com.amazon.carbonado.util.QuickConstructorGenerator; */ public class GenericStorableCodec implements StorableCodec { private static final String BLANK_KEY_FIELD_NAME = "blankKey$"; - private static final String CODEC_FIELD_NAME = "codec$"; - private static final String ASSIGN_CODEC_METHOD_NAME = "assignCodec$"; - // Maps GenericEncodingStrategy instances to GenericStorableCodec instances. + // Maps GenericEncodingStrategy instances to Storable classes. private static final Map cCache = new SoftValuedHashMap(); /** @@ -100,7 +99,8 @@ public class GenericStorableCodec implements StorableCodec - (factory, + (key, + factory, encodingStrategy.getType(), storableImpl, encodingStrategy, @@ -128,30 +128,10 @@ public class GenericStorableCodec implements StorableCodec implements StorableCodec implements StorableCodec mType; private final Class mStorableClass; @@ -330,9 +306,6 @@ public class GenericStorableCodec implements StorableCodec mPrimaryKeyFactory; - // Maps OrderedProperty[] keys to SearchKeyFactory instances. - private final Map mSearchKeyFactories = new SoftValuedHashMap(); - private final Layout mLayout; private final RawSupport mSupport; @@ -340,11 +313,16 @@ public class GenericStorableCodec implements StorableCodec type, Class storableClass, GenericEncodingStrategy encodingStrategy, Layout layout, RawSupport support) { + mCodecKey = codecKey; mFactory = factory; mType = type; mStorableClass = storableClass; @@ -354,16 +332,6 @@ public class GenericStorableCodec implements StorableCodec implements StorableCodec getSearchKeyFactory(OrderedProperty[] properties) { // This KeyFactory makes arrays work as hashtable keys. - Object key = org.cojen.util.KeyFactory.createKey(properties); + Object key = KeyFactory.createKey(new Object[] {mCodecKey, properties}); - synchronized (mSearchKeyFactories) { - SearchKeyFactory factory = (SearchKeyFactory) mSearchKeyFactories.get(key); + synchronized (cCodecSearchKeyFactories) { + SearchKeyFactory factory = (SearchKeyFactory) cCodecSearchKeyFactories.get(key); if (factory == null) { factory = generateSearchKeyFactory(properties); - mSearchKeyFactories.put(key, factory); + cCodecSearchKeyFactories.put(key, factory); } return factory; } } + @Override + public void decode(S dest, int generation, byte[] data) throws CorruptEncodingException { + try { + getDecoder(generation).decode(dest, data); + } catch (CorruptEncodingException e) { + throw e; + } catch (RepositoryException e) { + throw new CorruptEncodingException(e); + } + } + /** * Returns a data decoder for the given generation. * * @throws FetchNoneException if generation is unknown + * @deprecated use direct decode method */ + @Deprecated public Decoder getDecoder(int generation) throws FetchNoneException, FetchException { try { synchronized (mLayout) { @@ -524,7 +505,22 @@ public class GenericStorableCodec implements StorableCodec decoder = (Decoder) decoders.get(generation); if (decoder == null) { - decoder = generateDecoder(generation); + synchronized (cCodecDecoders) { + Object key = KeyFactory.createKey(new Object[] {mCodecKey, generation}); + decoder = (Decoder) cCodecDecoders.get(key); + if (decoder == null) { + decoder = generateDecoder(generation); + cCodecDecoders.put(key, decoder); + } else { + // Confirm that layout still exists. + try { + mLayout.getGeneration(generation); + } catch (FetchNoneException e) { + cCodecDecoders.remove(key); + throw e; + } + } + } mDecoders.put(generation, decoder); } return decoder; @@ -915,6 +911,6 @@ public class GenericStorableCodec implements StorableCodec extends MasterSupport { * @throws PersistException if blob is unrecognized */ long getLocator(Clob clob) throws PersistException; + + /** + * Used for decoding different generations of Storable. If layout + * generations are not supported, simply throw a CorruptEncodingException. + * + * @param dest storable to receive decoded properties + * @param int storable layout generation number + * @param data decoded into properties, some of which may be dropped if + * destination storable doesn't have it + * @throws CorruptEncodingException if generation is unknown or if data cannot be decoded + * @since 1.2.1 + */ + void decode(S dest, int generation, byte[] data) throws CorruptEncodingException; } diff --git a/src/main/java/com/amazon/carbonado/raw/StorableCodec.java b/src/main/java/com/amazon/carbonado/raw/StorableCodec.java index fefb0fe..6874305 100644 --- a/src/main/java/com/amazon/carbonado/raw/StorableCodec.java +++ b/src/main/java/com/amazon/carbonado/raw/StorableCodec.java @@ -18,6 +18,7 @@ package com.amazon.carbonado.raw; +import com.amazon.carbonado.CorruptEncodingException; import com.amazon.carbonado.FetchException; import com.amazon.carbonado.Storable; import com.amazon.carbonado.info.StorableIndex; @@ -133,6 +134,19 @@ public interface StorableCodec { */ byte[] encodePrimaryKeyPrefix(); + /** + * Used for decoding different generations of Storable. If layout + * generations are not supported, simply throw a CorruptEncodingException. + * + * @param dest storable to receive decoded properties + * @param int storable layout generation number + * @param data decoded into properties, some of which may be dropped if + * destination storable doesn't have it + * @throws CorruptEncodingException if generation is unknown or if data cannot be decoded + * @since 1.2.1 + */ + void decode(S dest, int generation, byte[] data) throws CorruptEncodingException; + /** * Returns the default {@link RawSupport} object that is supplied to * Storable instances produced by this codec. diff --git a/src/main/java/com/amazon/carbonado/repo/indexed/IndexEntryGenerator.java b/src/main/java/com/amazon/carbonado/repo/indexed/IndexEntryGenerator.java index a126224..7fb80dc 100644 --- a/src/main/java/com/amazon/carbonado/repo/indexed/IndexEntryGenerator.java +++ b/src/main/java/com/amazon/carbonado/repo/indexed/IndexEntryGenerator.java @@ -28,6 +28,7 @@ import com.amazon.carbonado.Storable; import com.amazon.carbonado.SupportException; import com.amazon.carbonado.info.StorableIndex; import com.amazon.carbonado.info.StorableProperty; +import com.amazon.carbonado.synthetic.SyntheticStorableReferenceAccess; import com.amazon.carbonado.synthetic.SyntheticStorableReferenceBuilder; /** @@ -100,7 +101,7 @@ class IndexEntryGenerator { } } - private SyntheticStorableReferenceBuilder mBuilder; + private SyntheticStorableReferenceAccess mIndexAccess; /** * Convenience class for gluing new "builder" style synthetics to the traditional @@ -112,13 +113,17 @@ class IndexEntryGenerator { // but we have nothing better available to us Class type = index.getProperty(0).getEnclosingType(); - mBuilder = new SyntheticStorableReferenceBuilder(type, index.isUnique()); + SyntheticStorableReferenceBuilder builder = + new SyntheticStorableReferenceBuilder(type, index.isUnique()); for (int i=0; i { * @return class of index entry, which is a custom Storable */ public Class getIndexEntryClass() { - return mBuilder.getStorableClass(); + return mIndexAccess.getReferenceClass(); } /** @@ -138,7 +143,7 @@ class IndexEntryGenerator { * @param master master whose primary key properties will be set */ public void copyToMasterPrimaryKey(Storable indexEntry, S master) { - mBuilder.copyToMasterPrimaryKey(indexEntry, master); + mIndexAccess.copyToMasterPrimaryKey(indexEntry, master); } /** @@ -149,7 +154,7 @@ class IndexEntryGenerator { * @param master source of property values */ public void copyFromMaster(Storable indexEntry, S master) { - mBuilder.copyFromMaster(indexEntry, master); + mIndexAccess.copyFromMaster(indexEntry, master); } /** @@ -161,13 +166,13 @@ class IndexEntryGenerator { * @param master source of property values */ public boolean isConsistent(Storable indexEntry, S master) { - return mBuilder.isConsistent(indexEntry, master); + return mIndexAccess.isConsistent(indexEntry, master); } /** * Returns a comparator for ordering index entries. */ public Comparator getComparator() { - return mBuilder.getComparator(); + return mIndexAccess.getComparator(); } } diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java index f4a010b..1733167 100644 --- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java +++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java @@ -24,6 +24,7 @@ import java.util.Map; import org.cojen.classfile.TypeDesc; +import com.amazon.carbonado.CorruptEncodingException; import com.amazon.carbonado.Cursor; import com.amazon.carbonado.FetchDeadlockException; import com.amazon.carbonado.FetchException; @@ -104,7 +105,7 @@ abstract class BDBStorage implements Storage, Storag private final Class mType; /** Does most of the work in generating storables, used for preparing and querying */ - private StorableCodec mStorableCodec; + StorableCodec mStorableCodec; /** * Reference to an instance of Proxy, defined in this class, which binds @@ -135,7 +136,7 @@ abstract class BDBStorage implements Storage, Storag { mRepository = repository; mType = type; - mRawSupport = new Support(repository, this); + mRawSupport = new Support(repository, this); mTriggerManager = new TriggerManager(); try { // Ask if any lobs via static method first, to prevent stack @@ -1009,7 +1010,7 @@ abstract class BDBStorage implements Storage, Storag // Note: BDBStorage could just implement the RawSupport interface, but // then these hidden methods would be public. A simple cast of Storage to // RawSupport would expose them. - private static class Support implements RawSupport { + private class Support implements RawSupport { private final BDBRepository mRepository; private final BDBStorage mStorage; private Map> mProperties; @@ -1128,6 +1129,10 @@ abstract class BDBStorage implements Storage, Storag return mStorage.getLocator(clob); } + public void decode(S dest, int generation, byte[] data) throws CorruptEncodingException { + mStorableCodec.decode(dest, generation, data); + } + public SequenceValueProducer getSequenceValueProducer(String name) throws PersistException { diff --git a/src/main/java/com/amazon/carbonado/synthetic/SyntheticStorableReferenceAccess.java b/src/main/java/com/amazon/carbonado/synthetic/SyntheticStorableReferenceAccess.java new file mode 100644 index 0000000..778b4ac --- /dev/null +++ b/src/main/java/com/amazon/carbonado/synthetic/SyntheticStorableReferenceAccess.java @@ -0,0 +1,152 @@ +/* + * Copyright 2008 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.synthetic; + +import java.lang.reflect.Method; +import java.lang.reflect.UndeclaredThrowableException; + +import java.util.Comparator; +import java.util.Iterator; + +import com.amazon.carbonado.Storable; + +import com.amazon.carbonado.cursor.SortedCursor; + +import com.amazon.carbonado.util.ThrowUnchecked; + +/** + * Provides access to the generated storable reference class and utility + * methods. + * + * @author Brian S O'Neill + * @see SyntheticStorableReferenceBuilder + * @since 1.2.1 + */ +public class SyntheticStorableReferenceAccess { + private final Class mMasterClass; + private final Class mReferenceClass; + + private final Comparator mComparator; + + private final Method mCopyFromMasterMethod; + private final Method mIsConsistentMethod; + private final Method mCopyToMasterPkMethod; + + SyntheticStorableReferenceAccess(Class masterClass, + Class referenceClass, + SyntheticStorableReferenceBuilder builder) + { + mMasterClass = masterClass; + mReferenceClass = referenceClass; + + // We need a comparator which follows the same order as the generated + // storable. + SyntheticKey pk = builder.mPrimaryKey; + String[] orderBy = new String[pk.getPropertyCount()]; + int i=0; + Iterator it = pk.getProperties(); + while (it.hasNext()) { + orderBy[i++] = it.next(); + } + mComparator = SortedCursor.createComparator(referenceClass, orderBy); + + try { + mCopyFromMasterMethod = + referenceClass.getMethod(builder.mCopyFromMasterMethodName, masterClass); + + mIsConsistentMethod = + referenceClass.getMethod(builder.mIsConsistentMethodName, masterClass); + + mCopyToMasterPkMethod = + referenceClass.getMethod(builder.mCopyToMasterPkMethodName, masterClass); + } catch (NoSuchMethodException e) { + throw new UndeclaredThrowableException(e); + } + } + + /** + * Returns the storable class which is referenced. + */ + public Class getMasterClass() { + return mMasterClass; + } + + /** + * Returns the generated storable reference class. + */ + public Class getReferenceClass() { + return mReferenceClass; + } + + /** + * Returns a comparator for ordering storable reference instances. This + * order matches the primary key of the master storable. + */ + public Comparator getComparator() { + return mComparator; + } + + /** + * Sets all the primary key properties of the given master, using the + * applicable properties of the given reference. + * + * @param reference source of property values + * @param master master whose primary key properties will be set + */ + public void copyToMasterPrimaryKey(Storable reference, S master) { + try { + mCopyToMasterPkMethod.invoke(reference, master); + } catch (Exception e) { + ThrowUnchecked.fireFirstDeclaredCause(e); + } + } + + /** + * Sets all the properties of the given reference, using the applicable + * properties of the given master. + * + * @param reference reference whose properties will be set + * @param master source of property values + */ + public void copyFromMaster(Storable reference, S master) { + try { + mCopyFromMasterMethod.invoke(reference, master); + } catch (Exception e) { + ThrowUnchecked.fireFirstDeclaredCause(e); + } + } + + /** + * Returns true if the properties of the given reference match those + * contained in the master, excluding any version property. This will + * always return true after a call to copyFromMaster. + * + * @param reference reference whose properties will be tested + * @param master source of property values + */ + public boolean isConsistent(Storable reference, S master) { + try { + return (Boolean) mIsConsistentMethod.invoke(reference, master); + } catch (Exception e) { + ThrowUnchecked.fireFirstDeclaredCause(e); + // Not reached. + return false; + } + } +} diff --git a/src/main/java/com/amazon/carbonado/synthetic/SyntheticStorableReferenceBuilder.java b/src/main/java/com/amazon/carbonado/synthetic/SyntheticStorableReferenceBuilder.java index 4fddcd5..a06c557 100644 --- a/src/main/java/com/amazon/carbonado/synthetic/SyntheticStorableReferenceBuilder.java +++ b/src/main/java/com/amazon/carbonado/synthetic/SyntheticStorableReferenceBuilder.java @@ -18,7 +18,6 @@ package com.amazon.carbonado.synthetic; import java.lang.reflect.Method; -import java.lang.reflect.UndeclaredThrowableException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -29,13 +28,11 @@ import java.util.Set; import com.amazon.carbonado.Storable; import com.amazon.carbonado.SupportException; -import com.amazon.carbonado.cursor.SortedCursor; import com.amazon.carbonado.info.Direction; import com.amazon.carbonado.info.StorableInfo; import com.amazon.carbonado.info.StorableIntrospector; import com.amazon.carbonado.info.StorableProperty; import com.amazon.carbonado.gen.CodeBuilderUtil; -import com.amazon.carbonado.util.ThrowUnchecked; import org.cojen.classfile.ClassFile; import org.cojen.classfile.CodeBuilder; @@ -80,7 +77,7 @@ public class SyntheticStorableReferenceBuilder // Information about the storable from which this one is derived // private StorableInfo mBaseStorableInfo; - private Class mMasterStorableClass; + private Class mMasterStorableClass; // Stashed copy of the results of calling StorableIntrospector.examine(...) // on the master storable class. @@ -90,7 +87,7 @@ public class SyntheticStorableReferenceBuilder private SyntheticStorableBuilder mBuilder; // Primary key of generated storable. - private SyntheticKey mPrimaryKey; + SyntheticKey mPrimaryKey; // Elements added to primary key to ensure uniqueness private Set mExtraPkProps; @@ -99,9 +96,6 @@ public class SyntheticStorableReferenceBuilder // uniquely identify a unique instance of the referent. private boolean mIsUnique = true; - // The result of building - private Class mSyntheticClass; - // The list of properties explicitly added to this reference builder private List mUserProps; @@ -110,16 +104,12 @@ public class SyntheticStorableReferenceBuilder // are retrieved from the master. private List mCommonProps; - private String mCopyFromMasterMethodName; - private Method mCopyFromMasterMethod; - - private String mIsConsistentMethodName; - private Method mIsConsistentMethod; - - private String mCopyToMasterPkMethodName; - private Method mCopyToMasterPkMethod; + String mCopyFromMasterMethodName; + String mIsConsistentMethodName; + String mCopyToMasterPkMethodName; - private Comparator mComparator; + // The result of building. + private SyntheticStorableReferenceAccess mReferenceAccess; /** * @param storableClass @@ -210,26 +200,27 @@ public class SyntheticStorableReferenceBuilder return cfg; } + /** + * Build and return access to the generated storable reference class. + * + * @since 1.2.1 + */ + public SyntheticStorableReferenceAccess getReferenceAccess() { + if (mReferenceAccess == null) { + Class referenceClass = mBuilder.getStorableClass(); + mReferenceAccess = new SyntheticStorableReferenceAccess + (mMasterStorableClass, referenceClass, this); + } + return mReferenceAccess; + } + /* * (non-Javadoc) * * @see com.amazon.carbonado.synthetic.SyntheticBuilder#getStorableClass() */ public Class getStorableClass() throws IllegalStateException { - if (mSyntheticClass == null) { - mSyntheticClass = mBuilder.getStorableClass(); - - // We need a comparator which follows the same order as the generated - // storable. We can't construct it until we get here. - String[] orderBy = new String[mPrimaryKey.getPropertyCount()]; - int i=0; - Iterator it = mPrimaryKey.getProperties(); - while (it.hasNext()) { - orderBy[i++] = it.next(); - } - mComparator = SortedCursor.createComparator(mSyntheticClass, orderBy); - } - return mSyntheticClass; + return getReferenceAccess().getReferenceClass(); } /* @@ -356,22 +347,11 @@ public class SyntheticStorableReferenceBuilder * * @param indexEntry source of property values * @param master master whose primary key properties will be set + * @deprecated call getReferenceAccess */ + @Deprecated public void copyToMasterPrimaryKey(Storable indexEntry, S master) { - if (mCopyToMasterPkMethod == null) { - try { - mCopyToMasterPkMethod = - mSyntheticClass.getMethod(mCopyToMasterPkMethodName, mMasterStorableClass); - } catch (NoSuchMethodException e) { - throw new UndeclaredThrowableException(e); - } - } - - try { - mCopyToMasterPkMethod.invoke(indexEntry, master); - } catch (Exception e) { - ThrowUnchecked.fireFirstDeclaredCause(e); - } + getReferenceAccess().copyToMasterPrimaryKey(indexEntry, master); } /** @@ -380,22 +360,11 @@ public class SyntheticStorableReferenceBuilder * * @param indexEntry index entry whose properties will be set * @param master source of property values + * @deprecated call getReferenceAccess */ + @Deprecated public void copyFromMaster(Storable indexEntry, S master) { - if (mCopyFromMasterMethod == null) { - try { - mCopyFromMasterMethod = - mSyntheticClass.getMethod(mCopyFromMasterMethodName, mMasterStorableClass); - } catch (NoSuchMethodException e) { - throw new UndeclaredThrowableException(e); - } - } - - try { - mCopyFromMasterMethod.invoke(indexEntry, master); - } catch (Exception e) { - ThrowUnchecked.fireFirstDeclaredCause(e); - } + getReferenceAccess().copyFromMaster(indexEntry, master); } /** @@ -407,31 +376,20 @@ public class SyntheticStorableReferenceBuilder * index entry whose properties will be tested * @param master * source of property values + * @deprecated call getReferenceAccess */ + @Deprecated public boolean isConsistent(Storable indexEntry, S master) { - if (mIsConsistentMethod == null) { - try { - mIsConsistentMethod = - mSyntheticClass.getMethod(mIsConsistentMethodName, mMasterStorableClass); - } catch (NoSuchMethodException e) { - throw new UndeclaredThrowableException(e); - } - } - - try { - return (Boolean) mIsConsistentMethod.invoke(indexEntry, master); - } catch (Exception e) { - ThrowUnchecked.fireFirstDeclaredCause(e); - // Not reached. - return false; - } + return getReferenceAccess().isConsistent(indexEntry, master); } /** * Returns a comparator for ordering index entries. + * @deprecated call getReferenceAccess */ + @Deprecated public Comparator getComparator() { - return mComparator; + return getReferenceAccess().getComparator(); } /** -- cgit v1.2.3