From 55d57b7d9f075c275e4fe30d29577a5b914f05db Mon Sep 17 00:00:00 2001 From: "Brian S. O'Neill" Date: Thu, 5 Nov 2009 23:58:22 +0000 Subject: Add support for compressed records. --- .../java/com/amazon/carbonado/layout/Layout.java | 37 +++++- .../com/amazon/carbonado/layout/LayoutFactory.java | 37 +++++- .../com/amazon/carbonado/layout/LayoutOptions.java | 131 +++++++++++++++++++++ .../com/amazon/carbonado/layout/StoredLayout.java | 8 -- .../carbonado/layout/StoredLayoutProperty.java | 11 -- .../carbonado/raw/CompressedEncodingStrategy.java | 97 +++++++++++++++ .../raw/CompressedStorableCodecFactory.java | 84 +++++++++++++ .../com/amazon/carbonado/raw/CompressionType.java | 35 ++++++ .../carbonado/raw/GenericEncodingStrategy.java | 35 +++++- .../amazon/carbonado/raw/GenericStorableCodec.java | 6 +- .../carbonado/raw/GenericStorableCodecFactory.java | 28 ++++- .../com/amazon/carbonado/raw/GzipCompressor.java | 101 ++++++++++++++++ .../amazon/carbonado/raw/StorableCodecFactory.java | 8 ++ .../carbonado/repo/sleepycat/BDBRepository.java | 37 +++--- .../repo/sleepycat/BDBRepositoryBuilder.java | 45 ++++++- .../carbonado/repo/sleepycat/BDBStorage.java | 10 +- 16 files changed, 659 insertions(+), 51 deletions(-) create mode 100644 src/main/java/com/amazon/carbonado/layout/LayoutOptions.java create mode 100644 src/main/java/com/amazon/carbonado/raw/CompressedEncodingStrategy.java create mode 100644 src/main/java/com/amazon/carbonado/raw/CompressedStorableCodecFactory.java create mode 100644 src/main/java/com/amazon/carbonado/raw/CompressionType.java create mode 100644 src/main/java/com/amazon/carbonado/raw/GzipCompressor.java (limited to 'src') diff --git a/src/main/java/com/amazon/carbonado/layout/Layout.java b/src/main/java/com/amazon/carbonado/layout/Layout.java index b6c6c0d..7aea758 100644 --- a/src/main/java/com/amazon/carbonado/layout/Layout.java +++ b/src/main/java/com/amazon/carbonado/layout/Layout.java @@ -18,6 +18,7 @@ package com.amazon.carbonado.layout; +import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; @@ -32,6 +33,7 @@ import org.apache.commons.logging.LogFactory; import org.cojen.util.SoftValuedHashMap; +import com.amazon.carbonado.CorruptEncodingException; import com.amazon.carbonado.Cursor; import com.amazon.carbonado.FetchException; import com.amazon.carbonado.FetchNoneException; @@ -131,22 +133,36 @@ public class Layout { private final LayoutFactory mLayoutFactory; private final StoredLayout mStoredLayout; + private final LayoutOptions mOptions; private transient List mAllProperties; /** * Creates a Layout around an existing storable. */ - Layout(LayoutFactory factory, StoredLayout storedLayout) { + Layout(LayoutFactory factory, StoredLayout storedLayout) throws CorruptEncodingException { mLayoutFactory = factory; mStoredLayout = storedLayout; + + byte[] extra = storedLayout.getExtraData(); + if (extra == null) { + mOptions = null; + } else { + mOptions = new LayoutOptions(); + try { + mOptions.decode(extra); + } catch (IOException e) { + throw new CorruptEncodingException(e); + } + mOptions.readOnly(); + } } /** * Copies layout information into freshly prepared storables. Call insert * (on this class) to persist them. */ - Layout(LayoutFactory factory, StorableInfo info, long layoutID) { + Layout(LayoutFactory factory, StorableInfo info, LayoutOptions options, long layoutID) { mLayoutFactory = factory; StoredLayout storedLayout = factory.mLayoutStorage.prepare(); @@ -168,6 +184,14 @@ public class Layout { // Can't get host, no big deal. } + if (options == null) { + mOptions = null; + } else { + options.readOnly(); + storedLayout.setExtraData(options.encode()); + mOptions = options; + } + Collection> properties = info.getAllProperties().values(); List list = new ArrayList(properties.size()); int ordinal = 0; @@ -268,6 +292,15 @@ public class Layout { return mStoredLayout.getCreationHost(); } + /** + * Returns additional options, or null if none. + * + * @return read-only object or null + */ + public LayoutOptions getOptions() { + return mOptions; + } + /** * Returns the layout for a particular generation of this layout's type. * diff --git a/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java b/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java index 1b2f0fc..54b6933 100644 --- a/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java +++ b/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java @@ -87,6 +87,23 @@ public class LayoutFactory implements LayoutCapability { public Layout layoutFor(Class type) throws FetchException, PersistException { + return layoutFor(type, null); + } + + /** + * Returns the layout matching the current definition of the given type. + * + * @throws PersistException if type represents a new generation, but + * persisting this information failed + */ + public Layout layoutFor(Class type, LayoutOptions options) + throws FetchException, PersistException + { + if (options != null) { + // Make side-effect consistently applied. + options.readOnly(); + } + synchronized (this) { if (mReconstructed != null) { Layout layout = mReconstructed.get(type); @@ -120,10 +137,10 @@ public class LayoutFactory implements LayoutCapability { for (int i=0; i info, int multiplier) { + private long mixInHash(long hash, StorableInfo info, LayoutOptions options, int multiplier) + { hash = mixInHash(hash, info.getStorableType().getName(), multiplier); + hash = mixInHash(hash, options, multiplier); for (StorableProperty property : info.getAllProperties().values()) { if (!property.isJoin()) { @@ -314,6 +333,18 @@ public class LayoutFactory implements LayoutCapability { return hash; } + private long mixInHash(long hash, LayoutOptions options, int multiplier) { + if (options != null) { + byte[] data = options.encode(); + if (data != null) { + for (int b : data) { + hash = hash * multiplier + (b & 0xff); + } + } + } + return hash; + } + /** * Returns an annotation hash code using a algorithm similar to the * default. The difference is in the handling of class and enum values. The diff --git a/src/main/java/com/amazon/carbonado/layout/LayoutOptions.java b/src/main/java/com/amazon/carbonado/layout/LayoutOptions.java new file mode 100644 index 0000000..1e57407 --- /dev/null +++ b/src/main/java/com/amazon/carbonado/layout/LayoutOptions.java @@ -0,0 +1,131 @@ +/* + * Copyright 2009 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.layout; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +import java.util.HashMap; +import java.util.Map; + +/** + * Extra options encoded with a Storable layout. + * + * @author Brian S O'Neill + */ +public class LayoutOptions { + /** + * Data is compression algorithm name, encoded by DataOutput.writeUTF. + */ + static final byte COMPRESSION_TYPE = 1; + + private final Map mData; + + private boolean mReadOnly; + + public LayoutOptions() { + mData = new HashMap(1); + } + + /** + * @return null if not compressed + */ + public synchronized String getCompressionType() { + return (String) mData.get(COMPRESSION_TYPE); + } + + /** + * @param type null if not compressed + */ + public void setCompressionType(String type) { + put(COMPRESSION_TYPE, type); + } + + private synchronized void put(byte op, Object value) { + if (mReadOnly) { + throw new IllegalStateException("Options are read only"); + } + if (value == null) { + mData.remove(op); + } else { + mData.put(op, value); + } + } + + synchronized void readOnly() { + mReadOnly = true; + } + + /** + * @return null if empty + */ + synchronized byte[] encode() { + if (mData.isEmpty()) { + return null; + } + + try { + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + DataOutputStream dout = new DataOutputStream(bout); + + for (Map.Entry entry : mData.entrySet()) { + switch (entry.getKey()) { + default: + break; + case COMPRESSION_TYPE: + dout.write(COMPRESSION_TYPE); + dout.writeUTF((String) entry.getValue()); + } + } + + dout.close(); + return bout.toByteArray(); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + /** + * @param source can be null if empty + */ + synchronized void decode(byte[] source) throws IOException { + mData.clear(); + + if (source == null || source.length == 0) { + return; + } + + ByteArrayInputStream bin = new ByteArrayInputStream(source); + DataInputStream din = new DataInputStream(bin); + + while (bin.available() > 0) { + byte op = din.readByte(); + switch (op) { + default: + throw new IOException("Unknown extra data type: " + op); + case COMPRESSION_TYPE: + mData.put(COMPRESSION_TYPE, din.readUTF()); + break; + } + } + } +} diff --git a/src/main/java/com/amazon/carbonado/layout/StoredLayout.java b/src/main/java/com/amazon/carbonado/layout/StoredLayout.java index ea3c1ec..0ed8ffa 100644 --- a/src/main/java/com/amazon/carbonado/layout/StoredLayout.java +++ b/src/main/java/com/amazon/carbonado/layout/StoredLayout.java @@ -42,21 +42,18 @@ import com.amazon.carbonado.Version; @Alias("CARBONADO_LAYOUT") public interface StoredLayout extends Storable, Unevolvable { long getLayoutID(); - void setLayoutID(long typeID); /** * Storable type name is a fully qualified Java class name. */ String getStorableTypeName(); - void setStorableTypeName(String typeName); /** * Generation of storable, where 0 represents the first generation. */ int getGeneration(); - void setGeneration(int generation); /** @@ -64,7 +61,6 @@ public interface StoredLayout extends Storable, Unevolvable { * created. */ long getCreationTimestamp(); - void setCreationTimestamp(long timestamp); /** @@ -72,7 +68,6 @@ public interface StoredLayout extends Storable, Unevolvable { */ @Nullable String getCreationUser(); - void setCreationUser(String user); /** @@ -80,7 +75,6 @@ public interface StoredLayout extends Storable, Unevolvable { */ @Nullable String getCreationHost(); - void setCreationHost(String host); /** @@ -89,7 +83,6 @@ public interface StoredLayout extends Storable, Unevolvable { */ @Version int getVersionNumber(); - void setVersionNumber(int version); /** @@ -100,6 +93,5 @@ public interface StoredLayout extends Storable, Unevolvable { */ @Nullable byte[] getExtraData(); - void setExtraData(byte[] data); } diff --git a/src/main/java/com/amazon/carbonado/layout/StoredLayoutProperty.java b/src/main/java/com/amazon/carbonado/layout/StoredLayoutProperty.java index 0777f10..b98a2a7 100644 --- a/src/main/java/com/amazon/carbonado/layout/StoredLayoutProperty.java +++ b/src/main/java/com/amazon/carbonado/layout/StoredLayoutProperty.java @@ -42,7 +42,6 @@ import com.amazon.carbonado.Version; @Alias("CARBONADO_LAYOUT_PROPERTY") public interface StoredLayoutProperty extends Storable, Unevolvable { long getLayoutID(); - void setLayoutID(long typeID); /** @@ -50,32 +49,27 @@ public interface StoredLayoutProperty extends Storable, Un * layout. */ int getOrdinal(); - void setOrdinal(int ordinal); String getPropertyName(); - void setPropertyName(String name); /** * Property type descriptor is a Java type descriptor. */ String getPropertyTypeDescriptor(); - void setPropertyTypeDescriptor(String type); /** * Returns true of property value can be set to null. */ boolean isNullable(); - void setNullable(boolean nullable); /** * Returns true if property is a member of the primary key. */ boolean isPrimaryKeyMember(); - void setPrimaryKeyMember(boolean pk); /** @@ -83,7 +77,6 @@ public interface StoredLayoutProperty extends Storable, Un * Storable. */ boolean isVersion(); - void setVersion(boolean version); /** @@ -92,7 +85,6 @@ public interface StoredLayoutProperty extends Storable, Un */ @Nullable String getAdapterTypeName(); - void setAdapterTypeName(String name); /** @@ -100,7 +92,6 @@ public interface StoredLayoutProperty extends Storable, Un */ @Nullable String getAdapterParams(); - void setAdapterParams(String params); /** @@ -109,7 +100,6 @@ public interface StoredLayoutProperty extends Storable, Un */ @Version int getVersionNumber(); - void setVersionNumber(int version); /** @@ -120,6 +110,5 @@ public interface StoredLayoutProperty extends Storable, Un */ @Nullable byte[] getExtraData(); - void setExtraData(byte[] data); } diff --git a/src/main/java/com/amazon/carbonado/raw/CompressedEncodingStrategy.java b/src/main/java/com/amazon/carbonado/raw/CompressedEncodingStrategy.java new file mode 100644 index 0000000..08f5eeb --- /dev/null +++ b/src/main/java/com/amazon/carbonado/raw/CompressedEncodingStrategy.java @@ -0,0 +1,97 @@ +/* + * Copyright 2009 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.raw; + +import org.cojen.classfile.CodeAssembler; +import org.cojen.classfile.Label; +import org.cojen.classfile.LocalVariable; +import org.cojen.classfile.Opcode; +import org.cojen.classfile.TypeDesc; + +import com.amazon.carbonado.Storable; +import com.amazon.carbonado.SupportException; + +import com.amazon.carbonado.info.StorableIndex; +import com.amazon.carbonado.info.StorableProperty; + +/** + * Extension of GenericEncodingStrategy that allows for compression. + * + * @author Olga Kuznetsova + * @author Brian S O'Neill + */ +public class CompressedEncodingStrategy extends GenericEncodingStrategy { + private final CompressionType mCompressionType; + + public CompressedEncodingStrategy(Class type, + StorableIndex pkIndex, + CompressionType compressionType) { + super(type, pkIndex); + mCompressionType = compressionType; + } + + @Override + protected void extraDataEncoding(CodeAssembler a, + LocalVariable dataVar, int prefix, int suffix) + { + switch (mCompressionType) { + case GZIP: + TypeDesc byteArrayType = TypeDesc.forClass(byte[].class); + a.loadLocal(dataVar); + a.loadConstant(prefix); + a.invokeStatic(GzipCompressor.class.getName(), "compress", byteArrayType, + new TypeDesc[] {byteArrayType, TypeDesc.INT}); + a.storeLocal(dataVar); + break; + } + } + + @Override + protected void extraDataDecoding(CodeAssembler a, + LocalVariable dataVar, int prefix, int suffix) + { + switch (mCompressionType) { + case GZIP: + TypeDesc byteArrayType = TypeDesc.forClass(byte[].class); + a.loadLocal(dataVar); + a.loadConstant(prefix); + a.invokeStatic(GzipCompressor.class.getName(), "decompress", byteArrayType, + new TypeDesc[] {byteArrayType, TypeDesc.INT}); + a.storeLocal(dataVar); + break; + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj instanceof CompressedEncodingStrategy) { + CompressedEncodingStrategy other = (CompressedEncodingStrategy) obj; + return super.equals(obj) && mCompressionType.equals(other.mCompressionType); + } + return false; + } + + @Override + public int hashCode() { + return super.hashCode() + mCompressionType.hashCode(); + } +} diff --git a/src/main/java/com/amazon/carbonado/raw/CompressedStorableCodecFactory.java b/src/main/java/com/amazon/carbonado/raw/CompressedStorableCodecFactory.java new file mode 100644 index 0000000..b4e6b62 --- /dev/null +++ b/src/main/java/com/amazon/carbonado/raw/CompressedStorableCodecFactory.java @@ -0,0 +1,84 @@ +/* + * Copyright 2009 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.raw; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import com.amazon.carbonado.Storable; +import com.amazon.carbonado.SupportException; + +import com.amazon.carbonado.info.StorableIndex; + +import com.amazon.carbonado.layout.LayoutOptions; + +/** + * Extension of GenericStorableCodecFactory that allows for compression. + * + * @author Olga Kuznetsova + * @author Brian S O'Neill + */ +public class CompressedStorableCodecFactory extends GenericStorableCodecFactory { + private final Map mCompressionMap; + + public CompressedStorableCodecFactory(Map compressionMap) { + if (compressionMap == null || compressionMap.isEmpty()) { + mCompressionMap = Collections.emptyMap(); + } else { + mCompressionMap = new HashMap(compressionMap); + } + } + + @Override + public LayoutOptions getLayoutOptions(Class type) { + CompressionType compType = getCompressionType(type); + if (compType == CompressionType.NONE) { + return null; + } + LayoutOptions options = new LayoutOptions(); + options.setCompressionType(compType.toString()); + return options; + } + + @Override + protected GenericEncodingStrategy createStrategy + (Class type, + StorableIndex pkIndex, + LayoutOptions options) + throws SupportException + { + CompressionType compType; + if (options == null) { + compType = getCompressionType(type); + } else { + compType = CompressionType.valueOf(options.getCompressionType()); + } + + return new CompressedEncodingStrategy(type, pkIndex, compType); + } + + /** + * @return non-null compression type for the given storable + */ + protected CompressionType getCompressionType(Class type) { + CompressionType compType = mCompressionMap.get(type.getName()); + return compType == null ? CompressionType.NONE : compType; + } +} diff --git a/src/main/java/com/amazon/carbonado/raw/CompressionType.java b/src/main/java/com/amazon/carbonado/raw/CompressionType.java new file mode 100644 index 0000000..4061e45 --- /dev/null +++ b/src/main/java/com/amazon/carbonado/raw/CompressionType.java @@ -0,0 +1,35 @@ +/* + * Copyright 2009 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.raw; + +/** + * Available compression types for any particular storable. + * + * @author Olga Kuznetsova + */ +public enum CompressionType { + /** + * No compression. + */ + NONE, + /** + * Gzip compression. + */ + GZIP; +} diff --git a/src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java b/src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java index 05cc5dd..70c5b7c 100644 --- a/src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java +++ b/src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java @@ -534,6 +534,28 @@ public class GenericEncodingStrategy { return infos; } + /** + * Second phase encoding, which does nothing by default. + * + * @param dataVar local variable referencing a byte array with data + * @param prefix prefix of byte array to preserve + * @param suffix suffix of byte array to preserve + */ + protected void extraDataEncoding(CodeAssembler a, + LocalVariable dataVar, int prefix, int suffix) + { + } + + /** + * Second phase decoding, which does nothing by default. + * + * @param dataVar local variable referencing a byte array with data + */ + protected void extraDataDecoding(CodeAssembler a, + LocalVariable dataVar, int prefix, int suffix) + { + } + private SupportException notSupported(StorableProperty property) { return notSupported(property.getName(), TypeDesc.forClass(property.getType()).getFullName()); @@ -735,6 +757,10 @@ public class GenericEncodingStrategy { encodeGeneration(a, encodedVar, prefix, generation); + if (mode == Mode.DATA) { + extraDataEncoding(a, encodedVar, prefix + generationPrefix, suffix); + } + return encodedVar; } } @@ -1252,6 +1278,10 @@ public class GenericEncodingStrategy { exitPoint.setLocation(); + if (mode == Mode.DATA) { + extraDataEncoding(a, encodedVar, prefix + generationPrefix, suffix); + } + return encodedVar; } @@ -1839,6 +1869,8 @@ public class GenericEncodingStrategy { break; } + decodeGeneration(a, encodedVar, prefix, generation, altGenerationHandler); + final int generationPrefix; if (generation < 0) { generationPrefix = 0; @@ -1858,6 +1890,7 @@ public class GenericEncodingStrategy { break; case DATA: suffix = mDataSuffixPadding; + extraDataDecoding(a, encodedVar, prefix + generationPrefix, suffix); break; } @@ -1865,8 +1898,6 @@ public class GenericEncodingStrategy { StorablePropertyInfo[] infos = checkSupport(properties); - decodeGeneration(a, encodedVar, prefix, generation, altGenerationHandler); - if (properties.length == 1) { StorableProperty property = properties[0]; StorablePropertyInfo info = infos[0]; diff --git a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java index 7bd9407..745f761 100644 --- a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java +++ b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java @@ -744,9 +744,9 @@ public class GenericStorableCodec implements StorableCodec altStorable; GenericEncodingStrategy altStrategy; try { - altStorable = mLayout.getGeneration(generation) - .reconstruct(mStorableClass.getClassLoader()); - altStrategy = mFactory.createStrategy(altStorable, null); + Layout altLayout = mLayout.getGeneration(generation); + altStorable = altLayout.reconstruct(mStorableClass.getClassLoader()); + altStrategy = mFactory.createStrategy(altStorable, null, altLayout.getOptions()); } catch (RepositoryException e) { throw new CorruptEncodingException(e); } diff --git a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java index 770e4e2..6a472a9 100644 --- a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java +++ b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java @@ -22,7 +22,9 @@ import com.amazon.carbonado.Storable; import com.amazon.carbonado.SupportException; import com.amazon.carbonado.info.StorableIndex; + import com.amazon.carbonado.layout.Layout; +import com.amazon.carbonado.layout.LayoutOptions; /** * Factory for generic codec that supports any kind of storable by @@ -41,6 +43,14 @@ public class GenericStorableCodecFactory implements StorableCodecFactory { return null; } + /** + * Returns null. + */ + @Override + public LayoutOptions getLayoutOptions(Class type) { + return null; + } + /** * @param type type of storable to create codec for * @param pkIndex suggested index for primary key (optional) @@ -80,7 +90,7 @@ public class GenericStorableCodecFactory implements StorableCodecFactory { throws SupportException { return GenericStorableCodec.getInstance - (this, createStrategy(type, pkIndex), isMaster, layout, support); + (this, createStrategy(type, pkIndex, null), isMaster, layout, support); } /** @@ -95,4 +105,20 @@ public class GenericStorableCodecFactory implements StorableCodecFactory { { return new GenericEncodingStrategy(type, pkIndex); } + + /** + * Override to return a different EncodingStrategy. + * + * @param type type of Storable to generate code for + * @param pkIndex specifies sequence and ordering of key properties (optional) + * @param options additional layout options (optional) + * @since 1.2.1 + */ + protected GenericEncodingStrategy createStrategy + (Class type, StorableIndex pkIndex, LayoutOptions options) + throws SupportException + { + // Call into original method for backwards compatibility. + return createStrategy(type, pkIndex); + } } diff --git a/src/main/java/com/amazon/carbonado/raw/GzipCompressor.java b/src/main/java/com/amazon/carbonado/raw/GzipCompressor.java new file mode 100644 index 0000000..e353d62 --- /dev/null +++ b/src/main/java/com/amazon/carbonado/raw/GzipCompressor.java @@ -0,0 +1,101 @@ +/* + * Copyright 2009 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.raw; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterOutputStream; + +import com.amazon.carbonado.CorruptEncodingException; +import com.amazon.carbonado.SupportException; + +/** + * Raw-level data compression using gzip. + * + * @author Olga Kuznetsova + * @author Brian S O'Neill + */ +public class GzipCompressor { + // NOTE: Class has to be public since it is accessed by generated code. + + final private static ThreadLocal cLocalDeflater = new ThreadLocal(); + final private static ThreadLocal cLocalInflater = new ThreadLocal(); + + /** + * Encodes into compressed form. + * + * @param value value to compress + * @param prefix prefix of byte array to preserve + * @return compressed value + * @throws SupportException thrown if compression failed + */ + public static byte[] compress(byte[] value, int prefix) throws SupportException { + Deflater compressor = cLocalDeflater.get(); + if (compressor == null) { + cLocalDeflater.set(compressor = new Deflater()); + } + + ByteArrayOutputStream bos = new ByteArrayOutputStream(value.length); + + try { + bos.write(value, 0, prefix); + DeflaterOutputStream dout = new DeflaterOutputStream(bos, compressor); + dout.write(value, prefix, value.length - prefix); + dout.close(); + return bos.toByteArray(); + } catch (IOException e) { + throw new SupportException(e); + } finally { + compressor.reset(); + } + } + + /** + * Decodes from compressed form. + * + * @param value value to decompress + * @param prefix prefix of byte array to preserve + * @return decompressed value + * @throws CorruptEncodingException thrown if value cannot be decompressed + */ + public static byte[] decompress(byte[] value, int prefix) throws CorruptEncodingException { + Inflater inflater = cLocalInflater.get(); + if (inflater == null) { + cLocalInflater.set(inflater = new Inflater()); + } + + ByteArrayOutputStream bos = new ByteArrayOutputStream(value.length * 2); + + try { + bos.write(value, 0, prefix); + InflaterOutputStream ios = new InflaterOutputStream(bos, inflater); + ios.write(value, prefix, value.length - prefix); + ios.close(); + return bos.toByteArray(); + } catch (IOException e) { + throw new CorruptEncodingException(e); + } finally { + inflater.reset(); + } + } +} diff --git a/src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java b/src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java index 8cc5be2..8c08ec0 100644 --- a/src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java +++ b/src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java @@ -23,6 +23,7 @@ import com.amazon.carbonado.SupportException; import com.amazon.carbonado.info.StorableIndex; import com.amazon.carbonado.layout.Layout; +import com.amazon.carbonado.layout.LayoutOptions; /** * Factory for creating instances of {@link StorableCodec}. @@ -38,6 +39,13 @@ public interface StorableCodecFactory { */ String getStorageName(Class type) throws SupportException; + /** + * Optionally return additional information regarding storable encoding. + * + * @since 1.2.1 + */ + LayoutOptions getLayoutOptions(Class type); + /** * @param type type of storable to create codec for * @param pkIndex suggested index for primary key (optional) diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java index 6fbc4f5..28fc347 100644 --- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java +++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java @@ -49,6 +49,7 @@ import com.amazon.carbonado.capability.StorableInfoCapability; import com.amazon.carbonado.info.StorableIntrospector; +import com.amazon.carbonado.layout.Layout; import com.amazon.carbonado.layout.LayoutCapability; import com.amazon.carbonado.layout.LayoutFactory; @@ -88,7 +89,8 @@ abstract class BDBRepository extends AbstractRepository EnvironmentCapability, ShutdownCapability, StorableInfoCapability, - SequenceCapability + SequenceCapability, + LayoutCapability { private final Log mLog = LogFactory.getLog(getClass()); @@ -168,19 +170,6 @@ abstract class BDBRepository extends AbstractRepository getLog().info("Opening repository \"" + getName() + '"'); } - @Override - @SuppressWarnings("unchecked") - public C getCapability(Class capabilityType) { - C cap = super.getCapability(capabilityType); - if (cap != null) { - return cap; - } - if (capabilityType == LayoutCapability.class) { - return (C) mLayoutFactory; - } - return null; - } - public IndexInfo[] getIndexInfo(Class storableType) throws RepositoryException { @@ -340,6 +329,26 @@ abstract class BDBRepository extends AbstractRepository return (BDBStorage) storageFor(type); } + @Override + public Layout layoutFor(Class type) + throws FetchException, PersistException + { + try { + return ((BDBStorage) storageFor(type)).getLayout(mStorableCodecFactory); + } catch (PersistException e) { + throw e; + } catch (RepositoryException e) { + throw e.toFetchException(); + } + } + + @Override + public Layout layoutFor(Class type, int generation) + throws FetchException + { + return mLayoutFactory.layoutFor(type, generation); + } + @Override protected void finalize() { close(); diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java index c1e6196..0beea65 100644 --- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java +++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java @@ -34,7 +34,8 @@ import com.amazon.carbonado.RepositoryException; import com.amazon.carbonado.Storable; import com.amazon.carbonado.repo.indexed.IndexedRepositoryBuilder; -import com.amazon.carbonado.raw.GenericStorableCodecFactory; +import com.amazon.carbonado.raw.CompressionType; +import com.amazon.carbonado.raw.CompressedStorableCodecFactory; import com.amazon.carbonado.raw.StorableCodecFactory; import com.amazon.carbonado.spi.AbstractRepositoryBuilder; @@ -101,6 +102,7 @@ public class BDBRepositoryBuilder extends AbstractRepositoryBuilder { private boolean mPrivate; private boolean mMultiversion; private boolean mLogInMemory; + private Integer mLogFileMaxSize; private boolean mRunFullRecovery; private boolean mRunCheckpointer = true; private int mCheckpointInterval = DEFAULT_CHECKPOINT_INTERVAL; @@ -110,10 +112,11 @@ public class BDBRepositoryBuilder extends AbstractRepositoryBuilder { private Boolean mChecksumEnabled; private Object mInitialEnvConfig = null; private Object mInitialDBConfig = null; - private StorableCodecFactory mStorableCodecFactory = new GenericStorableCodecFactory(); + private StorableCodecFactory mStorableCodecFactory; private Runnable mPreShutdownHook; private Runnable mPostShutdownHook; private DatabaseHook mDatabaseHook; + private Map mCompressionMap; public BDBRepositoryBuilder() { } @@ -136,6 +139,10 @@ public class BDBRepositoryBuilder extends AbstractRepositoryBuilder { } } + if (mStorableCodecFactory == null) { + mStorableCodecFactory = new CompressedStorableCodecFactory(mCompressionMap); + } + assertReady(); // Make environment directory if it doesn't exist. @@ -860,6 +867,40 @@ public class BDBRepositoryBuilder extends AbstractRepositoryBuilder { return mDatabaseHook; } + /** + * Set the compressor for the given class, overriding a custom StorableCodecFactory. + + * @param type Storable to compress. + * @param compressionType String representation of type of + * compression. Available options are "NONE" for no compression or "GZIP" + * for gzip compression + */ + public void setCompressor(String type, String compressionType) { + mStorableCodecFactory = null; + compressionType = compressionType.toUpperCase(); + if (mCompressionMap == null) { + mCompressionMap = new HashMap(); + } + CompressionType compressionEnum = CompressionType.valueOf(compressionType); + if (compressionEnum != null) { + mCompressionMap.put(type, compressionEnum); + } + } + + /** + * Return the compressor used for the given storable. + * @param type Storable to compress + * @return String representation of the type of compression used. Available options are "NONE" + * for no compression and "GZIP" for gzip compression. + */ + public String getCompressor(String type) { + if (mCompressionMap == null) { + return null; + } + + return mCompressionMap.get(type).toString(); + } + private long inMicros(double seconds) { if (seconds >= Long.MAX_VALUE) { return Long.MAX_VALUE; diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java index 4f1ff29..d00b7d5 100644 --- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java +++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java @@ -60,6 +60,7 @@ import com.amazon.carbonado.info.StorableProperty; import com.amazon.carbonado.layout.Layout; import com.amazon.carbonado.layout.LayoutFactory; +import com.amazon.carbonado.layout.LayoutOptions; import com.amazon.carbonado.layout.Unevolvable; import com.amazon.carbonado.lob.Blob; @@ -416,11 +417,9 @@ abstract class BDBStorage implements Storage, Storag protected void open(boolean readOnly, Txn openTxn, boolean installTriggers) throws RepositoryException { - final Layout layout = getLayout(); - StorableInfo info = StorableIntrospector.examine(getStorableType()); - StorableCodecFactory codecFactory = mRepository.getStorableCodecFactory(); + final Layout layout = getLayout(codecFactory); // Open primary database. Object primaryDatabase; @@ -767,7 +766,7 @@ abstract class BDBStorage implements Storage, Storag } } - private Layout getLayout() throws RepositoryException { + Layout getLayout(StorableCodecFactory codecFactory) throws RepositoryException { if (Unevolvable.class.isAssignableFrom(getStorableType())) { // Don't record generation for storables marked as unevolvable. return null; @@ -782,7 +781,8 @@ abstract class BDBStorage implements Storage, Storag return null; } - return factory.layoutFor(getStorableType()); + Class type = getStorableType(); + return factory.layoutFor(type, codecFactory.getLayoutOptions(type)); } /** -- cgit v1.2.3