From 55d57b7d9f075c275e4fe30d29577a5b914f05db Mon Sep 17 00:00:00 2001 From: "Brian S. O'Neill" Date: Thu, 5 Nov 2009 23:58:22 +0000 Subject: Add support for compressed records. --- .../carbonado/raw/CompressedEncodingStrategy.java | 97 ++++++++++++++++++++ .../raw/CompressedStorableCodecFactory.java | 84 +++++++++++++++++ .../com/amazon/carbonado/raw/CompressionType.java | 35 +++++++ .../carbonado/raw/GenericEncodingStrategy.java | 35 ++++++- .../amazon/carbonado/raw/GenericStorableCodec.java | 6 +- .../carbonado/raw/GenericStorableCodecFactory.java | 28 +++++- .../com/amazon/carbonado/raw/GzipCompressor.java | 101 +++++++++++++++++++++ .../amazon/carbonado/raw/StorableCodecFactory.java | 8 ++ 8 files changed, 388 insertions(+), 6 deletions(-) create mode 100644 src/main/java/com/amazon/carbonado/raw/CompressedEncodingStrategy.java create mode 100644 src/main/java/com/amazon/carbonado/raw/CompressedStorableCodecFactory.java create mode 100644 src/main/java/com/amazon/carbonado/raw/CompressionType.java create mode 100644 src/main/java/com/amazon/carbonado/raw/GzipCompressor.java (limited to 'src/main/java/com/amazon/carbonado/raw') diff --git a/src/main/java/com/amazon/carbonado/raw/CompressedEncodingStrategy.java b/src/main/java/com/amazon/carbonado/raw/CompressedEncodingStrategy.java new file mode 100644 index 0000000..08f5eeb --- /dev/null +++ b/src/main/java/com/amazon/carbonado/raw/CompressedEncodingStrategy.java @@ -0,0 +1,97 @@ +/* + * Copyright 2009 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.raw; + +import org.cojen.classfile.CodeAssembler; +import org.cojen.classfile.Label; +import org.cojen.classfile.LocalVariable; +import org.cojen.classfile.Opcode; +import org.cojen.classfile.TypeDesc; + +import com.amazon.carbonado.Storable; +import com.amazon.carbonado.SupportException; + +import com.amazon.carbonado.info.StorableIndex; +import com.amazon.carbonado.info.StorableProperty; + +/** + * Extension of GenericEncodingStrategy that allows for compression. + * + * @author Olga Kuznetsova + * @author Brian S O'Neill + */ +public class CompressedEncodingStrategy extends GenericEncodingStrategy { + private final CompressionType mCompressionType; + + public CompressedEncodingStrategy(Class type, + StorableIndex pkIndex, + CompressionType compressionType) { + super(type, pkIndex); + mCompressionType = compressionType; + } + + @Override + protected void extraDataEncoding(CodeAssembler a, + LocalVariable dataVar, int prefix, int suffix) + { + switch (mCompressionType) { + case GZIP: + TypeDesc byteArrayType = TypeDesc.forClass(byte[].class); + a.loadLocal(dataVar); + a.loadConstant(prefix); + a.invokeStatic(GzipCompressor.class.getName(), "compress", byteArrayType, + new TypeDesc[] {byteArrayType, TypeDesc.INT}); + a.storeLocal(dataVar); + break; + } + } + + @Override + protected void extraDataDecoding(CodeAssembler a, + LocalVariable dataVar, int prefix, int suffix) + { + switch (mCompressionType) { + case GZIP: + TypeDesc byteArrayType = TypeDesc.forClass(byte[].class); + a.loadLocal(dataVar); + a.loadConstant(prefix); + a.invokeStatic(GzipCompressor.class.getName(), "decompress", byteArrayType, + new TypeDesc[] {byteArrayType, TypeDesc.INT}); + a.storeLocal(dataVar); + break; + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj instanceof CompressedEncodingStrategy) { + CompressedEncodingStrategy other = (CompressedEncodingStrategy) obj; + return super.equals(obj) && mCompressionType.equals(other.mCompressionType); + } + return false; + } + + @Override + public int hashCode() { + return super.hashCode() + mCompressionType.hashCode(); + } +} diff --git a/src/main/java/com/amazon/carbonado/raw/CompressedStorableCodecFactory.java b/src/main/java/com/amazon/carbonado/raw/CompressedStorableCodecFactory.java new file mode 100644 index 0000000..b4e6b62 --- /dev/null +++ b/src/main/java/com/amazon/carbonado/raw/CompressedStorableCodecFactory.java @@ -0,0 +1,84 @@ +/* + * Copyright 2009 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.raw; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import com.amazon.carbonado.Storable; +import com.amazon.carbonado.SupportException; + +import com.amazon.carbonado.info.StorableIndex; + +import com.amazon.carbonado.layout.LayoutOptions; + +/** + * Extension of GenericStorableCodecFactory that allows for compression. + * + * @author Olga Kuznetsova + * @author Brian S O'Neill + */ +public class CompressedStorableCodecFactory extends GenericStorableCodecFactory { + private final Map mCompressionMap; + + public CompressedStorableCodecFactory(Map compressionMap) { + if (compressionMap == null || compressionMap.isEmpty()) { + mCompressionMap = Collections.emptyMap(); + } else { + mCompressionMap = new HashMap(compressionMap); + } + } + + @Override + public LayoutOptions getLayoutOptions(Class type) { + CompressionType compType = getCompressionType(type); + if (compType == CompressionType.NONE) { + return null; + } + LayoutOptions options = new LayoutOptions(); + options.setCompressionType(compType.toString()); + return options; + } + + @Override + protected GenericEncodingStrategy createStrategy + (Class type, + StorableIndex pkIndex, + LayoutOptions options) + throws SupportException + { + CompressionType compType; + if (options == null) { + compType = getCompressionType(type); + } else { + compType = CompressionType.valueOf(options.getCompressionType()); + } + + return new CompressedEncodingStrategy(type, pkIndex, compType); + } + + /** + * @return non-null compression type for the given storable + */ + protected CompressionType getCompressionType(Class type) { + CompressionType compType = mCompressionMap.get(type.getName()); + return compType == null ? CompressionType.NONE : compType; + } +} diff --git a/src/main/java/com/amazon/carbonado/raw/CompressionType.java b/src/main/java/com/amazon/carbonado/raw/CompressionType.java new file mode 100644 index 0000000..4061e45 --- /dev/null +++ b/src/main/java/com/amazon/carbonado/raw/CompressionType.java @@ -0,0 +1,35 @@ +/* + * Copyright 2009 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.raw; + +/** + * Available compression types for any particular storable. + * + * @author Olga Kuznetsova + */ +public enum CompressionType { + /** + * No compression. + */ + NONE, + /** + * Gzip compression. + */ + GZIP; +} diff --git a/src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java b/src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java index 05cc5dd..70c5b7c 100644 --- a/src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java +++ b/src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java @@ -534,6 +534,28 @@ public class GenericEncodingStrategy { return infos; } + /** + * Second phase encoding, which does nothing by default. + * + * @param dataVar local variable referencing a byte array with data + * @param prefix prefix of byte array to preserve + * @param suffix suffix of byte array to preserve + */ + protected void extraDataEncoding(CodeAssembler a, + LocalVariable dataVar, int prefix, int suffix) + { + } + + /** + * Second phase decoding, which does nothing by default. + * + * @param dataVar local variable referencing a byte array with data + */ + protected void extraDataDecoding(CodeAssembler a, + LocalVariable dataVar, int prefix, int suffix) + { + } + private SupportException notSupported(StorableProperty property) { return notSupported(property.getName(), TypeDesc.forClass(property.getType()).getFullName()); @@ -735,6 +757,10 @@ public class GenericEncodingStrategy { encodeGeneration(a, encodedVar, prefix, generation); + if (mode == Mode.DATA) { + extraDataEncoding(a, encodedVar, prefix + generationPrefix, suffix); + } + return encodedVar; } } @@ -1252,6 +1278,10 @@ public class GenericEncodingStrategy { exitPoint.setLocation(); + if (mode == Mode.DATA) { + extraDataEncoding(a, encodedVar, prefix + generationPrefix, suffix); + } + return encodedVar; } @@ -1839,6 +1869,8 @@ public class GenericEncodingStrategy { break; } + decodeGeneration(a, encodedVar, prefix, generation, altGenerationHandler); + final int generationPrefix; if (generation < 0) { generationPrefix = 0; @@ -1858,6 +1890,7 @@ public class GenericEncodingStrategy { break; case DATA: suffix = mDataSuffixPadding; + extraDataDecoding(a, encodedVar, prefix + generationPrefix, suffix); break; } @@ -1865,8 +1898,6 @@ public class GenericEncodingStrategy { StorablePropertyInfo[] infos = checkSupport(properties); - decodeGeneration(a, encodedVar, prefix, generation, altGenerationHandler); - if (properties.length == 1) { StorableProperty property = properties[0]; StorablePropertyInfo info = infos[0]; diff --git a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java index 7bd9407..745f761 100644 --- a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java +++ b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java @@ -744,9 +744,9 @@ public class GenericStorableCodec implements StorableCodec altStorable; GenericEncodingStrategy altStrategy; try { - altStorable = mLayout.getGeneration(generation) - .reconstruct(mStorableClass.getClassLoader()); - altStrategy = mFactory.createStrategy(altStorable, null); + Layout altLayout = mLayout.getGeneration(generation); + altStorable = altLayout.reconstruct(mStorableClass.getClassLoader()); + altStrategy = mFactory.createStrategy(altStorable, null, altLayout.getOptions()); } catch (RepositoryException e) { throw new CorruptEncodingException(e); } diff --git a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java index 770e4e2..6a472a9 100644 --- a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java +++ b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java @@ -22,7 +22,9 @@ import com.amazon.carbonado.Storable; import com.amazon.carbonado.SupportException; import com.amazon.carbonado.info.StorableIndex; + import com.amazon.carbonado.layout.Layout; +import com.amazon.carbonado.layout.LayoutOptions; /** * Factory for generic codec that supports any kind of storable by @@ -41,6 +43,14 @@ public class GenericStorableCodecFactory implements StorableCodecFactory { return null; } + /** + * Returns null. + */ + @Override + public LayoutOptions getLayoutOptions(Class type) { + return null; + } + /** * @param type type of storable to create codec for * @param pkIndex suggested index for primary key (optional) @@ -80,7 +90,7 @@ public class GenericStorableCodecFactory implements StorableCodecFactory { throws SupportException { return GenericStorableCodec.getInstance - (this, createStrategy(type, pkIndex), isMaster, layout, support); + (this, createStrategy(type, pkIndex, null), isMaster, layout, support); } /** @@ -95,4 +105,20 @@ public class GenericStorableCodecFactory implements StorableCodecFactory { { return new GenericEncodingStrategy(type, pkIndex); } + + /** + * Override to return a different EncodingStrategy. + * + * @param type type of Storable to generate code for + * @param pkIndex specifies sequence and ordering of key properties (optional) + * @param options additional layout options (optional) + * @since 1.2.1 + */ + protected GenericEncodingStrategy createStrategy + (Class type, StorableIndex pkIndex, LayoutOptions options) + throws SupportException + { + // Call into original method for backwards compatibility. + return createStrategy(type, pkIndex); + } } diff --git a/src/main/java/com/amazon/carbonado/raw/GzipCompressor.java b/src/main/java/com/amazon/carbonado/raw/GzipCompressor.java new file mode 100644 index 0000000..e353d62 --- /dev/null +++ b/src/main/java/com/amazon/carbonado/raw/GzipCompressor.java @@ -0,0 +1,101 @@ +/* + * Copyright 2009 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.raw; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterOutputStream; + +import com.amazon.carbonado.CorruptEncodingException; +import com.amazon.carbonado.SupportException; + +/** + * Raw-level data compression using gzip. + * + * @author Olga Kuznetsova + * @author Brian S O'Neill + */ +public class GzipCompressor { + // NOTE: Class has to be public since it is accessed by generated code. + + final private static ThreadLocal cLocalDeflater = new ThreadLocal(); + final private static ThreadLocal cLocalInflater = new ThreadLocal(); + + /** + * Encodes into compressed form. + * + * @param value value to compress + * @param prefix prefix of byte array to preserve + * @return compressed value + * @throws SupportException thrown if compression failed + */ + public static byte[] compress(byte[] value, int prefix) throws SupportException { + Deflater compressor = cLocalDeflater.get(); + if (compressor == null) { + cLocalDeflater.set(compressor = new Deflater()); + } + + ByteArrayOutputStream bos = new ByteArrayOutputStream(value.length); + + try { + bos.write(value, 0, prefix); + DeflaterOutputStream dout = new DeflaterOutputStream(bos, compressor); + dout.write(value, prefix, value.length - prefix); + dout.close(); + return bos.toByteArray(); + } catch (IOException e) { + throw new SupportException(e); + } finally { + compressor.reset(); + } + } + + /** + * Decodes from compressed form. + * + * @param value value to decompress + * @param prefix prefix of byte array to preserve + * @return decompressed value + * @throws CorruptEncodingException thrown if value cannot be decompressed + */ + public static byte[] decompress(byte[] value, int prefix) throws CorruptEncodingException { + Inflater inflater = cLocalInflater.get(); + if (inflater == null) { + cLocalInflater.set(inflater = new Inflater()); + } + + ByteArrayOutputStream bos = new ByteArrayOutputStream(value.length * 2); + + try { + bos.write(value, 0, prefix); + InflaterOutputStream ios = new InflaterOutputStream(bos, inflater); + ios.write(value, prefix, value.length - prefix); + ios.close(); + return bos.toByteArray(); + } catch (IOException e) { + throw new CorruptEncodingException(e); + } finally { + inflater.reset(); + } + } +} diff --git a/src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java b/src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java index 8cc5be2..8c08ec0 100644 --- a/src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java +++ b/src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java @@ -23,6 +23,7 @@ import com.amazon.carbonado.SupportException; import com.amazon.carbonado.info.StorableIndex; import com.amazon.carbonado.layout.Layout; +import com.amazon.carbonado.layout.LayoutOptions; /** * Factory for creating instances of {@link StorableCodec}. @@ -38,6 +39,13 @@ public interface StorableCodecFactory { */ String getStorageName(Class type) throws SupportException; + /** + * Optionally return additional information regarding storable encoding. + * + * @since 1.2.1 + */ + LayoutOptions getLayoutOptions(Class type); + /** * @param type type of storable to create codec for * @param pkIndex suggested index for primary key (optional) -- cgit v1.2.3