summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBrian S. O'Neill <bronee@gmail.com>2009-11-05 23:58:22 +0000
committerBrian S. O'Neill <bronee@gmail.com>2009-11-05 23:58:22 +0000
commit55d57b7d9f075c275e4fe30d29577a5b914f05db (patch)
treed43dea9ea36d67bbff91f96f3a74807f15be3518
parentc8ceb3bf28a8aae8efef735d78a3e6da45049f95 (diff)
Add support for compressed records.
-rw-r--r--src/main/java/com/amazon/carbonado/layout/Layout.java37
-rw-r--r--src/main/java/com/amazon/carbonado/layout/LayoutFactory.java37
-rw-r--r--src/main/java/com/amazon/carbonado/layout/LayoutOptions.java131
-rw-r--r--src/main/java/com/amazon/carbonado/layout/StoredLayout.java8
-rw-r--r--src/main/java/com/amazon/carbonado/layout/StoredLayoutProperty.java11
-rw-r--r--src/main/java/com/amazon/carbonado/raw/CompressedEncodingStrategy.java97
-rw-r--r--src/main/java/com/amazon/carbonado/raw/CompressedStorableCodecFactory.java84
-rw-r--r--src/main/java/com/amazon/carbonado/raw/CompressionType.java35
-rw-r--r--src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java35
-rw-r--r--src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java6
-rw-r--r--src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java28
-rw-r--r--src/main/java/com/amazon/carbonado/raw/GzipCompressor.java101
-rw-r--r--src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java8
-rw-r--r--src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java37
-rw-r--r--src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java45
-rw-r--r--src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java10
16 files changed, 659 insertions, 51 deletions
diff --git a/src/main/java/com/amazon/carbonado/layout/Layout.java b/src/main/java/com/amazon/carbonado/layout/Layout.java
index b6c6c0d..7aea758 100644
--- a/src/main/java/com/amazon/carbonado/layout/Layout.java
+++ b/src/main/java/com/amazon/carbonado/layout/Layout.java
@@ -18,6 +18,7 @@
package com.amazon.carbonado.layout;
+import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -32,6 +33,7 @@ import org.apache.commons.logging.LogFactory;
import org.cojen.util.SoftValuedHashMap;
+import com.amazon.carbonado.CorruptEncodingException;
import com.amazon.carbonado.Cursor;
import com.amazon.carbonado.FetchException;
import com.amazon.carbonado.FetchNoneException;
@@ -131,22 +133,36 @@ public class Layout {
private final LayoutFactory mLayoutFactory;
private final StoredLayout mStoredLayout;
+ private final LayoutOptions mOptions;
private transient List<LayoutProperty> mAllProperties;
/**
* Creates a Layout around an existing storable.
*/
- Layout(LayoutFactory factory, StoredLayout storedLayout) {
+ Layout(LayoutFactory factory, StoredLayout storedLayout) throws CorruptEncodingException {
mLayoutFactory = factory;
mStoredLayout = storedLayout;
+
+ byte[] extra = storedLayout.getExtraData();
+ if (extra == null) {
+ mOptions = null;
+ } else {
+ mOptions = new LayoutOptions();
+ try {
+ mOptions.decode(extra);
+ } catch (IOException e) {
+ throw new CorruptEncodingException(e);
+ }
+ mOptions.readOnly();
+ }
}
/**
* Copies layout information into freshly prepared storables. Call insert
* (on this class) to persist them.
*/
- Layout(LayoutFactory factory, StorableInfo<?> info, long layoutID) {
+ Layout(LayoutFactory factory, StorableInfo<?> info, LayoutOptions options, long layoutID) {
mLayoutFactory = factory;
StoredLayout storedLayout = factory.mLayoutStorage.prepare();
@@ -168,6 +184,14 @@ public class Layout {
// Can't get host, no big deal.
}
+ if (options == null) {
+ mOptions = null;
+ } else {
+ options.readOnly();
+ storedLayout.setExtraData(options.encode());
+ mOptions = options;
+ }
+
Collection<? extends StorableProperty<?>> properties = info.getAllProperties().values();
List<LayoutProperty> list = new ArrayList<LayoutProperty>(properties.size());
int ordinal = 0;
@@ -269,6 +293,15 @@ public class Layout {
}
/**
+ * Returns additional options, or null if none.
+ *
+ * @return read-only object or null
+ */
+ public LayoutOptions getOptions() {
+ return mOptions;
+ }
+
+ /**
* Returns the layout for a particular generation of this layout's type.
*
* @throws FetchNoneException if generation not found
diff --git a/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java b/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java
index 1b2f0fc..54b6933 100644
--- a/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java
+++ b/src/main/java/com/amazon/carbonado/layout/LayoutFactory.java
@@ -87,6 +87,23 @@ public class LayoutFactory implements LayoutCapability {
public Layout layoutFor(Class<? extends Storable> type)
throws FetchException, PersistException
{
+ return layoutFor(type, null);
+ }
+
+ /**
+ * Returns the layout matching the current definition of the given type.
+ *
+ * @throws PersistException if type represents a new generation, but
+ * persisting this information failed
+ */
+ public Layout layoutFor(Class<? extends Storable> type, LayoutOptions options)
+ throws FetchException, PersistException
+ {
+ if (options != null) {
+ // Make side-effect consistently applied.
+ options.readOnly();
+ }
+
synchronized (this) {
if (mReconstructed != null) {
Layout layout = mReconstructed.get(type);
@@ -120,10 +137,10 @@ public class LayoutFactory implements LayoutCapability {
for (int i=0; i<HASH_MULTIPLIERS.length; i++) {
// Generate an identifier which has a high likelyhood of being unique.
- long layoutID = mixInHash(0L, info, HASH_MULTIPLIERS[i]);
+ long layoutID = mixInHash(0L, info, options, HASH_MULTIPLIERS[i]);
// Initially use for comparison purposes.
- newLayout = new Layout(this, info, layoutID);
+ newLayout = new Layout(this, info, options, layoutID);
StoredLayout storedLayout = mLayoutStorage.prepare();
storedLayout.setLayoutID(layoutID);
@@ -262,8 +279,10 @@ public class LayoutFactory implements LayoutCapability {
* Creates a long hash code that attempts to mix in all relevant layout
* elements.
*/
- private long mixInHash(long hash, StorableInfo<?> info, int multiplier) {
+ private long mixInHash(long hash, StorableInfo<?> info, LayoutOptions options, int multiplier)
+ {
hash = mixInHash(hash, info.getStorableType().getName(), multiplier);
+ hash = mixInHash(hash, options, multiplier);
for (StorableProperty<?> property : info.getAllProperties().values()) {
if (!property.isJoin()) {
@@ -314,6 +333,18 @@ public class LayoutFactory implements LayoutCapability {
return hash;
}
+ private long mixInHash(long hash, LayoutOptions options, int multiplier) {
+ if (options != null) {
+ byte[] data = options.encode();
+ if (data != null) {
+ for (int b : data) {
+ hash = hash * multiplier + (b & 0xff);
+ }
+ }
+ }
+ return hash;
+ }
+
/**
* Returns an annotation hash code using a algorithm similar to the
* default. The difference is in the handling of class and enum values. The
diff --git a/src/main/java/com/amazon/carbonado/layout/LayoutOptions.java b/src/main/java/com/amazon/carbonado/layout/LayoutOptions.java
new file mode 100644
index 0000000..1e57407
--- /dev/null
+++ b/src/main/java/com/amazon/carbonado/layout/LayoutOptions.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2009 Amazon Technologies, Inc. or its affiliates.
+ * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks
+ * of Amazon Technologies, Inc. or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.amazon.carbonado.layout;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Extra options encoded with a Storable layout.
+ *
+ * @author Brian S O'Neill
+ */
+public class LayoutOptions {
+ /**
+ * Data is compression algorithm name, encoded by DataOutput.writeUTF.
+ */
+ static final byte COMPRESSION_TYPE = 1;
+
+ private final Map<Byte, Object> mData;
+
+ private boolean mReadOnly;
+
+ public LayoutOptions() {
+ mData = new HashMap<Byte, Object>(1);
+ }
+
+ /**
+ * @return null if not compressed
+ */
+ public synchronized String getCompressionType() {
+ return (String) mData.get(COMPRESSION_TYPE);
+ }
+
+ /**
+ * @param type null if not compressed
+ */
+ public void setCompressionType(String type) {
+ put(COMPRESSION_TYPE, type);
+ }
+
+ private synchronized void put(byte op, Object value) {
+ if (mReadOnly) {
+ throw new IllegalStateException("Options are read only");
+ }
+ if (value == null) {
+ mData.remove(op);
+ } else {
+ mData.put(op, value);
+ }
+ }
+
+ synchronized void readOnly() {
+ mReadOnly = true;
+ }
+
+ /**
+ * @return null if empty
+ */
+ synchronized byte[] encode() {
+ if (mData.isEmpty()) {
+ return null;
+ }
+
+ try {
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ DataOutputStream dout = new DataOutputStream(bout);
+
+ for (Map.Entry<Byte, Object> entry : mData.entrySet()) {
+ switch (entry.getKey()) {
+ default:
+ break;
+ case COMPRESSION_TYPE:
+ dout.write(COMPRESSION_TYPE);
+ dout.writeUTF((String) entry.getValue());
+ }
+ }
+
+ dout.close();
+ return bout.toByteArray();
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ /**
+ * @param source can be null if empty
+ */
+ synchronized void decode(byte[] source) throws IOException {
+ mData.clear();
+
+ if (source == null || source.length == 0) {
+ return;
+ }
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(source);
+ DataInputStream din = new DataInputStream(bin);
+
+ while (bin.available() > 0) {
+ byte op = din.readByte();
+ switch (op) {
+ default:
+ throw new IOException("Unknown extra data type: " + op);
+ case COMPRESSION_TYPE:
+ mData.put(COMPRESSION_TYPE, din.readUTF());
+ break;
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/amazon/carbonado/layout/StoredLayout.java b/src/main/java/com/amazon/carbonado/layout/StoredLayout.java
index ea3c1ec..0ed8ffa 100644
--- a/src/main/java/com/amazon/carbonado/layout/StoredLayout.java
+++ b/src/main/java/com/amazon/carbonado/layout/StoredLayout.java
@@ -42,21 +42,18 @@ import com.amazon.carbonado.Version;
@Alias("CARBONADO_LAYOUT")
public interface StoredLayout extends Storable<StoredLayout>, Unevolvable {
long getLayoutID();
-
void setLayoutID(long typeID);
/**
* Storable type name is a fully qualified Java class name.
*/
String getStorableTypeName();
-
void setStorableTypeName(String typeName);
/**
* Generation of storable, where 0 represents the first generation.
*/
int getGeneration();
-
void setGeneration(int generation);
/**
@@ -64,7 +61,6 @@ public interface StoredLayout extends Storable<StoredLayout>, Unevolvable {
* created.
*/
long getCreationTimestamp();
-
void setCreationTimestamp(long timestamp);
/**
@@ -72,7 +68,6 @@ public interface StoredLayout extends Storable<StoredLayout>, Unevolvable {
*/
@Nullable
String getCreationUser();
-
void setCreationUser(String user);
/**
@@ -80,7 +75,6 @@ public interface StoredLayout extends Storable<StoredLayout>, Unevolvable {
*/
@Nullable
String getCreationHost();
-
void setCreationHost(String host);
/**
@@ -89,7 +83,6 @@ public interface StoredLayout extends Storable<StoredLayout>, Unevolvable {
*/
@Version
int getVersionNumber();
-
void setVersionNumber(int version);
/**
@@ -100,6 +93,5 @@ public interface StoredLayout extends Storable<StoredLayout>, Unevolvable {
*/
@Nullable
byte[] getExtraData();
-
void setExtraData(byte[] data);
}
diff --git a/src/main/java/com/amazon/carbonado/layout/StoredLayoutProperty.java b/src/main/java/com/amazon/carbonado/layout/StoredLayoutProperty.java
index 0777f10..b98a2a7 100644
--- a/src/main/java/com/amazon/carbonado/layout/StoredLayoutProperty.java
+++ b/src/main/java/com/amazon/carbonado/layout/StoredLayoutProperty.java
@@ -42,7 +42,6 @@ import com.amazon.carbonado.Version;
@Alias("CARBONADO_LAYOUT_PROPERTY")
public interface StoredLayoutProperty extends Storable<StoredLayoutProperty>, Unevolvable {
long getLayoutID();
-
void setLayoutID(long typeID);
/**
@@ -50,32 +49,27 @@ public interface StoredLayoutProperty extends Storable<StoredLayoutProperty>, Un
* layout.
*/
int getOrdinal();
-
void setOrdinal(int ordinal);
String getPropertyName();
-
void setPropertyName(String name);
/**
* Property type descriptor is a Java type descriptor.
*/
String getPropertyTypeDescriptor();
-
void setPropertyTypeDescriptor(String type);
/**
* Returns true of property value can be set to null.
*/
boolean isNullable();
-
void setNullable(boolean nullable);
/**
* Returns true if property is a member of the primary key.
*/
boolean isPrimaryKeyMember();
-
void setPrimaryKeyMember(boolean pk);
/**
@@ -83,7 +77,6 @@ public interface StoredLayoutProperty extends Storable<StoredLayoutProperty>, Un
* Storable.
*/
boolean isVersion();
-
void setVersion(boolean version);
/**
@@ -92,7 +85,6 @@ public interface StoredLayoutProperty extends Storable<StoredLayoutProperty>, Un
*/
@Nullable
String getAdapterTypeName();
-
void setAdapterTypeName(String name);
/**
@@ -100,7 +92,6 @@ public interface StoredLayoutProperty extends Storable<StoredLayoutProperty>, Un
*/
@Nullable
String getAdapterParams();
-
void setAdapterParams(String params);
/**
@@ -109,7 +100,6 @@ public interface StoredLayoutProperty extends Storable<StoredLayoutProperty>, Un
*/
@Version
int getVersionNumber();
-
void setVersionNumber(int version);
/**
@@ -120,6 +110,5 @@ public interface StoredLayoutProperty extends Storable<StoredLayoutProperty>, Un
*/
@Nullable
byte[] getExtraData();
-
void setExtraData(byte[] data);
}
diff --git a/src/main/java/com/amazon/carbonado/raw/CompressedEncodingStrategy.java b/src/main/java/com/amazon/carbonado/raw/CompressedEncodingStrategy.java
new file mode 100644
index 0000000..08f5eeb
--- /dev/null
+++ b/src/main/java/com/amazon/carbonado/raw/CompressedEncodingStrategy.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2009 Amazon Technologies, Inc. or its affiliates.
+ * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks
+ * of Amazon Technologies, Inc. or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.amazon.carbonado.raw;
+
+import org.cojen.classfile.CodeAssembler;
+import org.cojen.classfile.Label;
+import org.cojen.classfile.LocalVariable;
+import org.cojen.classfile.Opcode;
+import org.cojen.classfile.TypeDesc;
+
+import com.amazon.carbonado.Storable;
+import com.amazon.carbonado.SupportException;
+
+import com.amazon.carbonado.info.StorableIndex;
+import com.amazon.carbonado.info.StorableProperty;
+
+/**
+ * Extension of GenericEncodingStrategy that allows for compression.
+ *
+ * @author Olga Kuznetsova
+ * @author Brian S O'Neill
+ */
+public class CompressedEncodingStrategy<S extends Storable> extends GenericEncodingStrategy<S> {
+ private final CompressionType mCompressionType;
+
+ public CompressedEncodingStrategy(Class<S> type,
+ StorableIndex<S> pkIndex,
+ CompressionType compressionType) {
+ super(type, pkIndex);
+ mCompressionType = compressionType;
+ }
+
+ @Override
+ protected void extraDataEncoding(CodeAssembler a,
+ LocalVariable dataVar, int prefix, int suffix)
+ {
+ switch (mCompressionType) {
+ case GZIP:
+ TypeDesc byteArrayType = TypeDesc.forClass(byte[].class);
+ a.loadLocal(dataVar);
+ a.loadConstant(prefix);
+ a.invokeStatic(GzipCompressor.class.getName(), "compress", byteArrayType,
+ new TypeDesc[] {byteArrayType, TypeDesc.INT});
+ a.storeLocal(dataVar);
+ break;
+ }
+ }
+
+ @Override
+ protected void extraDataDecoding(CodeAssembler a,
+ LocalVariable dataVar, int prefix, int suffix)
+ {
+ switch (mCompressionType) {
+ case GZIP:
+ TypeDesc byteArrayType = TypeDesc.forClass(byte[].class);
+ a.loadLocal(dataVar);
+ a.loadConstant(prefix);
+ a.invokeStatic(GzipCompressor.class.getName(), "decompress", byteArrayType,
+ new TypeDesc[] {byteArrayType, TypeDesc.INT});
+ a.storeLocal(dataVar);
+ break;
+ }
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj instanceof CompressedEncodingStrategy) {
+ CompressedEncodingStrategy other = (CompressedEncodingStrategy) obj;
+ return super.equals(obj) && mCompressionType.equals(other.mCompressionType);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode() + mCompressionType.hashCode();
+ }
+}
diff --git a/src/main/java/com/amazon/carbonado/raw/CompressedStorableCodecFactory.java b/src/main/java/com/amazon/carbonado/raw/CompressedStorableCodecFactory.java
new file mode 100644
index 0000000..b4e6b62
--- /dev/null
+++ b/src/main/java/com/amazon/carbonado/raw/CompressedStorableCodecFactory.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2009 Amazon Technologies, Inc. or its affiliates.
+ * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks
+ * of Amazon Technologies, Inc. or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.amazon.carbonado.raw;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.amazon.carbonado.Storable;
+import com.amazon.carbonado.SupportException;
+
+import com.amazon.carbonado.info.StorableIndex;
+
+import com.amazon.carbonado.layout.LayoutOptions;
+
+/**
+ * Extension of GenericStorableCodecFactory that allows for compression.
+ *
+ * @author Olga Kuznetsova
+ * @author Brian S O'Neill
+ */
+public class CompressedStorableCodecFactory extends GenericStorableCodecFactory {
+ private final Map<String, CompressionType> mCompressionMap;
+
+ public CompressedStorableCodecFactory(Map<String, CompressionType> compressionMap) {
+ if (compressionMap == null || compressionMap.isEmpty()) {
+ mCompressionMap = Collections.emptyMap();
+ } else {
+ mCompressionMap = new HashMap<String, CompressionType>(compressionMap);
+ }
+ }
+
+ @Override
+ public LayoutOptions getLayoutOptions(Class<? extends Storable> type) {
+ CompressionType compType = getCompressionType(type);
+ if (compType == CompressionType.NONE) {
+ return null;
+ }
+ LayoutOptions options = new LayoutOptions();
+ options.setCompressionType(compType.toString());
+ return options;
+ }
+
+ @Override
+ protected <S extends Storable> GenericEncodingStrategy<S> createStrategy
+ (Class<S> type,
+ StorableIndex<S> pkIndex,
+ LayoutOptions options)
+ throws SupportException
+ {
+ CompressionType compType;
+ if (options == null) {
+ compType = getCompressionType(type);
+ } else {
+ compType = CompressionType.valueOf(options.getCompressionType());
+ }
+
+ return new CompressedEncodingStrategy<S>(type, pkIndex, compType);
+ }
+
+ /**
+ * @return non-null compression type for the given storable
+ */
+ protected CompressionType getCompressionType(Class<? extends Storable> type) {
+ CompressionType compType = mCompressionMap.get(type.getName());
+ return compType == null ? CompressionType.NONE : compType;
+ }
+}
diff --git a/src/main/java/com/amazon/carbonado/raw/CompressionType.java b/src/main/java/com/amazon/carbonado/raw/CompressionType.java
new file mode 100644
index 0000000..4061e45
--- /dev/null
+++ b/src/main/java/com/amazon/carbonado/raw/CompressionType.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009 Amazon Technologies, Inc. or its affiliates.
+ * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks
+ * of Amazon Technologies, Inc. or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.amazon.carbonado.raw;
+
+/**
+ * Available compression types for any particular storable.
+ *
+ * @author Olga Kuznetsova
+ */
+public enum CompressionType {
+ /**
+ * No compression.
+ */
+ NONE,
+ /**
+ * Gzip compression.
+ */
+ GZIP;
+}
diff --git a/src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java b/src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java
index 05cc5dd..70c5b7c 100644
--- a/src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java
+++ b/src/main/java/com/amazon/carbonado/raw/GenericEncodingStrategy.java
@@ -534,6 +534,28 @@ public class GenericEncodingStrategy<S extends Storable> {
return infos;
}
+ /**
+ * Second phase encoding, which does nothing by default.
+ *
+ * @param dataVar local variable referencing a byte array with data
+ * @param prefix prefix of byte array to preserve
+ * @param suffix suffix of byte array to preserve
+ */
+ protected void extraDataEncoding(CodeAssembler a,
+ LocalVariable dataVar, int prefix, int suffix)
+ {
+ }
+
+ /**
+ * Second phase decoding, which does nothing by default.
+ *
+ * @param dataVar local variable referencing a byte array with data
+ */
+ protected void extraDataDecoding(CodeAssembler a,
+ LocalVariable dataVar, int prefix, int suffix)
+ {
+ }
+
private SupportException notSupported(StorableProperty<S> property) {
return notSupported(property.getName(),
TypeDesc.forClass(property.getType()).getFullName());
@@ -735,6 +757,10 @@ public class GenericEncodingStrategy<S extends Storable> {
encodeGeneration(a, encodedVar, prefix, generation);
+ if (mode == Mode.DATA) {
+ extraDataEncoding(a, encodedVar, prefix + generationPrefix, suffix);
+ }
+
return encodedVar;
}
}
@@ -1252,6 +1278,10 @@ public class GenericEncodingStrategy<S extends Storable> {
exitPoint.setLocation();
+ if (mode == Mode.DATA) {
+ extraDataEncoding(a, encodedVar, prefix + generationPrefix, suffix);
+ }
+
return encodedVar;
}
@@ -1839,6 +1869,8 @@ public class GenericEncodingStrategy<S extends Storable> {
break;
}
+ decodeGeneration(a, encodedVar, prefix, generation, altGenerationHandler);
+
final int generationPrefix;
if (generation < 0) {
generationPrefix = 0;
@@ -1858,6 +1890,7 @@ public class GenericEncodingStrategy<S extends Storable> {
break;
case DATA:
suffix = mDataSuffixPadding;
+ extraDataDecoding(a, encodedVar, prefix + generationPrefix, suffix);
break;
}
@@ -1865,8 +1898,6 @@ public class GenericEncodingStrategy<S extends Storable> {
StorablePropertyInfo[] infos = checkSupport(properties);
- decodeGeneration(a, encodedVar, prefix, generation, altGenerationHandler);
-
if (properties.length == 1) {
StorableProperty<S> property = properties[0];
StorablePropertyInfo info = infos[0];
diff --git a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java
index 7bd9407..745f761 100644
--- a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java
+++ b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodec.java
@@ -744,9 +744,9 @@ public class GenericStorableCodec<S extends Storable> implements StorableCodec<S
Class<? extends Storable> altStorable;
GenericEncodingStrategy<? extends Storable> altStrategy;
try {
- altStorable = mLayout.getGeneration(generation)
- .reconstruct(mStorableClass.getClassLoader());
- altStrategy = mFactory.createStrategy(altStorable, null);
+ Layout altLayout = mLayout.getGeneration(generation);
+ altStorable = altLayout.reconstruct(mStorableClass.getClassLoader());
+ altStrategy = mFactory.createStrategy(altStorable, null, altLayout.getOptions());
} catch (RepositoryException e) {
throw new CorruptEncodingException(e);
}
diff --git a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java
index 770e4e2..6a472a9 100644
--- a/src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java
+++ b/src/main/java/com/amazon/carbonado/raw/GenericStorableCodecFactory.java
@@ -22,7 +22,9 @@ import com.amazon.carbonado.Storable;
import com.amazon.carbonado.SupportException;
import com.amazon.carbonado.info.StorableIndex;
+
import com.amazon.carbonado.layout.Layout;
+import com.amazon.carbonado.layout.LayoutOptions;
/**
* Factory for generic codec that supports any kind of storable by
@@ -42,6 +44,14 @@ public class GenericStorableCodecFactory implements StorableCodecFactory {
}
/**
+ * Returns null.
+ */
+ @Override
+ public LayoutOptions getLayoutOptions(Class<? extends Storable> type) {
+ return null;
+ }
+
+ /**
* @param type type of storable to create codec for
* @param pkIndex suggested index for primary key (optional)
* @param isMaster when true, version properties and sequences are managed
@@ -80,7 +90,7 @@ public class GenericStorableCodecFactory implements StorableCodecFactory {
throws SupportException
{
return GenericStorableCodec.getInstance
- (this, createStrategy(type, pkIndex), isMaster, layout, support);
+ (this, createStrategy(type, pkIndex, null), isMaster, layout, support);
}
/**
@@ -95,4 +105,20 @@ public class GenericStorableCodecFactory implements StorableCodecFactory {
{
return new GenericEncodingStrategy<S>(type, pkIndex);
}
+
+ /**
+ * Override to return a different EncodingStrategy.
+ *
+ * @param type type of Storable to generate code for
+ * @param pkIndex specifies sequence and ordering of key properties (optional)
+ * @param options additional layout options (optional)
+ * @since 1.2.1
+ */
+ protected <S extends Storable> GenericEncodingStrategy<S> createStrategy
+ (Class<S> type, StorableIndex<S> pkIndex, LayoutOptions options)
+ throws SupportException
+ {
+ // Call into original method for backwards compatibility.
+ return createStrategy(type, pkIndex);
+ }
}
diff --git a/src/main/java/com/amazon/carbonado/raw/GzipCompressor.java b/src/main/java/com/amazon/carbonado/raw/GzipCompressor.java
new file mode 100644
index 0000000..e353d62
--- /dev/null
+++ b/src/main/java/com/amazon/carbonado/raw/GzipCompressor.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2009 Amazon Technologies, Inc. or its affiliates.
+ * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks
+ * of Amazon Technologies, Inc. or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.amazon.carbonado.raw;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import java.util.zip.Deflater;
+import java.util.zip.DeflaterOutputStream;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterOutputStream;
+
+import com.amazon.carbonado.CorruptEncodingException;
+import com.amazon.carbonado.SupportException;
+
+/**
+ * Raw-level data compression using gzip.
+ *
+ * @author Olga Kuznetsova
+ * @author Brian S O'Neill
+ */
+public class GzipCompressor {
+ // NOTE: Class has to be public since it is accessed by generated code.
+
+ final private static ThreadLocal<Deflater> cLocalDeflater = new ThreadLocal<Deflater>();
+ final private static ThreadLocal<Inflater> cLocalInflater = new ThreadLocal<Inflater>();
+
+ /**
+ * Encodes into compressed form.
+ *
+ * @param value value to compress
+ * @param prefix prefix of byte array to preserve
+ * @return compressed value
+ * @throws SupportException thrown if compression failed
+ */
+ public static byte[] compress(byte[] value, int prefix) throws SupportException {
+ Deflater compressor = cLocalDeflater.get();
+ if (compressor == null) {
+ cLocalDeflater.set(compressor = new Deflater());
+ }
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream(value.length);
+
+ try {
+ bos.write(value, 0, prefix);
+ DeflaterOutputStream dout = new DeflaterOutputStream(bos, compressor);
+ dout.write(value, prefix, value.length - prefix);
+ dout.close();
+ return bos.toByteArray();
+ } catch (IOException e) {
+ throw new SupportException(e);
+ } finally {
+ compressor.reset();
+ }
+ }
+
+ /**
+ * Decodes from compressed form.
+ *
+ * @param value value to decompress
+ * @param prefix prefix of byte array to preserve
+ * @return decompressed value
+ * @throws CorruptEncodingException thrown if value cannot be decompressed
+ */
+ public static byte[] decompress(byte[] value, int prefix) throws CorruptEncodingException {
+ Inflater inflater = cLocalInflater.get();
+ if (inflater == null) {
+ cLocalInflater.set(inflater = new Inflater());
+ }
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream(value.length * 2);
+
+ try {
+ bos.write(value, 0, prefix);
+ InflaterOutputStream ios = new InflaterOutputStream(bos, inflater);
+ ios.write(value, prefix, value.length - prefix);
+ ios.close();
+ return bos.toByteArray();
+ } catch (IOException e) {
+ throw new CorruptEncodingException(e);
+ } finally {
+ inflater.reset();
+ }
+ }
+}
diff --git a/src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java b/src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java
index 8cc5be2..8c08ec0 100644
--- a/src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java
+++ b/src/main/java/com/amazon/carbonado/raw/StorableCodecFactory.java
@@ -23,6 +23,7 @@ import com.amazon.carbonado.SupportException;
import com.amazon.carbonado.info.StorableIndex;
import com.amazon.carbonado.layout.Layout;
+import com.amazon.carbonado.layout.LayoutOptions;
/**
* Factory for creating instances of {@link StorableCodec}.
@@ -39,6 +40,13 @@ public interface StorableCodecFactory {
String getStorageName(Class<? extends Storable> type) throws SupportException;
/**
+ * Optionally return additional information regarding storable encoding.
+ *
+ * @since 1.2.1
+ */
+ LayoutOptions getLayoutOptions(Class<? extends Storable> type);
+
+ /**
* @param type type of storable to create codec for
* @param pkIndex suggested index for primary key (optional)
* @param isMaster when true, version properties and sequences are managed
diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java
index 6fbc4f5..28fc347 100644
--- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java
+++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java
@@ -49,6 +49,7 @@ import com.amazon.carbonado.capability.StorableInfoCapability;
import com.amazon.carbonado.info.StorableIntrospector;
+import com.amazon.carbonado.layout.Layout;
import com.amazon.carbonado.layout.LayoutCapability;
import com.amazon.carbonado.layout.LayoutFactory;
@@ -88,7 +89,8 @@ abstract class BDBRepository<Txn> extends AbstractRepository<Txn>
EnvironmentCapability,
ShutdownCapability,
StorableInfoCapability,
- SequenceCapability
+ SequenceCapability,
+ LayoutCapability
{
private final Log mLog = LogFactory.getLog(getClass());
@@ -168,19 +170,6 @@ abstract class BDBRepository<Txn> extends AbstractRepository<Txn>
getLog().info("Opening repository \"" + getName() + '"');
}
- @Override
- @SuppressWarnings("unchecked")
- public <C extends Capability> C getCapability(Class<C> capabilityType) {
- C cap = super.getCapability(capabilityType);
- if (cap != null) {
- return cap;
- }
- if (capabilityType == LayoutCapability.class) {
- return (C) mLayoutFactory;
- }
- return null;
- }
-
public <S extends Storable> IndexInfo[] getIndexInfo(Class<S> storableType)
throws RepositoryException
{
@@ -341,6 +330,26 @@ abstract class BDBRepository<Txn> extends AbstractRepository<Txn>
}
@Override
+ public Layout layoutFor(Class<? extends Storable> type)
+ throws FetchException, PersistException
+ {
+ try {
+ return ((BDBStorage) storageFor(type)).getLayout(mStorableCodecFactory);
+ } catch (PersistException e) {
+ throw e;
+ } catch (RepositoryException e) {
+ throw e.toFetchException();
+ }
+ }
+
+ @Override
+ public Layout layoutFor(Class<? extends Storable> type, int generation)
+ throws FetchException
+ {
+ return mLayoutFactory.layoutFor(type, generation);
+ }
+
+ @Override
protected void finalize() {
close();
}
diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java
index c1e6196..0beea65 100644
--- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java
+++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java
@@ -34,7 +34,8 @@ import com.amazon.carbonado.RepositoryException;
import com.amazon.carbonado.Storable;
import com.amazon.carbonado.repo.indexed.IndexedRepositoryBuilder;
-import com.amazon.carbonado.raw.GenericStorableCodecFactory;
+import com.amazon.carbonado.raw.CompressionType;
+import com.amazon.carbonado.raw.CompressedStorableCodecFactory;
import com.amazon.carbonado.raw.StorableCodecFactory;
import com.amazon.carbonado.spi.AbstractRepositoryBuilder;
@@ -101,6 +102,7 @@ public class BDBRepositoryBuilder extends AbstractRepositoryBuilder {
private boolean mPrivate;
private boolean mMultiversion;
private boolean mLogInMemory;
+ private Integer mLogFileMaxSize;
private boolean mRunFullRecovery;
private boolean mRunCheckpointer = true;
private int mCheckpointInterval = DEFAULT_CHECKPOINT_INTERVAL;
@@ -110,10 +112,11 @@ public class BDBRepositoryBuilder extends AbstractRepositoryBuilder {
private Boolean mChecksumEnabled;
private Object mInitialEnvConfig = null;
private Object mInitialDBConfig = null;
- private StorableCodecFactory mStorableCodecFactory = new GenericStorableCodecFactory();
+ private StorableCodecFactory mStorableCodecFactory;
private Runnable mPreShutdownHook;
private Runnable mPostShutdownHook;
private DatabaseHook mDatabaseHook;
+ private Map<String, CompressionType> mCompressionMap;
public BDBRepositoryBuilder() {
}
@@ -136,6 +139,10 @@ public class BDBRepositoryBuilder extends AbstractRepositoryBuilder {
}
}
+ if (mStorableCodecFactory == null) {
+ mStorableCodecFactory = new CompressedStorableCodecFactory(mCompressionMap);
+ }
+
assertReady();
// Make environment directory if it doesn't exist.
@@ -860,6 +867,40 @@ public class BDBRepositoryBuilder extends AbstractRepositoryBuilder {
return mDatabaseHook;
}
+ /**
+ * Set the compressor for the given class, overriding a custom StorableCodecFactory.
+
+ * @param type Storable to compress.
+ * @param compressionType String representation of type of
+ * compression. Available options are "NONE" for no compression or "GZIP"
+ * for gzip compression
+ */
+ public void setCompressor(String type, String compressionType) {
+ mStorableCodecFactory = null;
+ compressionType = compressionType.toUpperCase();
+ if (mCompressionMap == null) {
+ mCompressionMap = new HashMap<String, CompressionType>();
+ }
+ CompressionType compressionEnum = CompressionType.valueOf(compressionType);
+ if (compressionEnum != null) {
+ mCompressionMap.put(type, compressionEnum);
+ }
+ }
+
+ /**
+ * Return the compressor used for the given storable.
+ * @param type Storable to compress
+ * @return String representation of the type of compression used. Available options are "NONE"
+ * for no compression and "GZIP" for gzip compression.
+ */
+ public String getCompressor(String type) {
+ if (mCompressionMap == null) {
+ return null;
+ }
+
+ return mCompressionMap.get(type).toString();
+ }
+
private long inMicros(double seconds) {
if (seconds >= Long.MAX_VALUE) {
return Long.MAX_VALUE;
diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java
index 4f1ff29..d00b7d5 100644
--- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java
+++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBStorage.java
@@ -60,6 +60,7 @@ import com.amazon.carbonado.info.StorableProperty;
import com.amazon.carbonado.layout.Layout;
import com.amazon.carbonado.layout.LayoutFactory;
+import com.amazon.carbonado.layout.LayoutOptions;
import com.amazon.carbonado.layout.Unevolvable;
import com.amazon.carbonado.lob.Blob;
@@ -416,11 +417,9 @@ abstract class BDBStorage<Txn, S extends Storable> implements Storage<S>, Storag
protected void open(boolean readOnly, Txn openTxn, boolean installTriggers)
throws RepositoryException
{
- final Layout layout = getLayout();
-
StorableInfo<S> info = StorableIntrospector.examine(getStorableType());
-
StorableCodecFactory codecFactory = mRepository.getStorableCodecFactory();
+ final Layout layout = getLayout(codecFactory);
// Open primary database.
Object primaryDatabase;
@@ -767,7 +766,7 @@ abstract class BDBStorage<Txn, S extends Storable> implements Storage<S>, Storag
}
}
- private Layout getLayout() throws RepositoryException {
+ Layout getLayout(StorableCodecFactory codecFactory) throws RepositoryException {
if (Unevolvable.class.isAssignableFrom(getStorableType())) {
// Don't record generation for storables marked as unevolvable.
return null;
@@ -782,7 +781,8 @@ abstract class BDBStorage<Txn, S extends Storable> implements Storage<S>, Storag
return null;
}
- return factory.layoutFor(getStorableType());
+ Class<S> type = getStorableType();
+ return factory.layoutFor(type, codecFactory.getLayoutOptions(type));
}
/**