From 48d08bb552557dac9a923c26a54dd7025318592a Mon Sep 17 00:00:00 2001 From: "Brian S. O'Neill" Date: Thu, 29 Mar 2012 15:22:18 +0000 Subject: Added BDB panic handling support. --- .../carbonado/repo/sleepycat/BDBPanicHandler.java | 35 + .../carbonado/repo/sleepycat/BDBRepository.java | 2251 +++++++++---------- .../repo/sleepycat/BDBRepositoryBuilder.java | 2305 ++++++++++---------- 3 files changed, 2326 insertions(+), 2265 deletions(-) create mode 100644 src/main/java/com/amazon/carbonado/repo/sleepycat/BDBPanicHandler.java (limited to 'src/main/java/com/amazon') diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBPanicHandler.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBPanicHandler.java new file mode 100644 index 0000000..3e5aabe --- /dev/null +++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBPanicHandler.java @@ -0,0 +1,35 @@ +/* + * Copyright 2012 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.repo.sleepycat; + +/** + * Interface for a generic panic handler for any BDB products. + * + * @author Jesse Morgan + * + */ +public interface BDBPanicHandler { + /** + * Called when an Environment panics or an EnvironmentFailureException is thrown. + * + * @param environment The affected environment or null if the environment could not be opened. + * @param exception The related exception. + */ + void onPanic(Object environment, Exception exception); +} diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java index 07d0192..04cc87d 100644 --- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java +++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java @@ -1,1123 +1,1128 @@ -/* - * Copyright 2006-2012 Amazon Technologies, Inc. or its affiliates. - * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks - * of Amazon Technologies, Inc. or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.amazon.carbonado.repo.sleepycat; - -import java.io.File; -import java.io.PrintStream; -import java.lang.ref.WeakReference; -import java.util.ArrayList; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import com.amazon.carbonado.ConfigurationException; -import com.amazon.carbonado.Cursor; -import com.amazon.carbonado.FetchException; -import com.amazon.carbonado.IsolationLevel; -import com.amazon.carbonado.MalformedArgumentException; -import com.amazon.carbonado.PersistException; -import com.amazon.carbonado.Repository; -import com.amazon.carbonado.RepositoryException; -import com.amazon.carbonado.Storable; -import com.amazon.carbonado.Storage; -import com.amazon.carbonado.Transaction; -import com.amazon.carbonado.TriggerFactory; - -import com.amazon.carbonado.capability.Capability; -import com.amazon.carbonado.capability.IndexInfo; -import com.amazon.carbonado.capability.IndexInfoCapability; -import com.amazon.carbonado.capability.ShutdownCapability; -import com.amazon.carbonado.capability.StorableInfoCapability; - -import com.amazon.carbonado.info.StorableIntrospector; - -import com.amazon.carbonado.layout.Layout; -import com.amazon.carbonado.layout.LayoutCapability; -import com.amazon.carbonado.layout.LayoutFactory; - -import com.amazon.carbonado.qe.RepositoryAccess; -import com.amazon.carbonado.qe.StorageAccess; - -import com.amazon.carbonado.raw.StorableCodecFactory; - -import com.amazon.carbonado.sequence.SequenceCapability; -import com.amazon.carbonado.sequence.SequenceValueGenerator; -import com.amazon.carbonado.sequence.SequenceValueProducer; - -import com.amazon.carbonado.spi.AbstractRepository; -import com.amazon.carbonado.spi.ExceptionTransformer; -import com.amazon.carbonado.spi.LobEngine; - -import com.amazon.carbonado.txn.TransactionManager; -import com.amazon.carbonado.txn.TransactionScope; - -/** - * Repository implementation backed by a Berkeley DB. Data is encoded in the - * BDB in a specialized format, and so this repository should not be used to - * open arbitrary Berkeley databases. BDBRepository has total schema ownership, - * and so it updates type definitions in the storage layer automatically. - * - * @author Brian S O'Neill - * @author Vidya Iyer - * @author Nicole Deflaux - * @author bcastill - */ -abstract class BDBRepository extends AbstractRepository - implements Repository, - RepositoryAccess, - IndexInfoCapability, - HotBackupCapability, - CheckpointCapability, - EnvironmentCapability, - ShutdownCapability, - StorableInfoCapability, - SequenceCapability, - LayoutCapability -{ - private final Log mLog = LogFactory.getLog(getClass()); - - private final boolean mIsMaster; - final Iterable mTriggerFactories; - private final AtomicReference mRootRef; - private final StorableCodecFactory mStorableCodecFactory; - private final ExceptionTransformer mExTransformer; - private final BDBTransactionManager mTxnMgr; - - Checkpointer mCheckpointer; - DeadlockDetector mDeadlockDetector; - - private final Runnable mPreShutdownHook; - private final Runnable mPostShutdownHook; - - private final Object mInitialDBConfig; - private final BDBRepositoryBuilder.DatabaseHook mDatabaseHook; - private final Map, Integer> mDatabasePageSizes; - - final boolean mRunCheckpointer; - final boolean mKeepOldLogFiles; - final boolean mLogInMemory; - final boolean mRunDeadlockDetector; - - final File mDataHome; - final File mEnvHome; - final String mSingleFileName; - final Map mFileNameMap; - - final Object mBackupLock = new Object(); - int mBackupCount = 0; - int mIncrementalBackupCount = 0; - - private LayoutFactory mLayoutFactory; - - private LobEngine mLobEngine; - - /** - * Subclass must call protected start method to fully initialize - * BDBRepository. - * - * @param builder repository configuration - * @param exTransformer transformer for exceptions - * @throws IllegalArgumentException if name or environment home is null - */ - @SuppressWarnings("unchecked") - BDBRepository(AtomicReference rootRef, - BDBRepositoryBuilder builder, - ExceptionTransformer exTransformer) - throws ConfigurationException - { - super(builder.getName()); - - builder.assertReady(); - - if (exTransformer == null) { - throw new IllegalArgumentException("Exception transformer must not be null"); - } - - mIsMaster = builder.isMaster(); - mTriggerFactories = builder.getTriggerFactories(); - mRootRef = rootRef; - mExTransformer = exTransformer; - mTxnMgr = new BDBTransactionManager(mExTransformer, this); - - mRunCheckpointer = !builder.getReadOnly() && builder.getRunCheckpointer(); - mKeepOldLogFiles = builder.getKeepOldLogFiles(); - mLogInMemory = builder.getLogInMemory(); - mRunDeadlockDetector = builder.getRunDeadlockDetector(); - mStorableCodecFactory = builder.getStorableCodecFactory(); - mPreShutdownHook = builder.getPreShutdownHook(); - mPostShutdownHook = builder.getShutdownHook(); - mInitialDBConfig = builder.getInitialDatabaseConfig(); - mDatabaseHook = builder.getDatabaseHook(); - mDatabasePageSizes = builder.getDatabasePagesMap(); - mDataHome = builder.getDataHomeFile(); - mEnvHome = builder.getEnvironmentHomeFile(); - mSingleFileName = builder.getSingleFileName(); - mFileNameMap = builder.getFileNameMap(); - - getLog().info("Opening repository \"" + getName() + '"'); - } - - public IndexInfo[] getIndexInfo(Class storableType) - throws RepositoryException - { - return ((BDBStorage) storageFor(storableType)).getIndexInfo(); - } - - public String[] getUserStorableTypeNames() throws RepositoryException { - Repository metaRepo = getRootRepository(); - - Cursor cursor = - metaRepo.storageFor(StoredDatabaseInfo.class) - .query().orderBy("databaseName").fetch(); - - try { - ArrayList names = new ArrayList(); - while (cursor.hasNext()) { - StoredDatabaseInfo info = cursor.next(); - // Ordinary user types support evolution. - if (info.getEvolutionStrategy() != StoredDatabaseInfo.EVOLUTION_NONE) { - names.add(info.getDatabaseName()); - } - } - - return names.toArray(new String[names.size()]); - } finally { - cursor.close(); - } - } - - public boolean isSupported(Class type) { - if (type == null) { - return false; - } - StorableIntrospector.examine(type); - return true; - } - - public boolean isPropertySupported(Class type, String name) { - if (type == null || name == null) { - return false; - } - return StorableIntrospector.examine(type).getAllProperties().get(name) != null; - } - - @Override - public Backup startBackup() throws RepositoryException { - return startBackup(false); - } - - @Override - public Backup startBackup(boolean deleteOldLogFiles) throws RepositoryException { - if (mLogInMemory) { - throw new IllegalStateException - ("Log files are only kept in memory and backups cannot be performed"); - } - - synchronized (mBackupLock) { - int count = mBackupCount; - if (count == 0) { - try { - if (deleteOldLogFiles) { - // TODO: If backup rejects log deletion, queue up for later. - enterBackupMode(true); - } else { - // Call old API for backwards compatibility. - enterBackupMode(); - } - } catch (Exception e) { - throw mExTransformer.toRepositoryException(e); - } - } - mBackupCount = count + 1; - - return new FullBackup(); - } - } - - @Override - public Backup startIncrementalBackup(long lastLogNumber) - throws RepositoryException - { - return startIncrementalBackup(lastLogNumber, false); - } - - @Override - public Backup startIncrementalBackup(long lastLogNumber, boolean deleteOldLogFiles) - throws RepositoryException - { - if (mLogInMemory) { - throw new IllegalStateException - ("Log files are only kept in memory and incremental backup cannot be performed"); - } - - if (lastLogNumber < 0) { - throw new IllegalArgumentException - ("The number of the last backup cannot be negative: " + lastLogNumber); - } - synchronized (mBackupLock) { - try { - enterIncrementalBackupMode(lastLogNumber, deleteOldLogFiles); - ++mIncrementalBackupCount; - } catch (Exception e) { - throw mExTransformer.toRepositoryException(e); - } - } - return new IncrementalBackup(lastLogNumber); - } - - /** - * Suspend the checkpointer until the suspension time has expired or until - * manually resumed. If a checkpoint is in progress, this method will block - * until it is finished. If checkpointing is disabled, calling this method - * has no effect. - * - *

Calling this method repeatedly resets the suspension time. This - * technique should be used by hot backup processes to ensure that its - * failure does not leave the checkpointer permanently suspended. Each - * invocation of suspendCheckpointer is like a lease renewal or heartbeat. - * - * @param suspensionTime minimum length of suspension, in milliseconds, - * unless checkpointer is manually resumed - */ - public void suspendCheckpointer(long suspensionTime) { - if (mCheckpointer != null) { - mCheckpointer.suspendCheckpointer(suspensionTime); - } - } - - /** - * Resumes the checkpointer if it was suspended. If checkpointing is - * disabled or if not suspended, calling this method has no effect. - */ - public void resumeCheckpointer() { - if (mCheckpointer != null) { - mCheckpointer.resumeCheckpointer(); - } - } - - /** - * Forces a checkpoint to run now, even if checkpointer is suspended or - * disabled. If a checkpoint is in progress, then this method will block - * until it is finished, and then run another checkpoint. This method does - * not return until the requested checkpoint has finished. - */ - public void forceCheckpoint() throws PersistException { - if (mCheckpointer != null) { - mCheckpointer.forceCheckpoint(); - } else { - try { - env_checkpoint(); - } catch (Exception e) { - throw toPersistException(e); - } - } - } - - public void sync() throws PersistException { - try { - env_sync(); - } catch (Exception e) { - throw toPersistException(e); - } - } - - public Repository getRootRepository() { - return mRootRef.get(); - } - - public StorageAccess storageAccessFor(Class type) - throws RepositoryException - { - return (BDBStorage) storageFor(type); - } - - @Override - public Layout layoutFor(Class type) - throws FetchException, PersistException - { - try { - return ((BDBStorage) storageFor(type)).getLayout(true, mStorableCodecFactory); - } catch (PersistException e) { - throw e; - } catch (RepositoryException e) { - throw e.toFetchException(); - } - } - - @Override - public Layout layoutFor(Class type, int generation) - throws FetchException - { - return mLayoutFactory.layoutFor(type, generation); - } - - @Override - protected void finalize() { - close(); - } - - @Override - protected void shutdownHook() { - // Run any external shutdown logic that needs to happen before the - // databases and the environment are actually closed - if (mPreShutdownHook != null) { - mPreShutdownHook.run(); - } - - // Close database handles. - for (Storage storage : allStorage()) { - try { - if (storage instanceof BDBStorage) { - ((BDBStorage) storage).close(); - } - } catch (Throwable e) { - getLog().error(null, e); - } - } - - // Wait for checkpointer to finish. - if (mCheckpointer != null) { - mCheckpointer.interrupt(); - try { - mCheckpointer.join(); - } catch (InterruptedException e) { - } - } - - // Wait for deadlock detector to finish. - if (mDeadlockDetector != null) { - mDeadlockDetector.interrupt(); - try { - mDeadlockDetector.join(); - } catch (InterruptedException e) { - } - } - - // Close environment. - try { - env_close(); - } catch (Throwable e) { - getLog().error(null, e); - } - - if (mPostShutdownHook != null) { - mPostShutdownHook.run(); - } - } - - @Override - protected Log getLog() { - return mLog; - } - - @Override - protected Storage createStorage(Class type) - throws RepositoryException - { - try { - return createBDBStorage(type); - } catch (MalformedArgumentException e) { - throw e; - } catch (Exception e) { - throw toRepositoryException(e); - } - } - - @Override - protected SequenceValueProducer createSequenceValueProducer(String name) - throws RepositoryException - { - return new SequenceValueGenerator(BDBRepository.this, name); - } - - /** - * @see com.amazon.carbonado.spi.RepositoryBuilder#isMaster - */ - boolean isMaster() { - return mIsMaster; - } - - String[] getAllDatabaseNames() throws RepositoryException { - Repository metaRepo = getRootRepository(); - - Cursor cursor = - metaRepo.storageFor(StoredDatabaseInfo.class) - .query().orderBy("databaseName").fetch(); - - ArrayList names = new ArrayList(); - // This one needs to manually added since it is the metadata db itself. - names.add(StoredDatabaseInfo.class.getName()); - - try { - while (cursor.hasNext()) { - names.add(cursor.next().getDatabaseName()); - } - } finally { - cursor.close(); - } - - return names.toArray(new String[names.size()]); - } - - String getDatabaseFileName(final String dbName) { - String singleFileName = mSingleFileName; - if (singleFileName == null && mFileNameMap != null) { - singleFileName = mFileNameMap.get(dbName); - if (singleFileName == null && dbName != null) { - singleFileName = mFileNameMap.get(null); - } - } - - String dbFileName = dbName; - - if (singleFileName == null) { - if (mDatabaseHook != null) { - dbFileName = mDatabaseHook.databaseName(dbName); - } - } else { - dbFileName = singleFileName; - } - - if (mDataHome != null && !mDataHome.equals(mEnvHome)) { - dbFileName = new File(mDataHome, dbFileName).getPath(); - } - - return dbFileName; - } - - /** - * Returns null if name should not be used. - */ - String getDatabaseName(String dbName) { - if (mFileNameMap == null) { - return null; - } - String name = mFileNameMap.get(dbName); - if (name == null && dbName != null) { - name = mFileNameMap.get(null); - } - if (name == null) { - return null; - } - if (mDatabaseHook != null) { - try { - dbName = mDatabaseHook.databaseName(dbName); - } catch (IncompatibleClassChangeError e) { - // Method not implemented. - } - } - return dbName; - } - - StorableCodecFactory getStorableCodecFactory() { - return mStorableCodecFactory; - } - - LayoutFactory getLayoutFactory() throws RepositoryException { - if (mLayoutFactory == null) { - mLayoutFactory = new LayoutFactory(getRootRepository()); - } - return mLayoutFactory; - } - - LobEngine getLobEngine() throws RepositoryException { - if (mLobEngine == null) { - mLobEngine = new LobEngine(this, getRootRepository()); - } - return mLobEngine; - } - - /** - * Returns the optional BDB specific database configuration to use - * for all databases created. - */ - public Object getInitialDatabaseConfig() { - return mInitialDBConfig; - } - - /** - * Returns the desired page size for the given type, or null for default. - */ - Integer getDatabasePageSize(Class type) { - if (mDatabasePageSizes == null) { - return null; - } - Integer size = mDatabasePageSizes.get(type); - if (size == null && type != null) { - size = mDatabasePageSizes.get(null); - } - return size; - } - - void runDatabasePrepareForOpeningHook(Object database) throws RepositoryException { - if (mDatabaseHook != null) { - mDatabaseHook.prepareForOpening(database); - } - } - - /** - * Start background tasks and enable auto shutdown. - * - * @param checkpointInterval how often to run checkpoints, in milliseconds, - * or zero if never. Ignored if repository is read only or builder has - * checkpoints disabled. - * @param deadlockDetectorInterval how often to run deadlock detector, in - * milliseconds, or zero if never. Ignored if builder has deadlock detector - * disabled. - * - * @deprecated Overloaded for backwards compatiblity with older - * CarbonadoSleepycat packages - */ - void start(long checkpointInterval, long deadlockDetectorInterval) { - getLog().info("Opened repository \"" + getName() + '"'); - - if (mRunCheckpointer && checkpointInterval > 0) { - mCheckpointer = new Checkpointer(this, checkpointInterval, 1024, 5); - mCheckpointer.start(); - } else { - mCheckpointer = null; - } - - if (mRunDeadlockDetector && deadlockDetectorInterval > 0) { - mDeadlockDetector = new DeadlockDetector(this, deadlockDetectorInterval); - mDeadlockDetector.start(); - } else { - mDeadlockDetector = null; - } - - setAutoShutdownEnabled(true); - } - - /** - * Start background tasks and enable auto shutdown. - * - * @param checkpointInterval how often to run checkpoints, in milliseconds, - * or zero if never. Ignored if repository is read only or builder has - * checkpoints disabled. - * @param deadlockDetectorInterval how often to run deadlock detector, in - * milliseconds, or zero if never. Ignored if builder has deadlock detector - * disabled. - * @param builder containing additonal background task properties. - */ - void start(long checkpointInterval, long deadlockDetectorInterval, - BDBRepositoryBuilder builder) { - getLog().info("Opened repository \"" + getName() + '"'); - - if (mRunCheckpointer && checkpointInterval > 0) { - mCheckpointer = new Checkpointer(this, checkpointInterval, - builder.getCheckpointThresholdKB(), - builder.getCheckpointThresholdMinutes()); - mCheckpointer.start(); - } else { - mCheckpointer = null; - } - - if (mRunDeadlockDetector && deadlockDetectorInterval > 0) { - mDeadlockDetector = new DeadlockDetector(this, deadlockDetectorInterval); - mDeadlockDetector.start(); - } else { - mDeadlockDetector = null; - } - - setAutoShutdownEnabled(true); - } - - abstract boolean verify(PrintStream out) throws Exception; - - abstract IsolationLevel selectIsolationLevel(Transaction parent, IsolationLevel level); - - abstract Txn txn_begin(Txn parent, IsolationLevel level) throws Exception; - - // Subclass should override this method to actually apply the timeout - Txn txn_begin(Txn parent, IsolationLevel level, int timeout, TimeUnit unit) throws Exception { - return txn_begin(parent, level); - } - - abstract Txn txn_begin_nowait(Txn parent, IsolationLevel level) throws Exception; - - abstract void txn_commit(Txn txn) throws Exception; - - abstract void txn_abort(Txn txn) throws Exception; - - /** - * Force a checkpoint to run. - */ - abstract void env_checkpoint() throws Exception; - - /** - * Synchronously flush changes to stable storage. - */ - abstract void env_sync() throws Exception; - - /** - * @param kBytes run checkpoint if at least this many kilobytes in log - * @param minutes run checkpoint if at least this many minutes passed since - * last checkpoint - */ - abstract void env_checkpoint(int kBytes, int minutes) throws Exception; - - /** - * Run the deadlock detector. - */ - abstract void env_detectDeadlocks() throws Exception; - - /** - * Close the environment. - */ - abstract void env_close() throws Exception; - - abstract BDBStorage createBDBStorage(Class type) - throws Exception; - - /** - * Called only the first time a backup is started. Old API is kept for - * backwards compatibility. - */ - void enterBackupMode() throws Exception { - enterBackupMode(false); - } - - /** - * Called only the first time a backup is started. - */ - abstract void enterBackupMode(boolean deleteOldLogFiles) throws Exception; - - /** - * Called only after the last backup ends. - */ - abstract void exitBackupMode() throws Exception; - - /** - * Called only when an incremental backup is started. - */ - abstract void enterIncrementalBackupMode(long lastLogNumber, boolean deleteOldLogFiles) - throws Exception; - - /** - * Called only after incremental backup ends. - */ - abstract void exitIncrementalBackupMode() throws Exception; - - /** - * Called only if in backup mode. Old API is kept for backwards - * compatibility. - */ - @Deprecated - File[] backupFiles() throws Exception { - return backupFiles(new long[1]); - } - - @Deprecated - File[] backupFiles(long[] newLastLogNum) throws Exception { - throw new UnsupportedOperationException(); - } - - /** - * Called only if in backup mode. - */ - abstract File[] backupDataFiles() throws Exception; - - /** - * Called only if in backup mode. - * - * @param newLastLogNum reference to last log number at [0] - */ - abstract File[] backupLogFiles(long[] newLastLogNum) throws Exception; - - /** - * Called only if in incremental backup mode. - * - * @param newLastLogNum reference to last log number at [0] - */ - abstract File[] incrementalBackup(long lastLogNumber, long[] newLastLogNum) throws Exception; - - FetchException toFetchException(Throwable e) { - return mExTransformer.toFetchException(e); - } - - PersistException toPersistException(Throwable e) { - return mExTransformer.toPersistException(e); - } - - RepositoryException toRepositoryException(Throwable e) { - return mExTransformer.toRepositoryException(e); - } - - @Override - protected final TransactionManager transactionManager() { - return mTxnMgr; - } - - @Override - protected final TransactionScope localTransactionScope() { - return mTxnMgr.localScope(); - } - - /** - * Periodically runs checkpoints on the environment. - */ - private static class Checkpointer extends Thread { - private final WeakReference mRepository; - private final long mSleepInterval; - private final int mKBytes; - private final int mMinutes; - - private boolean mInProgress; - private long mSuspendUntil = Long.MIN_VALUE; - - /** - * - * @param repository outer class - * @param sleepInterval milliseconds to sleep before running checkpoint - * @param kBytes run checkpoint if at least this many kilobytes in log - * @param minutes run checkpoint if at least this many minutes passed - * since last checkpoint - */ - Checkpointer(BDBRepository repository, long sleepInterval, int kBytes, int minutes) { - super(repository.getClass().getSimpleName() + " checkpointer (" + - repository.getName() + ')'); - setDaemon(true); - mRepository = new WeakReference(repository); - mSleepInterval = sleepInterval; - mKBytes = kBytes; - mMinutes = minutes; - } - - @Override - public void run() { - try { - while (true) { - synchronized (this) { - if (!mInProgress) { - try { - wait(mSleepInterval); - } catch (InterruptedException e) { - break; - } - } - } - - BDBRepository repository = mRepository.get(); - if (repository == null) { - break; - } - - long suspendUntil; - synchronized (this) { - suspendUntil = mSuspendUntil; - } - if (suspendUntil != Long.MIN_VALUE) { - if (System.currentTimeMillis() < suspendUntil) { - continue; - } - } - - Log log = repository.getLog(); - - if (log.isDebugEnabled()) { - log.debug("Running checkpoint on repository \"" + - repository.getName() + '"'); - } - - try { - synchronized (this) { - mInProgress = true; - } - repository.env_checkpoint(mKBytes, mMinutes); - if (log.isDebugEnabled()) { - log.debug("Finished running checkpoint on repository \"" + - repository.getName() + '"'); - } - } catch (ThreadDeath e) { - break; - } catch (Throwable e) { - log.error("Checkpoint failed", e); - } finally { - synchronized (this) { - mInProgress = false; - // Only wait condition is mInProgress, so okay to not call notifyAll. - notify(); - } - repository = null; - } - } - } finally { - synchronized (this) { - mInProgress = false; - // Only wait condition is mInProgress, so okay to not call notifyAll. - notify(); - } - } - } - - /** - * Blocks until checkpoint has finished. - */ - synchronized void suspendCheckpointer(long suspensionTime) { - while (mInProgress) { - try { - wait(); - } catch (InterruptedException e) { - } - } - - if (suspensionTime <= 0) { - return; - } - - long now = System.currentTimeMillis(); - long suspendUntil = now + suspensionTime; - if (now >= 0 && suspendUntil < 0) { - // Overflow. - suspendUntil = Long.MAX_VALUE; - } - mSuspendUntil = suspendUntil; - } - - synchronized void resumeCheckpointer() { - mSuspendUntil = Long.MIN_VALUE; - } - - /** - * Blocks until checkpoint has finished. - */ - synchronized void forceCheckpoint() throws PersistException { - while (mInProgress) { - try { - wait(); - } catch (InterruptedException e) { - return; - } - } - - BDBRepository repository = mRepository.get(); - if (repository != null) { - try { - repository.env_checkpoint(); - } catch (Exception e) { - throw repository.toPersistException(e); - } - } - } - } - - /** - * Periodically runs deadlock detection on the environment. - */ - private static class DeadlockDetector extends Thread { - private final WeakReference mRepository; - private final long mSleepInterval; - - /** - * @param repository outer class - * @param sleepInterval milliseconds to sleep before running deadlock detection - */ - DeadlockDetector(BDBRepository repository, long sleepInterval) { - super(repository.getClass().getSimpleName() + " deadlock detector (" + - repository.getName() + ')'); - setDaemon(true); - mRepository = new WeakReference(repository); - mSleepInterval = sleepInterval; - } - - @Override - public void run() { - while (true) { - try { - Thread.sleep(mSleepInterval); - } catch (InterruptedException e) { - break; - } - - BDBRepository repository = mRepository.get(); - if (repository == null) { - break; - } - - try { - repository.env_detectDeadlocks(); - } catch (ThreadDeath e) { - break; - } catch (Throwable e) { - repository.getLog().error("Deadlock detection failed", e); - } finally { - repository = null; - } - } - } - } - - abstract class AbstractBackup implements Backup { - boolean mDone; - long mFinalLogNumber; - - AbstractBackup() { - mFinalLogNumber = -1; - } - - @Override - public void endBackup() throws RepositoryException { - synchronized (mBackupLock) { - if (mDone) { - return; - } - mDone = true; - finishBackup(); - } - } - - @Override - @Deprecated - public File[] getFiles() throws RepositoryException { - synchronized (mBackupLock) { - File[] data = getDataFiles(); - File[] logs = getLogFiles(); - File[] all = new File[data.length + logs.length]; - System.arraycopy(data, 0, all, 0, data.length); - System.arraycopy(logs, 0, all, data.length, logs.length); - return all; - } - } - - @Override - public File[] getDataFiles() throws RepositoryException { - synchronized (mBackupLock) { - if (mDone) { - throw new IllegalStateException("Backup has ended"); - } - - try { - return getDataBackupFiles(); - } catch (Exception e) { - throw mExTransformer.toRepositoryException(e); - } - } - } - - @Override - public File[] getLogFiles() throws RepositoryException { - synchronized (mBackupLock) { - if (mDone) { - throw new IllegalStateException("Backup has ended"); - } - - try { - long[] newLastLogNum = {-1}; - File[] toReturn = getLogBackupFiles(newLastLogNum); - mFinalLogNumber = newLastLogNum[0]; - return toReturn; - } catch (Exception e) { - throw mExTransformer.toRepositoryException(e); - } - } - } - - @Override - public long getLastLogNumber() throws RepositoryException { - if (mFinalLogNumber < 0) { - throw new IllegalStateException - ("Must get files prior to retrieving the last log number"); - } - return mFinalLogNumber; - } - - abstract void finishBackup() throws RepositoryException; - - abstract File[] getDataBackupFiles() throws Exception; - - abstract File[] getLogBackupFiles(long[] newLastLogNum) throws Exception; - } - - class IncrementalBackup extends AbstractBackup { - private final long mLastLogNumber; - - IncrementalBackup(long lastLogNumber) { - super(); - mLastLogNumber = lastLogNumber; - } - - @Override - void finishBackup() throws RepositoryException { - --mIncrementalBackupCount; - - try { - exitIncrementalBackupMode(); - } catch (Exception e) { - throw mExTransformer.toRepositoryException(e); - } - } - - @Override - File[] getDataBackupFiles() throws Exception { - return new File[0]; - } - - @Override - File[] getLogBackupFiles(long[] newLastLogNum) throws Exception { - return incrementalBackup(mLastLogNumber, newLastLogNum); - } - } - - class FullBackup extends AbstractBackup { - @Override - void finishBackup() throws RepositoryException { - int count = mBackupCount - 1; - try { - if (count == 0) { - try { - exitBackupMode(); - } catch (Exception e) { - throw mExTransformer.toRepositoryException(e); - } - } - } finally { - mBackupCount = count; - } - } - - @Override - File[] getDataBackupFiles() throws Exception { - try { - return backupDataFiles(); - } catch (AbstractMethodError e) { - // Old API will be called for backwards compatibility in the - // getLogBackupFiles method. - return new File[0]; - } - } - - @Override - File[] getLogBackupFiles(long[] newLastLogNum) throws Exception { - try { - return backupLogFiles(newLastLogNum); - } catch (AbstractMethodError e) { - // Call old API for backwards compatibility. - try { - return backupFiles(newLastLogNum); - } catch (AbstractMethodError e2) { - // Call even older API for backwards compatibility. - return backupFiles(); - } - } - } - } -} +/* + * Copyright 2006-2012 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.repo.sleepycat; + +import java.io.File; +import java.io.PrintStream; +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.amazon.carbonado.ConfigurationException; +import com.amazon.carbonado.Cursor; +import com.amazon.carbonado.FetchException; +import com.amazon.carbonado.IsolationLevel; +import com.amazon.carbonado.MalformedArgumentException; +import com.amazon.carbonado.PersistException; +import com.amazon.carbonado.Repository; +import com.amazon.carbonado.RepositoryException; +import com.amazon.carbonado.Storable; +import com.amazon.carbonado.Storage; +import com.amazon.carbonado.Transaction; +import com.amazon.carbonado.TriggerFactory; + +import com.amazon.carbonado.capability.Capability; +import com.amazon.carbonado.capability.IndexInfo; +import com.amazon.carbonado.capability.IndexInfoCapability; +import com.amazon.carbonado.capability.ShutdownCapability; +import com.amazon.carbonado.capability.StorableInfoCapability; + +import com.amazon.carbonado.info.StorableIntrospector; + +import com.amazon.carbonado.layout.Layout; +import com.amazon.carbonado.layout.LayoutCapability; +import com.amazon.carbonado.layout.LayoutFactory; + +import com.amazon.carbonado.qe.RepositoryAccess; +import com.amazon.carbonado.qe.StorageAccess; + +import com.amazon.carbonado.raw.StorableCodecFactory; + +import com.amazon.carbonado.sequence.SequenceCapability; +import com.amazon.carbonado.sequence.SequenceValueGenerator; +import com.amazon.carbonado.sequence.SequenceValueProducer; + +import com.amazon.carbonado.spi.AbstractRepository; +import com.amazon.carbonado.spi.ExceptionTransformer; +import com.amazon.carbonado.spi.LobEngine; + +import com.amazon.carbonado.txn.TransactionManager; +import com.amazon.carbonado.txn.TransactionScope; + +/** + * Repository implementation backed by a Berkeley DB. Data is encoded in the + * BDB in a specialized format, and so this repository should not be used to + * open arbitrary Berkeley databases. BDBRepository has total schema ownership, + * and so it updates type definitions in the storage layer automatically. + * + * @author Brian S O'Neill + * @author Vidya Iyer + * @author Nicole Deflaux + * @author bcastill + */ +abstract class BDBRepository extends AbstractRepository + implements Repository, + RepositoryAccess, + IndexInfoCapability, + HotBackupCapability, + CheckpointCapability, + EnvironmentCapability, + ShutdownCapability, + StorableInfoCapability, + SequenceCapability, + LayoutCapability +{ + private final Log mLog = LogFactory.getLog(getClass()); + + private final boolean mIsMaster; + final Iterable mTriggerFactories; + private final AtomicReference mRootRef; + private final StorableCodecFactory mStorableCodecFactory; + private final ExceptionTransformer mExTransformer; + private final BDBTransactionManager mTxnMgr; + + Checkpointer mCheckpointer; + DeadlockDetector mDeadlockDetector; + + private final Runnable mPreShutdownHook; + private final Runnable mPostShutdownHook; + + private final Object mInitialDBConfig; + private final BDBRepositoryBuilder.DatabaseHook mDatabaseHook; + private final Map, Integer> mDatabasePageSizes; + + final boolean mRunCheckpointer; + final boolean mKeepOldLogFiles; + final boolean mLogInMemory; + final boolean mRunDeadlockDetector; + + final File mDataHome; + final File mEnvHome; + final String mSingleFileName; + final Map mFileNameMap; + + final Object mBackupLock = new Object(); + int mBackupCount = 0; + int mIncrementalBackupCount = 0; + + private LayoutFactory mLayoutFactory; + + private LobEngine mLobEngine; + + /** + * Subclass must call protected start method to fully initialize + * BDBRepository. + * + * @param builder repository configuration + * @param exTransformer transformer for exceptions + * @throws IllegalArgumentException if name or environment home is null + */ + @SuppressWarnings("unchecked") + BDBRepository(AtomicReference rootRef, + BDBRepositoryBuilder builder, + ExceptionTransformer exTransformer) + throws ConfigurationException + { + super(builder.getName()); + + builder.assertReady(); + + if (exTransformer == null) { + throw new IllegalArgumentException("Exception transformer must not be null"); + } + + mIsMaster = builder.isMaster(); + mTriggerFactories = builder.getTriggerFactories(); + mRootRef = rootRef; + mExTransformer = exTransformer; + mTxnMgr = new BDBTransactionManager(mExTransformer, this); + + mRunCheckpointer = !builder.getReadOnly() && builder.getRunCheckpointer(); + mKeepOldLogFiles = builder.getKeepOldLogFiles(); + mLogInMemory = builder.getLogInMemory(); + mRunDeadlockDetector = builder.getRunDeadlockDetector(); + mStorableCodecFactory = builder.getStorableCodecFactory(); + mPreShutdownHook = builder.getPreShutdownHook(); + mPostShutdownHook = builder.getShutdownHook(); + mInitialDBConfig = builder.getInitialDatabaseConfig(); + mDatabaseHook = builder.getDatabaseHook(); + mDatabasePageSizes = builder.getDatabasePagesMap(); + mDataHome = builder.getDataHomeFile(); + mEnvHome = builder.getEnvironmentHomeFile(); + mSingleFileName = builder.getSingleFileName(); + mFileNameMap = builder.getFileNameMap(); + + getLog().info("Opening repository \"" + getName() + '"'); + } + + public ExceptionTransformer getExceptionTransformer() { + return mExTransformer; + } + + public IndexInfo[] getIndexInfo(Class storableType) + throws RepositoryException + { + return ((BDBStorage) storageFor(storableType)).getIndexInfo(); + } + + public String[] getUserStorableTypeNames() throws RepositoryException { + Repository metaRepo = getRootRepository(); + + Cursor cursor = + metaRepo.storageFor(StoredDatabaseInfo.class) + .query().orderBy("databaseName").fetch(); + + try { + ArrayList names = new ArrayList(); + while (cursor.hasNext()) { + StoredDatabaseInfo info = cursor.next(); + // Ordinary user types support evolution. + if (info.getEvolutionStrategy() != StoredDatabaseInfo.EVOLUTION_NONE) { + names.add(info.getDatabaseName()); + } + } + + return names.toArray(new String[names.size()]); + } finally { + cursor.close(); + } + } + + public boolean isSupported(Class type) { + if (type == null) { + return false; + } + StorableIntrospector.examine(type); + return true; + } + + public boolean isPropertySupported(Class type, String name) { + if (type == null || name == null) { + return false; + } + return StorableIntrospector.examine(type).getAllProperties().get(name) != null; + } + + @Override + public Backup startBackup() throws RepositoryException { + return startBackup(false); + } + + @Override + public Backup startBackup(boolean deleteOldLogFiles) throws RepositoryException { + if (mLogInMemory) { + throw new IllegalStateException + ("Log files are only kept in memory and backups cannot be performed"); + } + + synchronized (mBackupLock) { + int count = mBackupCount; + if (count == 0) { + try { + if (deleteOldLogFiles) { + // TODO: If backup rejects log deletion, queue up for later. + enterBackupMode(true); + } else { + // Call old API for backwards compatibility. + enterBackupMode(); + } + } catch (Exception e) { + throw mExTransformer.toRepositoryException(e); + } + } + mBackupCount = count + 1; + + return new FullBackup(); + } + } + + @Override + public Backup startIncrementalBackup(long lastLogNumber) + throws RepositoryException + { + return startIncrementalBackup(lastLogNumber, false); + } + + @Override + public Backup startIncrementalBackup(long lastLogNumber, boolean deleteOldLogFiles) + throws RepositoryException + { + if (mLogInMemory) { + throw new IllegalStateException + ("Log files are only kept in memory and incremental backup cannot be performed"); + } + + if (lastLogNumber < 0) { + throw new IllegalArgumentException + ("The number of the last backup cannot be negative: " + lastLogNumber); + } + synchronized (mBackupLock) { + try { + enterIncrementalBackupMode(lastLogNumber, deleteOldLogFiles); + ++mIncrementalBackupCount; + } catch (Exception e) { + throw mExTransformer.toRepositoryException(e); + } + } + return new IncrementalBackup(lastLogNumber); + } + + /** + * Suspend the checkpointer until the suspension time has expired or until + * manually resumed. If a checkpoint is in progress, this method will block + * until it is finished. If checkpointing is disabled, calling this method + * has no effect. + * + *

Calling this method repeatedly resets the suspension time. This + * technique should be used by hot backup processes to ensure that its + * failure does not leave the checkpointer permanently suspended. Each + * invocation of suspendCheckpointer is like a lease renewal or heartbeat. + * + * @param suspensionTime minimum length of suspension, in milliseconds, + * unless checkpointer is manually resumed + */ + public void suspendCheckpointer(long suspensionTime) { + if (mCheckpointer != null) { + mCheckpointer.suspendCheckpointer(suspensionTime); + } + } + + /** + * Resumes the checkpointer if it was suspended. If checkpointing is + * disabled or if not suspended, calling this method has no effect. + */ + public void resumeCheckpointer() { + if (mCheckpointer != null) { + mCheckpointer.resumeCheckpointer(); + } + } + + /** + * Forces a checkpoint to run now, even if checkpointer is suspended or + * disabled. If a checkpoint is in progress, then this method will block + * until it is finished, and then run another checkpoint. This method does + * not return until the requested checkpoint has finished. + */ + public void forceCheckpoint() throws PersistException { + if (mCheckpointer != null) { + mCheckpointer.forceCheckpoint(); + } else { + try { + env_checkpoint(); + } catch (Exception e) { + throw toPersistException(e); + } + } + } + + public void sync() throws PersistException { + try { + env_sync(); + } catch (Exception e) { + throw toPersistException(e); + } + } + + public Repository getRootRepository() { + return mRootRef.get(); + } + + public StorageAccess storageAccessFor(Class type) + throws RepositoryException + { + return (BDBStorage) storageFor(type); + } + + @Override + public Layout layoutFor(Class type) + throws FetchException, PersistException + { + try { + return ((BDBStorage) storageFor(type)).getLayout(true, mStorableCodecFactory); + } catch (PersistException e) { + throw e; + } catch (RepositoryException e) { + throw e.toFetchException(); + } + } + + @Override + public Layout layoutFor(Class type, int generation) + throws FetchException + { + return mLayoutFactory.layoutFor(type, generation); + } + + @Override + protected void finalize() { + close(); + } + + @Override + protected void shutdownHook() { + // Run any external shutdown logic that needs to happen before the + // databases and the environment are actually closed + if (mPreShutdownHook != null) { + mPreShutdownHook.run(); + } + + // Close database handles. + for (Storage storage : allStorage()) { + try { + if (storage instanceof BDBStorage) { + ((BDBStorage) storage).close(); + } + } catch (Throwable e) { + getLog().error(null, e); + } + } + + // Wait for checkpointer to finish. + if (mCheckpointer != null) { + mCheckpointer.interrupt(); + try { + mCheckpointer.join(); + } catch (InterruptedException e) { + } + } + + // Wait for deadlock detector to finish. + if (mDeadlockDetector != null) { + mDeadlockDetector.interrupt(); + try { + mDeadlockDetector.join(); + } catch (InterruptedException e) { + } + } + + // Close environment. + try { + env_close(); + } catch (Throwable e) { + getLog().error(null, e); + } + + if (mPostShutdownHook != null) { + mPostShutdownHook.run(); + } + } + + @Override + protected Log getLog() { + return mLog; + } + + @Override + protected Storage createStorage(Class type) + throws RepositoryException + { + try { + return createBDBStorage(type); + } catch (MalformedArgumentException e) { + throw e; + } catch (Exception e) { + throw toRepositoryException(e); + } + } + + @Override + protected SequenceValueProducer createSequenceValueProducer(String name) + throws RepositoryException + { + return new SequenceValueGenerator(BDBRepository.this, name); + } + + /** + * @see com.amazon.carbonado.spi.RepositoryBuilder#isMaster + */ + boolean isMaster() { + return mIsMaster; + } + + String[] getAllDatabaseNames() throws RepositoryException { + Repository metaRepo = getRootRepository(); + + Cursor cursor = + metaRepo.storageFor(StoredDatabaseInfo.class) + .query().orderBy("databaseName").fetch(); + + ArrayList names = new ArrayList(); + // This one needs to manually added since it is the metadata db itself. + names.add(StoredDatabaseInfo.class.getName()); + + try { + while (cursor.hasNext()) { + names.add(cursor.next().getDatabaseName()); + } + } finally { + cursor.close(); + } + + return names.toArray(new String[names.size()]); + } + + String getDatabaseFileName(final String dbName) { + String singleFileName = mSingleFileName; + if (singleFileName == null && mFileNameMap != null) { + singleFileName = mFileNameMap.get(dbName); + if (singleFileName == null && dbName != null) { + singleFileName = mFileNameMap.get(null); + } + } + + String dbFileName = dbName; + + if (singleFileName == null) { + if (mDatabaseHook != null) { + dbFileName = mDatabaseHook.databaseName(dbName); + } + } else { + dbFileName = singleFileName; + } + + if (mDataHome != null && !mDataHome.equals(mEnvHome)) { + dbFileName = new File(mDataHome, dbFileName).getPath(); + } + + return dbFileName; + } + + /** + * Returns null if name should not be used. + */ + String getDatabaseName(String dbName) { + if (mFileNameMap == null) { + return null; + } + String name = mFileNameMap.get(dbName); + if (name == null && dbName != null) { + name = mFileNameMap.get(null); + } + if (name == null) { + return null; + } + if (mDatabaseHook != null) { + try { + dbName = mDatabaseHook.databaseName(dbName); + } catch (IncompatibleClassChangeError e) { + // Method not implemented. + } + } + return dbName; + } + + StorableCodecFactory getStorableCodecFactory() { + return mStorableCodecFactory; + } + + LayoutFactory getLayoutFactory() throws RepositoryException { + if (mLayoutFactory == null) { + mLayoutFactory = new LayoutFactory(getRootRepository()); + } + return mLayoutFactory; + } + + LobEngine getLobEngine() throws RepositoryException { + if (mLobEngine == null) { + mLobEngine = new LobEngine(this, getRootRepository()); + } + return mLobEngine; + } + + /** + * Returns the optional BDB specific database configuration to use + * for all databases created. + */ + public Object getInitialDatabaseConfig() { + return mInitialDBConfig; + } + + /** + * Returns the desired page size for the given type, or null for default. + */ + Integer getDatabasePageSize(Class type) { + if (mDatabasePageSizes == null) { + return null; + } + Integer size = mDatabasePageSizes.get(type); + if (size == null && type != null) { + size = mDatabasePageSizes.get(null); + } + return size; + } + + void runDatabasePrepareForOpeningHook(Object database) throws RepositoryException { + if (mDatabaseHook != null) { + mDatabaseHook.prepareForOpening(database); + } + } + + /** + * Start background tasks and enable auto shutdown. + * + * @param checkpointInterval how often to run checkpoints, in milliseconds, + * or zero if never. Ignored if repository is read only or builder has + * checkpoints disabled. + * @param deadlockDetectorInterval how often to run deadlock detector, in + * milliseconds, or zero if never. Ignored if builder has deadlock detector + * disabled. + * + * @deprecated Overloaded for backwards compatiblity with older + * CarbonadoSleepycat packages + */ + void start(long checkpointInterval, long deadlockDetectorInterval) { + getLog().info("Opened repository \"" + getName() + '"'); + + if (mRunCheckpointer && checkpointInterval > 0) { + mCheckpointer = new Checkpointer(this, checkpointInterval, 1024, 5); + mCheckpointer.start(); + } else { + mCheckpointer = null; + } + + if (mRunDeadlockDetector && deadlockDetectorInterval > 0) { + mDeadlockDetector = new DeadlockDetector(this, deadlockDetectorInterval); + mDeadlockDetector.start(); + } else { + mDeadlockDetector = null; + } + + setAutoShutdownEnabled(true); + } + + /** + * Start background tasks and enable auto shutdown. + * + * @param checkpointInterval how often to run checkpoints, in milliseconds, + * or zero if never. Ignored if repository is read only or builder has + * checkpoints disabled. + * @param deadlockDetectorInterval how often to run deadlock detector, in + * milliseconds, or zero if never. Ignored if builder has deadlock detector + * disabled. + * @param builder containing additonal background task properties. + */ + void start(long checkpointInterval, long deadlockDetectorInterval, + BDBRepositoryBuilder builder) { + getLog().info("Opened repository \"" + getName() + '"'); + + if (mRunCheckpointer && checkpointInterval > 0) { + mCheckpointer = new Checkpointer(this, checkpointInterval, + builder.getCheckpointThresholdKB(), + builder.getCheckpointThresholdMinutes()); + mCheckpointer.start(); + } else { + mCheckpointer = null; + } + + if (mRunDeadlockDetector && deadlockDetectorInterval > 0) { + mDeadlockDetector = new DeadlockDetector(this, deadlockDetectorInterval); + mDeadlockDetector.start(); + } else { + mDeadlockDetector = null; + } + + setAutoShutdownEnabled(true); + } + + abstract boolean verify(PrintStream out) throws Exception; + + abstract IsolationLevel selectIsolationLevel(Transaction parent, IsolationLevel level); + + abstract Txn txn_begin(Txn parent, IsolationLevel level) throws Exception; + + // Subclass should override this method to actually apply the timeout + Txn txn_begin(Txn parent, IsolationLevel level, int timeout, TimeUnit unit) throws Exception { + return txn_begin(parent, level); + } + + abstract Txn txn_begin_nowait(Txn parent, IsolationLevel level) throws Exception; + + abstract void txn_commit(Txn txn) throws Exception; + + abstract void txn_abort(Txn txn) throws Exception; + + /** + * Force a checkpoint to run. + */ + abstract void env_checkpoint() throws Exception; + + /** + * Synchronously flush changes to stable storage. + */ + abstract void env_sync() throws Exception; + + /** + * @param kBytes run checkpoint if at least this many kilobytes in log + * @param minutes run checkpoint if at least this many minutes passed since + * last checkpoint + */ + abstract void env_checkpoint(int kBytes, int minutes) throws Exception; + + /** + * Run the deadlock detector. + */ + abstract void env_detectDeadlocks() throws Exception; + + /** + * Close the environment. + */ + abstract void env_close() throws Exception; + + abstract BDBStorage createBDBStorage(Class type) + throws Exception; + + /** + * Called only the first time a backup is started. Old API is kept for + * backwards compatibility. + */ + void enterBackupMode() throws Exception { + enterBackupMode(false); + } + + /** + * Called only the first time a backup is started. + */ + abstract void enterBackupMode(boolean deleteOldLogFiles) throws Exception; + + /** + * Called only after the last backup ends. + */ + abstract void exitBackupMode() throws Exception; + + /** + * Called only when an incremental backup is started. + */ + abstract void enterIncrementalBackupMode(long lastLogNumber, boolean deleteOldLogFiles) + throws Exception; + + /** + * Called only after incremental backup ends. + */ + abstract void exitIncrementalBackupMode() throws Exception; + + /** + * Called only if in backup mode. Old API is kept for backwards + * compatibility. + */ + @Deprecated + File[] backupFiles() throws Exception { + return backupFiles(new long[1]); + } + + @Deprecated + File[] backupFiles(long[] newLastLogNum) throws Exception { + throw new UnsupportedOperationException(); + } + + /** + * Called only if in backup mode. + */ + abstract File[] backupDataFiles() throws Exception; + + /** + * Called only if in backup mode. + * + * @param newLastLogNum reference to last log number at [0] + */ + abstract File[] backupLogFiles(long[] newLastLogNum) throws Exception; + + /** + * Called only if in incremental backup mode. + * + * @param newLastLogNum reference to last log number at [0] + */ + abstract File[] incrementalBackup(long lastLogNumber, long[] newLastLogNum) throws Exception; + + FetchException toFetchException(Throwable e) { + return mExTransformer.toFetchException(e); + } + + PersistException toPersistException(Throwable e) { + return mExTransformer.toPersistException(e); + } + + RepositoryException toRepositoryException(Throwable e) { + return mExTransformer.toRepositoryException(e); + } + + @Override + protected final TransactionManager transactionManager() { + return mTxnMgr; + } + + @Override + protected final TransactionScope localTransactionScope() { + return mTxnMgr.localScope(); + } + + /** + * Periodically runs checkpoints on the environment. + */ + private static class Checkpointer extends Thread { + private final WeakReference mRepository; + private final long mSleepInterval; + private final int mKBytes; + private final int mMinutes; + + private boolean mInProgress; + private long mSuspendUntil = Long.MIN_VALUE; + + /** + * + * @param repository outer class + * @param sleepInterval milliseconds to sleep before running checkpoint + * @param kBytes run checkpoint if at least this many kilobytes in log + * @param minutes run checkpoint if at least this many minutes passed + * since last checkpoint + */ + Checkpointer(BDBRepository repository, long sleepInterval, int kBytes, int minutes) { + super(repository.getClass().getSimpleName() + " checkpointer (" + + repository.getName() + ')'); + setDaemon(true); + mRepository = new WeakReference(repository); + mSleepInterval = sleepInterval; + mKBytes = kBytes; + mMinutes = minutes; + } + + @Override + public void run() { + try { + while (true) { + synchronized (this) { + if (!mInProgress) { + try { + wait(mSleepInterval); + } catch (InterruptedException e) { + break; + } + } + } + + BDBRepository repository = mRepository.get(); + if (repository == null) { + break; + } + + long suspendUntil; + synchronized (this) { + suspendUntil = mSuspendUntil; + } + if (suspendUntil != Long.MIN_VALUE) { + if (System.currentTimeMillis() < suspendUntil) { + continue; + } + } + + Log log = repository.getLog(); + + if (log.isDebugEnabled()) { + log.debug("Running checkpoint on repository \"" + + repository.getName() + '"'); + } + + try { + synchronized (this) { + mInProgress = true; + } + repository.env_checkpoint(mKBytes, mMinutes); + if (log.isDebugEnabled()) { + log.debug("Finished running checkpoint on repository \"" + + repository.getName() + '"'); + } + } catch (ThreadDeath e) { + break; + } catch (Throwable e) { + log.error("Checkpoint failed", e); + } finally { + synchronized (this) { + mInProgress = false; + // Only wait condition is mInProgress, so okay to not call notifyAll. + notify(); + } + repository = null; + } + } + } finally { + synchronized (this) { + mInProgress = false; + // Only wait condition is mInProgress, so okay to not call notifyAll. + notify(); + } + } + } + + /** + * Blocks until checkpoint has finished. + */ + synchronized void suspendCheckpointer(long suspensionTime) { + while (mInProgress) { + try { + wait(); + } catch (InterruptedException e) { + } + } + + if (suspensionTime <= 0) { + return; + } + + long now = System.currentTimeMillis(); + long suspendUntil = now + suspensionTime; + if (now >= 0 && suspendUntil < 0) { + // Overflow. + suspendUntil = Long.MAX_VALUE; + } + mSuspendUntil = suspendUntil; + } + + synchronized void resumeCheckpointer() { + mSuspendUntil = Long.MIN_VALUE; + } + + /** + * Blocks until checkpoint has finished. + */ + synchronized void forceCheckpoint() throws PersistException { + while (mInProgress) { + try { + wait(); + } catch (InterruptedException e) { + return; + } + } + + BDBRepository repository = mRepository.get(); + if (repository != null) { + try { + repository.env_checkpoint(); + } catch (Exception e) { + throw repository.toPersistException(e); + } + } + } + } + + /** + * Periodically runs deadlock detection on the environment. + */ + private static class DeadlockDetector extends Thread { + private final WeakReference mRepository; + private final long mSleepInterval; + + /** + * @param repository outer class + * @param sleepInterval milliseconds to sleep before running deadlock detection + */ + DeadlockDetector(BDBRepository repository, long sleepInterval) { + super(repository.getClass().getSimpleName() + " deadlock detector (" + + repository.getName() + ')'); + setDaemon(true); + mRepository = new WeakReference(repository); + mSleepInterval = sleepInterval; + } + + @Override + public void run() { + while (true) { + try { + Thread.sleep(mSleepInterval); + } catch (InterruptedException e) { + break; + } + + BDBRepository repository = mRepository.get(); + if (repository == null) { + break; + } + + try { + repository.env_detectDeadlocks(); + } catch (ThreadDeath e) { + break; + } catch (Throwable e) { + repository.getLog().error("Deadlock detection failed", e); + } finally { + repository = null; + } + } + } + } + + abstract class AbstractBackup implements Backup { + boolean mDone; + long mFinalLogNumber; + + AbstractBackup() { + mFinalLogNumber = -1; + } + + @Override + public void endBackup() throws RepositoryException { + synchronized (mBackupLock) { + if (mDone) { + return; + } + mDone = true; + finishBackup(); + } + } + + @Override + @Deprecated + public File[] getFiles() throws RepositoryException { + synchronized (mBackupLock) { + File[] data = getDataFiles(); + File[] logs = getLogFiles(); + File[] all = new File[data.length + logs.length]; + System.arraycopy(data, 0, all, 0, data.length); + System.arraycopy(logs, 0, all, data.length, logs.length); + return all; + } + } + + @Override + public File[] getDataFiles() throws RepositoryException { + synchronized (mBackupLock) { + if (mDone) { + throw new IllegalStateException("Backup has ended"); + } + + try { + return getDataBackupFiles(); + } catch (Exception e) { + throw mExTransformer.toRepositoryException(e); + } + } + } + + @Override + public File[] getLogFiles() throws RepositoryException { + synchronized (mBackupLock) { + if (mDone) { + throw new IllegalStateException("Backup has ended"); + } + + try { + long[] newLastLogNum = {-1}; + File[] toReturn = getLogBackupFiles(newLastLogNum); + mFinalLogNumber = newLastLogNum[0]; + return toReturn; + } catch (Exception e) { + throw mExTransformer.toRepositoryException(e); + } + } + } + + @Override + public long getLastLogNumber() throws RepositoryException { + if (mFinalLogNumber < 0) { + throw new IllegalStateException + ("Must get files prior to retrieving the last log number"); + } + return mFinalLogNumber; + } + + abstract void finishBackup() throws RepositoryException; + + abstract File[] getDataBackupFiles() throws Exception; + + abstract File[] getLogBackupFiles(long[] newLastLogNum) throws Exception; + } + + class IncrementalBackup extends AbstractBackup { + private final long mLastLogNumber; + + IncrementalBackup(long lastLogNumber) { + super(); + mLastLogNumber = lastLogNumber; + } + + @Override + void finishBackup() throws RepositoryException { + --mIncrementalBackupCount; + + try { + exitIncrementalBackupMode(); + } catch (Exception e) { + throw mExTransformer.toRepositoryException(e); + } + } + + @Override + File[] getDataBackupFiles() throws Exception { + return new File[0]; + } + + @Override + File[] getLogBackupFiles(long[] newLastLogNum) throws Exception { + return incrementalBackup(mLastLogNumber, newLastLogNum); + } + } + + class FullBackup extends AbstractBackup { + @Override + void finishBackup() throws RepositoryException { + int count = mBackupCount - 1; + try { + if (count == 0) { + try { + exitBackupMode(); + } catch (Exception e) { + throw mExTransformer.toRepositoryException(e); + } + } + } finally { + mBackupCount = count; + } + } + + @Override + File[] getDataBackupFiles() throws Exception { + try { + return backupDataFiles(); + } catch (AbstractMethodError e) { + // Old API will be called for backwards compatibility in the + // getLogBackupFiles method. + return new File[0]; + } + } + + @Override + File[] getLogBackupFiles(long[] newLastLogNum) throws Exception { + try { + return backupLogFiles(newLastLogNum); + } catch (AbstractMethodError e) { + // Call old API for backwards compatibility. + try { + return backupFiles(newLastLogNum); + } catch (AbstractMethodError e2) { + // Call even older API for backwards compatibility. + return backupFiles(); + } + } + } + } +} + diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java index 82f51d9..e0cd87a 100644 --- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java +++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepositoryBuilder.java @@ -1,1142 +1,1163 @@ -/* - * Copyright 2006-2012 Amazon Technologies, Inc. or its affiliates. - * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks - * of Amazon Technologies, Inc. or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.amazon.carbonado.repo.sleepycat; - -import java.lang.reflect.Constructor; - -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -import java.util.concurrent.atomic.AtomicReference; - -import org.cojen.util.ThrowUnchecked; - -import com.amazon.carbonado.Repository; -import com.amazon.carbonado.RepositoryException; -import com.amazon.carbonado.Storable; -import com.amazon.carbonado.repo.indexed.IndexedRepositoryBuilder; - -import com.amazon.carbonado.raw.CompressionType; -import com.amazon.carbonado.raw.CompressedStorableCodecFactory; -import com.amazon.carbonado.raw.StorableCodecFactory; - -import com.amazon.carbonado.spi.AbstractRepositoryBuilder; - -import com.amazon.carbonado.ConfigurationException; - -/** - * Builder and configuration options for BDBRepository. - * - * 

- * BDBRepositoryBuilder builder = new BDBRepositoryBuilder();
- *
- * builder.setProduct("JE");
- * builder.setName("test");
- * builder.setEnvironmentHome("/tmp/testRepo");
- * builder.setTransactionWriteNoSync(true);
- *
- * Repository repo = builder.build();
- *
- * - *

- * The following extra capabilities are supported: - *

    - *
  • {@link com.amazon.carbonado.capability.IndexInfoCapability IndexInfoCapability} - *
  • {@link com.amazon.carbonado.capability.StorableInfoCapability StorableInfoCapability} - *
  • {@link com.amazon.carbonado.capability.ShutdownCapability ShutdownCapability} - *
  • {@link com.amazon.carbonado.layout.LayoutCapability LayoutCapability} - *
  • {@link com.amazon.carbonado.sequence.SequenceCapability SequenceCapability} - *
  • {@link CheckpointCapability CheckpointCapability} - *
  • {@link EnvironmentCapability EnvironmentCapability} - *
- * - * @author Brian S O'Neill - * @author Vidya Iyer - * @author Nicole Deflaux - */ -public final class BDBRepositoryBuilder extends AbstractRepositoryBuilder { - - private static final BDBProduct DEFAULT_PRODUCT = BDBProduct.JE; - - private static final int DEFAULT_CHECKPOINT_INTERVAL = 10000; - - private String mName; - private boolean mIsMaster = true; - private BDBProduct mProduct = DEFAULT_PRODUCT; - private File mEnvHome; - private File mDataHome; - private String mSingleFileName; - private Map mFileNames; - private boolean mIndexSupport = true; - private boolean mIndexRepairEnabled = true; - private double mIndexThrottle = 1.0; - private boolean mReadOnly; - private Long mCacheSize; - private Integer mCachePercent; - private Integer mLogRegionSize; - private double mLockTimeout = 0.5; - private Integer mMaxLocks; - private double mTxnTimeout = 300.0; - private boolean mTxnNoSync; - private boolean mTxnWriteNoSync; - private Integer mTxnMaxActive = 1000; - private Boolean mDatabasesTransactional = null; - private boolean mReverseSplitOff; - private Map, Integer> mDatabasePageSizes; - private boolean mPrivate; - private boolean mMultiversion; - private boolean mLogInMemory; - private Integer mLogFileMaxSize; - private boolean mInitializeLogging; - private boolean mRunFullRecovery; - private boolean mRunCheckpointer = true; - private int mCheckpointInterval = DEFAULT_CHECKPOINT_INTERVAL; - private int mCheckpointThresholdKB = 1024; - private int mCheckpointThresholdMinutes = 1; - private boolean mKeepOldLogFiles; - private boolean mRunDeadlockDetector = true; - private Boolean mChecksumEnabled; - private Object mInitialEnvConfig = null; - private Object mInitialDBConfig = null; - private StorableCodecFactory mStorableCodecFactory; - private Runnable mPreShutdownHook; - private Runnable mPostShutdownHook; - private DatabaseHook mDatabaseHook; - private Map mCompressionMap; - - public BDBRepositoryBuilder() { - } - - public Repository build(AtomicReference rootRef) throws RepositoryException { - if (mIndexSupport) { - // Wrap BDBRepository with IndexedRepository. - - // Temporarily set to false to avoid infinite recursion. - mIndexSupport = false; - try { - IndexedRepositoryBuilder ixBuilder = new IndexedRepositoryBuilder(); - ixBuilder.setWrappedRepository(this); - ixBuilder.setMaster(isMaster()); - ixBuilder.setIndexRepairEnabled(mIndexRepairEnabled); - ixBuilder.setIndexRepairThrottle(mIndexThrottle); - return ixBuilder.build(rootRef); - } finally { - mIndexSupport = true; - } - } - - if (mStorableCodecFactory == null) { - mStorableCodecFactory = new CompressedStorableCodecFactory(mCompressionMap); - } - - assertReady(); - - // Make environment directory if it doesn't exist. - File homeFile = getEnvironmentHomeFile(); - if (!homeFile.exists()) { - if (!homeFile.mkdirs()) { - throw new RepositoryException - ("Unable to make environment home directory: " + homeFile); - } - } - - BDBRepository repo; - - try { - repo = getRepositoryConstructor().newInstance(rootRef, this); - } catch (Exception e) { - ThrowUnchecked.fireFirstDeclaredCause(e, RepositoryException.class); - // Not reached. - return null; - } - - rootRef.set(repo); - return repo; - } - - /** - * Opens the BDB environment, checks if it is corrupt, and then closes it. - * Only one process should open the environment for verification. Expect it - * to take a long time. - * - * @param out optional stream to capture any verfication errors - * @return true if environment passes verification - */ - public boolean verify(PrintStream out) throws RepositoryException { - final StorableCodecFactory codecFactory = mStorableCodecFactory; - final String name = mName; - final boolean readOnly = mReadOnly; - final boolean runCheckpointer = mRunCheckpointer; - final boolean runDeadlockDetector = mRunDeadlockDetector; - final boolean isPrivate = mPrivate; - - if (mName == null) { - // Allow a dummy name for verification. - mName = "BDB verification"; - } - - if (mStorableCodecFactory == null) { - mStorableCodecFactory = new CompressedStorableCodecFactory(mCompressionMap); - } - - mReadOnly = true; - mRunCheckpointer = false; - mRunDeadlockDetector = false; - - try { - assertReady(); - - File homeFile = getEnvironmentHomeFile(); - if (!homeFile.exists()) { - throw new RepositoryException - ("Environment home directory does not exist: " + homeFile); - } - - AtomicReference rootRef = new AtomicReference(); - BDBRepository repo; - - try { - repo = getRepositoryConstructor().newInstance(rootRef, this); - } catch (Exception e) { - ThrowUnchecked.fireFirstDeclaredCause(e, RepositoryException.class); - // Not reached. - return false; - } - - rootRef.set(repo); - - try { - return repo.verify(out); - } catch (Exception e) { - throw repo.toRepositoryException(e); - } finally { - repo.close(); - } - } finally { - mName = name; - mStorableCodecFactory = codecFactory; - mReadOnly = readOnly; - mRunCheckpointer = runCheckpointer; - mRunDeadlockDetector = runDeadlockDetector; - } - } - - public String getName() { - return mName; - } - - public void setName(String name) { - mName = name; - } - - public boolean isMaster() { - return mIsMaster; - } - - public void setMaster(boolean b) { - mIsMaster = b; - } - - /** - * Sets the BDB product to use, which defaults to JE. Also supported is DB - * and DB_HA. If not supported, an IllegalArgumentException is thrown. - */ - public void setProduct(String product) { - mProduct = product == null ? DEFAULT_PRODUCT : BDBProduct.forString(product); - } - - /** - * Returns the BDB product to use, which is JE by default. - */ - public String getProduct() { - return mProduct.toString(); - } - - /** - * Sets the BDB product to use, which defaults to JE. - */ - public void setBDBProduct(BDBProduct product) { - mProduct = product == null ? DEFAULT_PRODUCT : product; - } - - /** - * Returns the BDB product to use, which is JE by default. - */ - public BDBProduct getBDBProduct() { - return mProduct; - } - - /** - * Sets the repository environment home directory, which is required. - */ - public void setEnvironmentHomeFile(File envHome) { - try { - // Switch to canonical for more detailed error messages. - envHome = envHome.getCanonicalFile(); - } catch (IOException e) { - } - mEnvHome = envHome; - } - - /** - * Returns the repository environment home directory. - */ - public File getEnvironmentHomeFile() { - return mEnvHome; - } - - /** - * Sets the repository environment home directory, which is required. - * - * @throws RepositoryException if environment home is not valid - */ - public void setEnvironmentHome(String envHome) { - setEnvironmentHomeFile(new File(envHome)); - } - - /** - * Returns the repository environment home directory. - */ - public String getEnvironmentHome() { - return mEnvHome.getPath(); - } - - /** - * By default, data files are stored relative to the environment home. Call - * this method to override. For BDBRepositories that are log files only, - * this configuration is ignored. - */ - public void setDataHomeFile(File dir) { - if (dir != null) { - try { - // Switch to canonical for more detailed error messages. - dir = dir.getCanonicalFile(); - } catch (IOException e) { - } - } - mDataHome = dir; - } - - /** - * Returns the optional directory to store data files. Returns null if data - * files are expected to be relative to the environment home. - */ - public File getDataHomeFile() { - if (mDataHome == null) { - return getEnvironmentHomeFile(); - } - return mDataHome; - } - - /** - * By default, data files are stored relative to the environment home. Call - * this method to override. For BDBRepositories that are log files only, - * this configuration is ignored. - */ - public void setDataHome(String dir) { - if (dir == null) { - mDataHome = null; - } else { - setDataHomeFile(new File(dir)); - } - } - - /** - * Returns the directory to store data files. - */ - public String getDataHome() { - return getDataHomeFile().getPath(); - } - - /** - * Specify that all BDB databases should reside in one file, except for log - * files and caches. The filename is relative to the environment home, - * unless data directories have been specified. For BDBRepositories that - * are log files only, this configuration is ignored. - * - *

Note: When setting this option, the storable codec factory must also - * be changed, since the default storable codec factory is unable to - * distinguish storable types that reside in a single database file. Call - * setFileName instead to use built-in BDB feature for supporting multiple - * databases in one file. - */ - public void setSingleFileName(String filename) { - mSingleFileName = filename; - mFileNames = null; - } - - /** - * Returns the single file that all BDB databases should reside in. - */ - public String getSingleFileName() { - return mSingleFileName; - } - - /** - * Specify the file that a BDB database should reside in, except for log - * files and caches. The filename is relative to the environment home, - * unless data directories have been specified. For BDBRepositories that - * are log files only, this configuration is ignored. - * - * @param filename BDB database filename - * @param typeName type to store in file; if null, the file is used by default - * for all types - */ - public void setFileName(String filename, String typeName) { - mSingleFileName = null; - if (mFileNames == null) { - mFileNames = new HashMap(); - } - mFileNames.put(typeName, filename); - } - - Map getFileNameMap() { - if (mFileNames == null) { - return null; - } - return new HashMap(mFileNames); - } - - /** - * By default, user specified indexes are supported. Pass false to disable - * this, and no indexes will be built. Another consequence of this option - * is that no unique constraint checks will be applied to alternate keys. - */ - public void setIndexSupport(boolean indexSupport) { - mIndexSupport = indexSupport; - } - - /** - * Returns true if indexes are supported, which is true by default. - */ - public boolean getIndexSupport() { - return mIndexSupport; - } - - /** - * @see #setIndexRepairEnabled(boolean) - * - * @return true by default - */ - public boolean isIndexRepairEnabled() { - return mIndexRepairEnabled; - } - - /** - * By default, index repair is enabled. In this mode, the first time a - * Storable type is used, new indexes are populated and old indexes are - * removed. Until finished, access to the Storable is blocked. - * - *

When index repair is disabled, the Storable is immediately - * available. This does have consequences, however. The set of indexes - * available for queries is defined by the intersection of the old - * and new index sets. The set of indexes that are kept up-to-date is - * defined by the union of the old and new index sets. - * - *

While index repair is disabled, another process can safely repair the - * indexes in the background. When it is complete, index repair can be - * enabled for this repository too. - */ - public void setIndexRepairEnabled(boolean enabled) { - mIndexRepairEnabled = enabled; - } - - /** - * Returns the throttle parameter used when indexes are added, dropped or - * bulk repaired. By default this value is 1.0, or maximum speed. - */ - public double getIndexRepairThrottle() { - return mIndexThrottle; - } - - /** - * Sets the throttle parameter used when indexes are added, dropped or bulk - * repaired. By default this value is 1.0, or maximum speed. - * - * @param desiredSpeed 1.0 = perform work at full speed, - * 0.5 = perform work at half speed, 0.0 = fully suspend work - */ - public void setIndexRepairThrottle(double desiredSpeed) { - mIndexThrottle = desiredSpeed; - } - - /** - * Sets the repository to read-only mode. By default, repository is opened - * for reads and writes. - */ - public void setReadOnly(boolean readOnly) { - mReadOnly = readOnly; - } - - /** - * Returns true if repository should be opened read-only. - */ - public boolean getReadOnly() { - return mReadOnly; - } - - /** - * Set the repository cache size, in bytes. Actual BDB implementation will - * select a suitable default if this is not set. - */ - public void setCacheSize(long cacheSize) { - mCacheSize = cacheSize; - } - - /** - * Set the repository cache size, in bytes. Actual BDB implementation will - * select a suitable default if this is not set. - * - * @param cacheSize cache size to use, or null for default - */ - public void setCacheSize(Long cacheSize) { - mCacheSize = cacheSize; - } - - /** - * Returns the repository cache size, or null if default should be - * selected. - */ - public Long getCacheSize() { - return mCacheSize; - } - - /** - * Set the repository log region size, in bytes. - */ - public void setLogRegionSize(int logRegionSize) { - mLogRegionSize = logRegionSize; - } - - /** - * Set the repository log region size, in bytes. - */ - public void setLogRegionSize(Integer logRegionSize) { - mLogRegionSize = logRegionSize; - } - - /** - * Returns the repository log region size, or null if the default - * should be selected. - */ - public Integer getLogRegionSize() { - return mLogRegionSize; - } - - /** - * Set the percent of JVM heap used by the repository cache. Actual - * BDB implementation will select a suitable default if this is not - * set. This is overridden by setting an explicit cacheSize. - */ - public void setCachePercent(int cachePercent) { - mCachePercent = cachePercent; - } - - /** - * Set the percent of JVM heap used by the repository cache. Actual - * BDB implementation will select a suitable default if this is not - * set. This is overridden by setting an explicit cacheSize. - * - * @param cachePercent percent of JVM heap to use, or null for default - */ - public void setCachePercent(Integer cachePercent) { - mCachePercent = cachePercent; - } - - /** - * Returns the percent of JVM heap used by the repository cache, or - * null if default should be selected. - */ - public Integer getCachePercent() { - return mCachePercent; - } - - /** - * Set the lock timeout, in seconds. Default value is 0.5 seconds. - */ - public void setLockTimeout(double lockTimeout) { - mLockTimeout = lockTimeout; - } - - /** - * Returns the lock timeout, in seconds. - */ - public double getLockTimeout() { - return mLockTimeout; - } - - /** - * Returns the lock timeout, in microseconds, limited to max long value. - */ - public long getLockTimeoutInMicroseconds() { - return inMicros(mLockTimeout); - } - - public void setMaxLocks(Integer max) { - mMaxLocks = max; - } - - public Integer getMaxLocks() { - return mMaxLocks; - } - - /** - * Set the transaction timeout, in seconds. Default value is 300 seconds. - */ - public void setTransactionTimeout(double txnTimeout) { - mTxnTimeout = txnTimeout; - } - - /** - * Returns the repository transaction timeout, in seconds. - */ - public double getTransactionTimeout() { - return mTxnTimeout; - } - - /** - * Returns the repository transaction timeout, in microseconds, limited to - * max long value. - */ - public long getTransactionTimeoutInMicroseconds() { - return inMicros(mTxnTimeout); - } - - /** - * When true, commits are not immediately written or flushed to disk. This - * improves performance, but there is a chance of losing the most recent - * commits if the process is killed or if the machine crashes. - */ - public void setTransactionNoSync(boolean noSync) { - mTxnNoSync = noSync; - } - - /** - * Returns true if transactions are not written or flushed to disk. - */ - public boolean getTransactionNoSync() { - return mTxnNoSync; - } - - /** - * When true, commits are written, but they are not flushed to disk. This - * improves performance, but there is a chance of losing the most recent - * commits if the machine crashes. - */ - public void setTransactionWriteNoSync(boolean noSync) { - mTxnWriteNoSync = noSync; - } - - /** - * Returns true if transactions are not flushed to disk. - */ - public boolean getTransactionWriteNoSync() { - return mTxnWriteNoSync; - } - - /** - * Set the maximum number of concurrent transactions, or pass null to use - * the default. This setting has no effect for BDB-JE. - */ - public void setTransactionMaxActive(Integer max) { - mTxnMaxActive = max; - } - - /** - * Returns the maximum number of concurrent transactions, or null if the - * default is used. - */ - public Integer getTransactionMaxActive() { - return mTxnMaxActive; - } - - /** - * When true, allows databases to be transactional. This setting affects - * the databases, not the environment. If this is not explicitly set, the - * environment getTransactional is used. - */ - public void setDatabasesTransactional(Boolean transactional) { - mDatabasesTransactional = transactional; - } - - /** - * Returns true if the databases are configured to be transactional, - * false if configured to not be transactional, null if this override was never set - */ - public Boolean getDatabasesTransactional() { - return mDatabasesTransactional; - } - - /** - * Pass true to disable reverse split of B-tree nodes to reduce deadlocks. - * This setting has no effect for BDB-JE. - */ - public void setReverseSplitOff(boolean off) { - mReverseSplitOff = off; - } - - public boolean isReverseSplitOff() { - return mReverseSplitOff; - } - - /** - * Sets the desired page size for a given type. If not specified, the page - * size applies to all types. - */ - public void setDatabasePageSize(Integer bytes, Class type) { - if (mDatabasePageSizes == null) { - mDatabasePageSizes = new HashMap, Integer>(); - } - mDatabasePageSizes.put(type, bytes); - } - - Map, Integer> getDatabasePagesMap() { - if (mDatabasePageSizes == null) { - return null; - } - return new HashMap, Integer>(mDatabasePageSizes); - } - - /** - * When true, BDB environment cannot be shared by other processes, and - * region files are not created. By default, environment is shared, if - * supported. - */ - public void setPrivate(boolean b) { - mPrivate = b; - } - - /** - * Returns true if BDB environment is private. By default, environment is - * shared, if supported. - */ - public boolean isPrivate() { - return mPrivate; - } - - /** - * Set true to enable multiversion concurrency control (MVCC) on BDB - * environment. This enables snapshot isolation, and is it is not supported - * by all BDB products and versions. - */ - public void setMultiversion(boolean multiversion) { - mMultiversion = multiversion; - } - - /** - * Returns false by default because multiversion concurrency control (MVCC) - * is not enabled. - */ - public boolean isMultiversion() { - return mMultiversion; - } - - /** - * Set true to store transaction logs in memory only instead of persistent - * storage. For BDB products which are entirely log based, no records are - * ever persisted. - */ - public void setLogInMemory(boolean logInMemory) { - mLogInMemory = logInMemory; - } - - /** - * Returns false by default, indicating that transaction logs are persisted. - */ - public boolean getLogInMemory() { - return mLogInMemory; - } - - /** - * Set the maximum transaction log file size for the BDB environment. - */ - public void setLogFileMaxSize(Integer sizeInBytes) { - mLogFileMaxSize = sizeInBytes; - } - - /** - * Returns null if default size will be used. - */ - public Integer getLogFileMaxSize() { - return mLogFileMaxSize; - } - - /** - * Ensure the transaction logging sub-system is initialized, which is - * usually implied. - */ - public void setInitializeLogging(boolean b) { - mInitializeLogging = b; - } - - public boolean getInitializeLogging() { - return mInitializeLogging; - } - - /** - * Pass true to override the default and run a full (catastrophic) recovery - * when environment is opened. This setting has no effect for BDB-JE. - */ - public void setRunFullRecovery(boolean runRecovery) { - mRunFullRecovery = runRecovery; - } - - /** - * Returns true if a full (catastrophic) recovery should be performed when - * environment is opened. - */ - public boolean getRunFullRecovery() { - return mRunFullRecovery; - } - - /** - * Disable automatic checkpointing of database if another process is - * responsible for that. The false setting is implied for read-only - * databases. - */ - public void setRunCheckpointer(boolean runCheckpointer) { - mRunCheckpointer = runCheckpointer; - } - - /** - * Returns true if checkpointer is run automatically. - */ - public boolean getRunCheckpointer() { - return mRunCheckpointer; - } - - /** - * Set the interval to run checkpoints. This setting is ignored if the - * checkpointer is not configured to run. - * - * @param intervalMillis interval between checkpoints, in milliseconds - */ - public void setCheckpointInterval(int intervalMillis) { - mCheckpointInterval = intervalMillis; - } - - /** - * @return interval between checkpoints, in milliseconds - */ - public int getCheckpointInterval() { - return mCheckpointInterval; - } - - /** - * Set the size threshold to run checkpoints. This setting is ignored if - * the checkpointer is not configured to run. Default value is 1024 KB. - * - *

Checkpoint threshold is only used by Carbonado's built-in - * checkpointer, and is ignored when using BDB-JE. - * - * @param thresholdKB run checkpoint if at least this many kilobytes in log - */ - public void setCheckpointThresholdKB(int thresholdKB) { - mCheckpointThresholdKB = thresholdKB; - } - - /** - * @return run checkpoint if at least this many kilobytes in log - */ - public int getCheckpointThresholdKB() { - return mCheckpointThresholdKB; - } - - /** - * Set the time threshold to run checkpoints. This setting is ignored if - * the checkpointer is not configured to run. Default value is 1 minute. - * - *

Checkpoint threshold is only used by Carbonado's built-in - * checkpointer, and is ignored when using BDB-JE. - * - * @param thresholdMinutes run checkpoint if at least this many minutes - * passed since last checkpoint - */ - public void setCheckpointThresholdMinutes(int thresholdMinutes) { - mCheckpointThresholdMinutes = thresholdMinutes; - } - - /** - * @return run checkpoint if at least this many minutes passed since last - * checkpoint - */ - public int getCheckpointThresholdMinutes() { - return mCheckpointThresholdMinutes; - } - - /** - * By default, transaction log files are deleted when no longer needed. - * Keeping log files can be used for incremental backups or for diagnosing - * problems. If using BDB-JE, old log files are renamed with a ".del" - * extension. If using BDB-core, the db_archive utility is required for - * identifying old log files. - */ - public void setKeepOldLogFiles(boolean keep) { - mKeepOldLogFiles = keep; - } - - /** - * Returns false by default. - */ - public boolean getKeepOldLogFiles() { - return mKeepOldLogFiles; - } - - /** - * Disable automatic deadlock detection of database if another thread is - * responsible for that. - */ - public void setRunDeadlockDetector(boolean runDeadlockDetector) { - mRunDeadlockDetector = runDeadlockDetector; - } - - /** - * Returns true if deadlock detector is configured to run. - */ - public boolean getRunDeadlockDetector() { - return mRunDeadlockDetector; - } - - /** - * When true, enable checksum verification of pages read into the cache - * from the backing filestore. By default checksum is enabled for BDB-JE, - * and disabled for BDB-C. - */ - public void setChecksumEnabled(Boolean checksumEnabled) { - mChecksumEnabled = checksumEnabled; - } - - /** - * Returns true if checksum verification is enabled. Returns null if the - * BDB default is used. - */ - public Boolean getChecksumEnabled() { - return mChecksumEnabled; - } - - /** - * Optionally set the BDB specific environment configuration to - * use. The builder will verify that needed configuration values are set. - */ - public void setInitialEnvironmentConfig(Object envConfig) { - mInitialEnvConfig = envConfig; - } - - /** - * Returns the optional BDB specific environment configuration to use. - */ - public Object getInitialEnvironmentConfig() { - return mInitialEnvConfig; - } - - /** - * Optionally set the BDB specific database configuration to use - * for all databases created. The storage will verify that needed - * configuration values are set. - */ - public void setInitialDatabaseConfig(Object dbConfig) { - mInitialDBConfig = dbConfig; - } - - /** - * Returns the optional BDB specific database configuration to use - * for all databases created. - */ - public Object getInitialDatabaseConfig() { - return mInitialDBConfig; - } - - /** - * Override the default storable codec factory. - */ - public void setStorableCodecFactory(StorableCodecFactory factory) { - mStorableCodecFactory = factory; - } - - /** - * Returns the storable codec factory used. - */ - public StorableCodecFactory getStorableCodecFactory() { - return mStorableCodecFactory; - } - - /** - * Sets a callback to be invoked before the repository has finished running - * its own shutdown hooks. This method is also invoked when repository is - * manually closed. - */ - public void setPreShutdownHook(Runnable hook) { - mPreShutdownHook = hook; - } - - /** - * Returns the custom shutdown hook that runs before the repository has - * finished running its own shutdown hooks, or null if none. - */ - public Runnable getPreShutdownHook() { - return mPreShutdownHook; - } - - /** - * Sets a callback to be invoked after repository has finished running its - * own shutdown hooks. This method is also invoked when repository is - * manually closed. - */ - public void setShutdownHook(Runnable hook) { - mPostShutdownHook = hook; - } - - /** - * Returns the custom shutdown hook that runs after the repository has - * finished running its own shutdown hooks, or null if none. - */ - public Runnable getShutdownHook() { - return mPostShutdownHook; - } - - /** - * Sets a hook to be called whenever a database is opened. - */ - public void setDatabaseHook(DatabaseHook hook) { - mDatabaseHook = hook; - } - - /** - * Returns the custom open database hook, or null if none. - */ - public DatabaseHook getDatabaseHook() { - return mDatabaseHook; - } - - /** - * Set the compressor for the given class, overriding a custom StorableCodecFactory. - - * @param type Storable to compress. - * @param compressionType String representation of type of - * compression. Available options are "NONE" for no compression or "GZIP" - * for gzip compression - */ - public void setCompressor(String type, String compressionType) { - mStorableCodecFactory = null; - compressionType = compressionType.toUpperCase(); - if (mCompressionMap == null) { - mCompressionMap = new HashMap(); - } - CompressionType compressionEnum = CompressionType.valueOf(compressionType); - if (compressionEnum != null) { - mCompressionMap.put(type, compressionEnum); - } - } - - /** - * Return the compressor used for the given storable. - * @param type Storable to compress - * @return String representation of the type of compression used. Available options are "NONE" - * for no compression and "GZIP" for gzip compression. - */ - public String getCompressor(String type) { - if (mCompressionMap == null) { - return null; - } - - return mCompressionMap.get(type).toString(); - } - - private long inMicros(double seconds) { - if (seconds >= Long.MAX_VALUE) { - return Long.MAX_VALUE; - } - if (seconds <= 0 || Double.isNaN(seconds)) { - return 0L; - } - return (long) (seconds * 1000000); - } - - @Override - public void errorCheck(Collection messages) throws ConfigurationException { - super.errorCheck(messages); - - checkClass: { - Exception error; - try { - getRepositoryConstructor(); - break checkClass; - } catch (ClassCastException e) { - error = e; - } catch (ClassNotFoundException e) { - error = e; - } catch (NoSuchMethodException e) { - error = e; - } - messages.add("BDB product \"" + getProduct() + "\" not supported: " + error); - } - - File envHome = getEnvironmentHomeFile(); - if (envHome == null) { - messages.add("environmentHome missing"); - } else { - if (envHome.exists() && !envHome.isDirectory()) { - messages.add("environment home is not a directory: " + envHome); - } - } - } - - /** - * Looks up appropriate repository via reflection, whose name is derived - * from the BDB product string. - */ - @SuppressWarnings("unchecked") - private Constructor getRepositoryConstructor() - throws ClassCastException, ClassNotFoundException, NoSuchMethodException - { - String packageName; - { - String thisClassName = getClass().getName(); - packageName = thisClassName.substring(0, thisClassName.lastIndexOf('.')); - } - String className = packageName + '.' + getBDBProduct().name() + "_Repository"; - Class repoClass = Class.forName(className); - if (BDBRepository.class.isAssignableFrom(repoClass)) { - return repoClass.getDeclaredConstructor - (AtomicReference.class, BDBRepositoryBuilder.class); - } - throw new ClassCastException("Not an instance of BDBRepository: " + repoClass.getName()); - } - - public static interface DatabaseHook { - /** - * Returns an appropriate database name for the given type. Simply - * return the type name as-is to support default behavior. - */ - String databaseName(String typeName); - - /** - * Called right before database is opened. - * - * @param db reference to database or config - actual type depends on BDB - * implementation. - */ - void prepareForOpening(Object db) throws RepositoryException; - } -} +/* + * Copyright 2006-2012 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.repo.sleepycat; + +import java.lang.reflect.Constructor; + +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import java.util.concurrent.atomic.AtomicReference; + +import org.cojen.util.ThrowUnchecked; + +import com.amazon.carbonado.Repository; +import com.amazon.carbonado.RepositoryException; +import com.amazon.carbonado.Storable; +import com.amazon.carbonado.repo.indexed.IndexedRepositoryBuilder; + +import com.amazon.carbonado.raw.CompressionType; +import com.amazon.carbonado.raw.CompressedStorableCodecFactory; +import com.amazon.carbonado.raw.StorableCodecFactory; + +import com.amazon.carbonado.spi.AbstractRepositoryBuilder; + +import com.amazon.carbonado.ConfigurationException; + +/** + * Builder and configuration options for BDBRepository. + * + * 

+ * BDBRepositoryBuilder builder = new BDBRepositoryBuilder();
+ *
+ * builder.setProduct("JE");
+ * builder.setName("test");
+ * builder.setEnvironmentHome("/tmp/testRepo");
+ * builder.setTransactionWriteNoSync(true);
+ *
+ * Repository repo = builder.build();
+ *
+ * + *

+ * The following extra capabilities are supported: + *

    + *
  • {@link com.amazon.carbonado.capability.IndexInfoCapability IndexInfoCapability} + *
  • {@link com.amazon.carbonado.capability.StorableInfoCapability StorableInfoCapability} + *
  • {@link com.amazon.carbonado.capability.ShutdownCapability ShutdownCapability} + *
  • {@link com.amazon.carbonado.layout.LayoutCapability LayoutCapability} + *
  • {@link com.amazon.carbonado.sequence.SequenceCapability SequenceCapability} + *
  • {@link CheckpointCapability CheckpointCapability} + *
  • {@link EnvironmentCapability EnvironmentCapability} + *
+ * + * @author Brian S O'Neill + * @author Vidya Iyer + * @author Nicole Deflaux + */ +public final class BDBRepositoryBuilder extends AbstractRepositoryBuilder { + + private static final BDBProduct DEFAULT_PRODUCT = BDBProduct.JE; + + private static final int DEFAULT_CHECKPOINT_INTERVAL = 10000; + + private String mName; + private boolean mIsMaster = true; + private BDBProduct mProduct = DEFAULT_PRODUCT; + private File mEnvHome; + private File mDataHome; + private String mSingleFileName; + private Map mFileNames; + private boolean mIndexSupport = true; + private boolean mIndexRepairEnabled = true; + private double mIndexThrottle = 1.0; + private boolean mReadOnly; + private Long mCacheSize; + private Integer mCachePercent; + private Integer mLogRegionSize; + private double mLockTimeout = 0.5; + private Integer mMaxLocks; + private double mTxnTimeout = 300.0; + private boolean mTxnNoSync; + private boolean mTxnWriteNoSync; + private Integer mTxnMaxActive = 1000; + private Boolean mDatabasesTransactional = null; + private boolean mReverseSplitOff; + private Map, Integer> mDatabasePageSizes; + private boolean mPrivate; + private boolean mMultiversion; + private boolean mLogInMemory; + private Integer mLogFileMaxSize; + private boolean mInitializeLogging; + private boolean mRunFullRecovery; + private boolean mRunCheckpointer = true; + private int mCheckpointInterval = DEFAULT_CHECKPOINT_INTERVAL; + private int mCheckpointThresholdKB = 1024; + private int mCheckpointThresholdMinutes = 1; + private boolean mKeepOldLogFiles; + private boolean mRunDeadlockDetector = true; + private Boolean mChecksumEnabled; + private Object mInitialEnvConfig = null; + private Object mInitialDBConfig = null; + private StorableCodecFactory mStorableCodecFactory; + private Runnable mPreShutdownHook; + private Runnable mPostShutdownHook; + private DatabaseHook mDatabaseHook; + private Map mCompressionMap; + + private BDBPanicHandler mPanicHandler; + + public BDBRepositoryBuilder() { + } + + public Repository build(AtomicReference rootRef) throws RepositoryException { + if (mIndexSupport) { + // Wrap BDBRepository with IndexedRepository. + + // Temporarily set to false to avoid infinite recursion. + mIndexSupport = false; + try { + IndexedRepositoryBuilder ixBuilder = new IndexedRepositoryBuilder(); + ixBuilder.setWrappedRepository(this); + ixBuilder.setMaster(isMaster()); + ixBuilder.setIndexRepairEnabled(mIndexRepairEnabled); + ixBuilder.setIndexRepairThrottle(mIndexThrottle); + return ixBuilder.build(rootRef); + } finally { + mIndexSupport = true; + } + } + + if (mStorableCodecFactory == null) { + mStorableCodecFactory = new CompressedStorableCodecFactory(mCompressionMap); + } + + assertReady(); + + // Make environment directory if it doesn't exist. + File homeFile = getEnvironmentHomeFile(); + if (!homeFile.exists()) { + if (!homeFile.mkdirs()) { + throw new RepositoryException + ("Unable to make environment home directory: " + homeFile); + } + } + + BDBRepository repo; + + try { + repo = getRepositoryConstructor().newInstance(rootRef, this); + } catch (Exception e) { + ThrowUnchecked.fireFirstDeclaredCause(e, RepositoryException.class); + // Not reached. + return null; + } + + rootRef.set(repo); + return repo; + } + + /** + * Opens the BDB environment, checks if it is corrupt, and then closes it. + * Only one process should open the environment for verification. Expect it + * to take a long time. + * + * @param out optional stream to capture any verfication errors + * @return true if environment passes verification + */ + public boolean verify(PrintStream out) throws RepositoryException { + final StorableCodecFactory codecFactory = mStorableCodecFactory; + final String name = mName; + final boolean readOnly = mReadOnly; + final boolean runCheckpointer = mRunCheckpointer; + final boolean runDeadlockDetector = mRunDeadlockDetector; + final boolean isPrivate = mPrivate; + + if (mName == null) { + // Allow a dummy name for verification. + mName = "BDB verification"; + } + + if (mStorableCodecFactory == null) { + mStorableCodecFactory = new CompressedStorableCodecFactory(mCompressionMap); + } + + mReadOnly = true; + mRunCheckpointer = false; + mRunDeadlockDetector = false; + + try { + assertReady(); + + File homeFile = getEnvironmentHomeFile(); + if (!homeFile.exists()) { + throw new RepositoryException + ("Environment home directory does not exist: " + homeFile); + } + + AtomicReference rootRef = new AtomicReference(); + BDBRepository repo; + + try { + repo = getRepositoryConstructor().newInstance(rootRef, this); + } catch (Exception e) { + ThrowUnchecked.fireFirstDeclaredCause(e, RepositoryException.class); + // Not reached. + return false; + } + + rootRef.set(repo); + + try { + return repo.verify(out); + } catch (Exception e) { + throw repo.toRepositoryException(e); + } finally { + repo.close(); + } + } finally { + mName = name; + mStorableCodecFactory = codecFactory; + mReadOnly = readOnly; + mRunCheckpointer = runCheckpointer; + mRunDeadlockDetector = runDeadlockDetector; + } + } + + public String getName() { + return mName; + } + + public void setName(String name) { + mName = name; + } + + public boolean isMaster() { + return mIsMaster; + } + + public void setMaster(boolean b) { + mIsMaster = b; + } + + /** + * Sets the BDB product to use, which defaults to JE. Also supported is DB + * and DB_HA. If not supported, an IllegalArgumentException is thrown. + */ + public void setProduct(String product) { + mProduct = product == null ? DEFAULT_PRODUCT : BDBProduct.forString(product); + } + + /** + * Returns the BDB product to use, which is JE by default. + */ + public String getProduct() { + return mProduct.toString(); + } + + /** + * Sets the BDB product to use, which defaults to JE. + */ + public void setBDBProduct(BDBProduct product) { + mProduct = product == null ? DEFAULT_PRODUCT : product; + } + + /** + * Returns the BDB product to use, which is JE by default. + */ + public BDBProduct getBDBProduct() { + return mProduct; + } + + /** + * Sets the repository environment home directory, which is required. + */ + public void setEnvironmentHomeFile(File envHome) { + try { + // Switch to canonical for more detailed error messages. + envHome = envHome.getCanonicalFile(); + } catch (IOException e) { + } + mEnvHome = envHome; + } + + /** + * Returns the repository environment home directory. + */ + public File getEnvironmentHomeFile() { + return mEnvHome; + } + + /** + * Sets the repository environment home directory, which is required. + * + * @throws RepositoryException if environment home is not valid + */ + public void setEnvironmentHome(String envHome) { + setEnvironmentHomeFile(new File(envHome)); + } + + /** + * Returns the repository environment home directory. + */ + public String getEnvironmentHome() { + return mEnvHome.getPath(); + } + + /** + * By default, data files are stored relative to the environment home. Call + * this method to override. For BDBRepositories that are log files only, + * this configuration is ignored. + */ + public void setDataHomeFile(File dir) { + if (dir != null) { + try { + // Switch to canonical for more detailed error messages. + dir = dir.getCanonicalFile(); + } catch (IOException e) { + } + } + mDataHome = dir; + } + + /** + * Returns the optional directory to store data files. Returns null if data + * files are expected to be relative to the environment home. + */ + public File getDataHomeFile() { + if (mDataHome == null) { + return getEnvironmentHomeFile(); + } + return mDataHome; + } + + /** + * By default, data files are stored relative to the environment home. Call + * this method to override. For BDBRepositories that are log files only, + * this configuration is ignored. + */ + public void setDataHome(String dir) { + if (dir == null) { + mDataHome = null; + } else { + setDataHomeFile(new File(dir)); + } + } + + /** + * Returns the directory to store data files. + */ + public String getDataHome() { + return getDataHomeFile().getPath(); + } + + /** + * Specify that all BDB databases should reside in one file, except for log + * files and caches. The filename is relative to the environment home, + * unless data directories have been specified. For BDBRepositories that + * are log files only, this configuration is ignored. + * + *

Note: When setting this option, the storable codec factory must also + * be changed, since the default storable codec factory is unable to + * distinguish storable types that reside in a single database file. Call + * setFileName instead to use built-in BDB feature for supporting multiple + * databases in one file. + */ + public void setSingleFileName(String filename) { + mSingleFileName = filename; + mFileNames = null; + } + + /** + * Returns the single file that all BDB databases should reside in. + */ + public String getSingleFileName() { + return mSingleFileName; + } + + /** + * Specify the file that a BDB database should reside in, except for log + * files and caches. The filename is relative to the environment home, + * unless data directories have been specified. For BDBRepositories that + * are log files only, this configuration is ignored. + * + * @param filename BDB database filename + * @param typeName type to store in file; if null, the file is used by default + * for all types + */ + public void setFileName(String filename, String typeName) { + mSingleFileName = null; + if (mFileNames == null) { + mFileNames = new HashMap(); + } + mFileNames.put(typeName, filename); + } + + Map getFileNameMap() { + if (mFileNames == null) { + return null; + } + return new HashMap(mFileNames); + } + + /** + * By default, user specified indexes are supported. Pass false to disable + * this, and no indexes will be built. Another consequence of this option + * is that no unique constraint checks will be applied to alternate keys. + */ + public void setIndexSupport(boolean indexSupport) { + mIndexSupport = indexSupport; + } + + /** + * Returns true if indexes are supported, which is true by default. + */ + public boolean getIndexSupport() { + return mIndexSupport; + } + + /** + * @see #setIndexRepairEnabled(boolean) + * + * @return true by default + */ + public boolean isIndexRepairEnabled() { + return mIndexRepairEnabled; + } + + /** + * By default, index repair is enabled. In this mode, the first time a + * Storable type is used, new indexes are populated and old indexes are + * removed. Until finished, access to the Storable is blocked. + * + *

When index repair is disabled, the Storable is immediately + * available. This does have consequences, however. The set of indexes + * available for queries is defined by the intersection of the old + * and new index sets. The set of indexes that are kept up-to-date is + * defined by the union of the old and new index sets. + * + *

While index repair is disabled, another process can safely repair the + * indexes in the background. When it is complete, index repair can be + * enabled for this repository too. + */ + public void setIndexRepairEnabled(boolean enabled) { + mIndexRepairEnabled = enabled; + } + + /** + * Returns the throttle parameter used when indexes are added, dropped or + * bulk repaired. By default this value is 1.0, or maximum speed. + */ + public double getIndexRepairThrottle() { + return mIndexThrottle; + } + + /** + * Sets the throttle parameter used when indexes are added, dropped or bulk + * repaired. By default this value is 1.0, or maximum speed. + * + * @param desiredSpeed 1.0 = perform work at full speed, + * 0.5 = perform work at half speed, 0.0 = fully suspend work + */ + public void setIndexRepairThrottle(double desiredSpeed) { + mIndexThrottle = desiredSpeed; + } + + /** + * Sets the repository to read-only mode. By default, repository is opened + * for reads and writes. + */ + public void setReadOnly(boolean readOnly) { + mReadOnly = readOnly; + } + + /** + * Returns true if repository should be opened read-only. + */ + public boolean getReadOnly() { + return mReadOnly; + } + + /** + * Set the repository cache size, in bytes. Actual BDB implementation will + * select a suitable default if this is not set. + */ + public void setCacheSize(long cacheSize) { + mCacheSize = cacheSize; + } + + /** + * Set the repository cache size, in bytes. Actual BDB implementation will + * select a suitable default if this is not set. + * + * @param cacheSize cache size to use, or null for default + */ + public void setCacheSize(Long cacheSize) { + mCacheSize = cacheSize; + } + + /** + * Returns the repository cache size, or null if default should be + * selected. + */ + public Long getCacheSize() { + return mCacheSize; + } + + /** + * Set the repository log region size, in bytes. + */ + public void setLogRegionSize(int logRegionSize) { + mLogRegionSize = logRegionSize; + } + + /** + * Set the repository log region size, in bytes. + */ + public void setLogRegionSize(Integer logRegionSize) { + mLogRegionSize = logRegionSize; + } + + /** + * Returns the repository log region size, or null if the default + * should be selected. + */ + public Integer getLogRegionSize() { + return mLogRegionSize; + } + + /** + * Set the percent of JVM heap used by the repository cache. Actual + * BDB implementation will select a suitable default if this is not + * set. This is overridden by setting an explicit cacheSize. + */ + public void setCachePercent(int cachePercent) { + mCachePercent = cachePercent; + } + + /** + * Set the percent of JVM heap used by the repository cache. Actual + * BDB implementation will select a suitable default if this is not + * set. This is overridden by setting an explicit cacheSize. + * + * @param cachePercent percent of JVM heap to use, or null for default + */ + public void setCachePercent(Integer cachePercent) { + mCachePercent = cachePercent; + } + + /** + * Returns the percent of JVM heap used by the repository cache, or + * null if default should be selected. + */ + public Integer getCachePercent() { + return mCachePercent; + } + + /** + * Set the lock timeout, in seconds. Default value is 0.5 seconds. + */ + public void setLockTimeout(double lockTimeout) { + mLockTimeout = lockTimeout; + } + + /** + * Returns the lock timeout, in seconds. + */ + public double getLockTimeout() { + return mLockTimeout; + } + + /** + * Returns the lock timeout, in microseconds, limited to max long value. + */ + public long getLockTimeoutInMicroseconds() { + return inMicros(mLockTimeout); + } + + public void setMaxLocks(Integer max) { + mMaxLocks = max; + } + + public Integer getMaxLocks() { + return mMaxLocks; + } + + /** + * Set the transaction timeout, in seconds. Default value is 300 seconds. + */ + public void setTransactionTimeout(double txnTimeout) { + mTxnTimeout = txnTimeout; + } + + /** + * Returns the repository transaction timeout, in seconds. + */ + public double getTransactionTimeout() { + return mTxnTimeout; + } + + /** + * Returns the repository transaction timeout, in microseconds, limited to + * max long value. + */ + public long getTransactionTimeoutInMicroseconds() { + return inMicros(mTxnTimeout); + } + + /** + * When true, commits are not immediately written or flushed to disk. This + * improves performance, but there is a chance of losing the most recent + * commits if the process is killed or if the machine crashes. + */ + public void setTransactionNoSync(boolean noSync) { + mTxnNoSync = noSync; + } + + /** + * Returns true if transactions are not written or flushed to disk. + */ + public boolean getTransactionNoSync() { + return mTxnNoSync; + } + + /** + * When true, commits are written, but they are not flushed to disk. This + * improves performance, but there is a chance of losing the most recent + * commits if the machine crashes. + */ + public void setTransactionWriteNoSync(boolean noSync) { + mTxnWriteNoSync = noSync; + } + + /** + * Returns true if transactions are not flushed to disk. + */ + public boolean getTransactionWriteNoSync() { + return mTxnWriteNoSync; + } + + /** + * Set the maximum number of concurrent transactions, or pass null to use + * the default. This setting has no effect for BDB-JE. + */ + public void setTransactionMaxActive(Integer max) { + mTxnMaxActive = max; + } + + /** + * Returns the maximum number of concurrent transactions, or null if the + * default is used. + */ + public Integer getTransactionMaxActive() { + return mTxnMaxActive; + } + + /** + * When true, allows databases to be transactional. This setting affects + * the databases, not the environment. If this is not explicitly set, the + * environment getTransactional is used. + */ + public void setDatabasesTransactional(Boolean transactional) { + mDatabasesTransactional = transactional; + } + + /** + * Returns true if the databases are configured to be transactional, + * false if configured to not be transactional, null if this override was never set + */ + public Boolean getDatabasesTransactional() { + return mDatabasesTransactional; + } + + /** + * Pass true to disable reverse split of B-tree nodes to reduce deadlocks. + * This setting has no effect for BDB-JE. + */ + public void setReverseSplitOff(boolean off) { + mReverseSplitOff = off; + } + + public boolean isReverseSplitOff() { + return mReverseSplitOff; + } + + /** + * Sets the desired page size for a given type. If not specified, the page + * size applies to all types. + */ + public void setDatabasePageSize(Integer bytes, Class type) { + if (mDatabasePageSizes == null) { + mDatabasePageSizes = new HashMap, Integer>(); + } + mDatabasePageSizes.put(type, bytes); + } + + Map, Integer> getDatabasePagesMap() { + if (mDatabasePageSizes == null) { + return null; + } + return new HashMap, Integer>(mDatabasePageSizes); + } + + /** + * When true, BDB environment cannot be shared by other processes, and + * region files are not created. By default, environment is shared, if + * supported. + */ + public void setPrivate(boolean b) { + mPrivate = b; + } + + /** + * Returns true if BDB environment is private. By default, environment is + * shared, if supported. + */ + public boolean isPrivate() { + return mPrivate; + } + + /** + * Set true to enable multiversion concurrency control (MVCC) on BDB + * environment. This enables snapshot isolation, and is it is not supported + * by all BDB products and versions. + */ + public void setMultiversion(boolean multiversion) { + mMultiversion = multiversion; + } + + /** + * Returns false by default because multiversion concurrency control (MVCC) + * is not enabled. + */ + public boolean isMultiversion() { + return mMultiversion; + } + + /** + * Set true to store transaction logs in memory only instead of persistent + * storage. For BDB products which are entirely log based, no records are + * ever persisted. + */ + public void setLogInMemory(boolean logInMemory) { + mLogInMemory = logInMemory; + } + + /** + * Returns false by default, indicating that transaction logs are persisted. + */ + public boolean getLogInMemory() { + return mLogInMemory; + } + + /** + * Set the maximum transaction log file size for the BDB environment. + */ + public void setLogFileMaxSize(Integer sizeInBytes) { + mLogFileMaxSize = sizeInBytes; + } + + /** + * Returns null if default size will be used. + */ + public Integer getLogFileMaxSize() { + return mLogFileMaxSize; + } + + /** + * Ensure the transaction logging sub-system is initialized, which is + * usually implied. + */ + public void setInitializeLogging(boolean b) { + mInitializeLogging = b; + } + + public boolean getInitializeLogging() { + return mInitializeLogging; + } + + /** + * Pass true to override the default and run a full (catastrophic) recovery + * when environment is opened. This setting has no effect for BDB-JE. + */ + public void setRunFullRecovery(boolean runRecovery) { + mRunFullRecovery = runRecovery; + } + + /** + * Returns true if a full (catastrophic) recovery should be performed when + * environment is opened. + */ + public boolean getRunFullRecovery() { + return mRunFullRecovery; + } + + /** + * Disable automatic checkpointing of database if another process is + * responsible for that. The false setting is implied for read-only + * databases. + */ + public void setRunCheckpointer(boolean runCheckpointer) { + mRunCheckpointer = runCheckpointer; + } + + /** + * Returns true if checkpointer is run automatically. + */ + public boolean getRunCheckpointer() { + return mRunCheckpointer; + } + + /** + * Set the interval to run checkpoints. This setting is ignored if the + * checkpointer is not configured to run. + * + * @param intervalMillis interval between checkpoints, in milliseconds + */ + public void setCheckpointInterval(int intervalMillis) { + mCheckpointInterval = intervalMillis; + } + + /** + * @return interval between checkpoints, in milliseconds + */ + public int getCheckpointInterval() { + return mCheckpointInterval; + } + + /** + * Set the size threshold to run checkpoints. This setting is ignored if + * the checkpointer is not configured to run. Default value is 1024 KB. + * + *

Checkpoint threshold is only used by Carbonado's built-in + * checkpointer, and is ignored when using BDB-JE. + * + * @param thresholdKB run checkpoint if at least this many kilobytes in log + */ + public void setCheckpointThresholdKB(int thresholdKB) { + mCheckpointThresholdKB = thresholdKB; + } + + /** + * @return run checkpoint if at least this many kilobytes in log + */ + public int getCheckpointThresholdKB() { + return mCheckpointThresholdKB; + } + + /** + * Set the time threshold to run checkpoints. This setting is ignored if + * the checkpointer is not configured to run. Default value is 1 minute. + * + *

Checkpoint threshold is only used by Carbonado's built-in + * checkpointer, and is ignored when using BDB-JE. + * + * @param thresholdMinutes run checkpoint if at least this many minutes + * passed since last checkpoint + */ + public void setCheckpointThresholdMinutes(int thresholdMinutes) { + mCheckpointThresholdMinutes = thresholdMinutes; + } + + /** + * @return run checkpoint if at least this many minutes passed since last + * checkpoint + */ + public int getCheckpointThresholdMinutes() { + return mCheckpointThresholdMinutes; + } + + /** + * By default, transaction log files are deleted when no longer needed. + * Keeping log files can be used for incremental backups or for diagnosing + * problems. If using BDB-JE, old log files are renamed with a ".del" + * extension. If using BDB-core, the db_archive utility is required for + * identifying old log files. + */ + public void setKeepOldLogFiles(boolean keep) { + mKeepOldLogFiles = keep; + } + + /** + * Returns false by default. + */ + public boolean getKeepOldLogFiles() { + return mKeepOldLogFiles; + } + + /** + * Disable automatic deadlock detection of database if another thread is + * responsible for that. + */ + public void setRunDeadlockDetector(boolean runDeadlockDetector) { + mRunDeadlockDetector = runDeadlockDetector; + } + + /** + * Returns true if deadlock detector is configured to run. + */ + public boolean getRunDeadlockDetector() { + return mRunDeadlockDetector; + } + + /** + * When true, enable checksum verification of pages read into the cache + * from the backing filestore. By default checksum is enabled for BDB-JE, + * and disabled for BDB-C. + */ + public void setChecksumEnabled(Boolean checksumEnabled) { + mChecksumEnabled = checksumEnabled; + } + + /** + * Returns true if checksum verification is enabled. Returns null if the + * BDB default is used. + */ + public Boolean getChecksumEnabled() { + return mChecksumEnabled; + } + + /** + * Optionally set the BDB specific environment configuration to + * use. The builder will verify that needed configuration values are set. + */ + public void setInitialEnvironmentConfig(Object envConfig) { + mInitialEnvConfig = envConfig; + } + + /** + * Returns the optional BDB specific environment configuration to use. + */ + public Object getInitialEnvironmentConfig() { + return mInitialEnvConfig; + } + + /** + * Optionally set the BDB specific database configuration to use + * for all databases created. The storage will verify that needed + * configuration values are set. + */ + public void setInitialDatabaseConfig(Object dbConfig) { + mInitialDBConfig = dbConfig; + } + + /** + * Returns the optional BDB specific database configuration to use + * for all databases created. + */ + public Object getInitialDatabaseConfig() { + return mInitialDBConfig; + } + + /** + * Override the default storable codec factory. + */ + public void setStorableCodecFactory(StorableCodecFactory factory) { + mStorableCodecFactory = factory; + } + + /** + * Returns the storable codec factory used. + */ + public StorableCodecFactory getStorableCodecFactory() { + return mStorableCodecFactory; + } + + /** + * Sets a callback to be invoked before the repository has finished running + * its own shutdown hooks. This method is also invoked when repository is + * manually closed. + */ + public void setPreShutdownHook(Runnable hook) { + mPreShutdownHook = hook; + } + + /** + * Returns the custom shutdown hook that runs before the repository has + * finished running its own shutdown hooks, or null if none. + */ + public Runnable getPreShutdownHook() { + return mPreShutdownHook; + } + + /** + * Sets a callback to be invoked after repository has finished running its + * own shutdown hooks. This method is also invoked when repository is + * manually closed. + */ + public void setShutdownHook(Runnable hook) { + mPostShutdownHook = hook; + } + + /** + * Returns the custom shutdown hook that runs after the repository has + * finished running its own shutdown hooks, or null if none. + */ + public Runnable getShutdownHook() { + return mPostShutdownHook; + } + + /** + * Sets a hook to be called whenever a database is opened. + */ + public void setDatabaseHook(DatabaseHook hook) { + mDatabaseHook = hook; + } + + /** + * Returns the custom open database hook, or null if none. + */ + public DatabaseHook getDatabaseHook() { + return mDatabaseHook; + } + + /** + * Set the compressor for the given class, overriding a custom StorableCodecFactory. + + * @param type Storable to compress. + * @param compressionType String representation of type of + * compression. Available options are "NONE" for no compression or "GZIP" + * for gzip compression + */ + public void setCompressor(String type, String compressionType) { + mStorableCodecFactory = null; + compressionType = compressionType.toUpperCase(); + if (mCompressionMap == null) { + mCompressionMap = new HashMap(); + } + CompressionType compressionEnum = CompressionType.valueOf(compressionType); + if (compressionEnum != null) { + mCompressionMap.put(type, compressionEnum); + } + } + + /** + * Return the compressor used for the given storable. + * @param type Storable to compress + * @return String representation of the type of compression used. Available options are "NONE" + * for no compression and "GZIP" for gzip compression. + */ + public String getCompressor(String type) { + if (mCompressionMap == null) { + return null; + } + + return mCompressionMap.get(type).toString(); + } + + /** + * Set the handler to call if the database panics. + * + * @param handler + */ + public void setPanicHandler(BDBPanicHandler handler) { + mPanicHandler = handler; + } + + /** + * Return the panic handler to call if the database panics. + * + * @return The BDBPanicHandler or null if unset. + */ + public BDBPanicHandler getPanicHandler() { + return mPanicHandler; + } + + private long inMicros(double seconds) { + if (seconds >= Long.MAX_VALUE) { + return Long.MAX_VALUE; + } + if (seconds <= 0 || Double.isNaN(seconds)) { + return 0L; + } + return (long) (seconds * 1000000); + } + + @Override + public void errorCheck(Collection messages) throws ConfigurationException { + super.errorCheck(messages); + + checkClass: { + Exception error; + try { + getRepositoryConstructor(); + break checkClass; + } catch (ClassCastException e) { + error = e; + } catch (ClassNotFoundException e) { + error = e; + } catch (NoSuchMethodException e) { + error = e; + } + messages.add("BDB product \"" + getProduct() + "\" not supported: " + error); + } + + File envHome = getEnvironmentHomeFile(); + if (envHome == null) { + messages.add("environmentHome missing"); + } else { + if (envHome.exists() && !envHome.isDirectory()) { + messages.add("environment home is not a directory: " + envHome); + } + } + } + + /** + * Looks up appropriate repository via reflection, whose name is derived + * from the BDB product string. + */ + @SuppressWarnings("unchecked") + private Constructor getRepositoryConstructor() + throws ClassCastException, ClassNotFoundException, NoSuchMethodException + { + String packageName; + { + String thisClassName = getClass().getName(); + packageName = thisClassName.substring(0, thisClassName.lastIndexOf('.')); + } + String className = packageName + '.' + getBDBProduct().name() + "_Repository"; + Class repoClass = Class.forName(className); + if (BDBRepository.class.isAssignableFrom(repoClass)) { + return repoClass.getDeclaredConstructor + (AtomicReference.class, BDBRepositoryBuilder.class); + } + throw new ClassCastException("Not an instance of BDBRepository: " + repoClass.getName()); + } + + public static interface DatabaseHook { + /** + * Returns an appropriate database name for the given type. Simply + * return the type name as-is to support default behavior. + */ + String databaseName(String typeName); + + /** + * Called right before database is opened. + * + * @param db reference to database or config - actual type depends on BDB + * implementation. + */ + void prepareForOpening(Object db) throws RepositoryException; + } +} + -- cgit v1.2.3