From 4cc824245c96d6caad657ea798974da0751c956b Mon Sep 17 00:00:00 2001 From: Olga Kuznetsova Date: Fri, 14 May 2010 23:54:10 +0000 Subject: Added Incremental Backup to HotBackupCapability --- .../carbonado/repo/sleepycat/BDBRepository.java | 208 ++++++++++++++++----- .../repo/sleepycat/HotBackupCapability.java | 67 ++++++- 2 files changed, 229 insertions(+), 46 deletions(-) (limited to 'src') diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java index be6549a..ed90278 100644 --- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java +++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java @@ -113,6 +113,7 @@ abstract class BDBRepository extends AbstractRepository final boolean mRunCheckpointer; final boolean mKeepOldLogFiles; + final boolean mLogInMemory; final boolean mRunDeadlockDetector; final File mDataHome; @@ -122,6 +123,7 @@ abstract class BDBRepository extends AbstractRepository final Object mBackupLock = new Object(); int mBackupCount = 0; + int mIncrementalBackupCount = 0; private LayoutFactory mLayoutFactory; @@ -157,6 +159,7 @@ abstract class BDBRepository extends AbstractRepository mRunCheckpointer = !builder.getReadOnly() && builder.getRunCheckpointer(); mKeepOldLogFiles = builder.getKeepOldLogFiles(); + mLogInMemory = builder.getLogInMemory(); mRunDeadlockDetector = builder.getRunDeadlockDetector(); mStorableCodecFactory = builder.getStorableCodecFactory(); mPreShutdownHook = builder.getPreShutdownHook(); @@ -215,62 +218,60 @@ abstract class BDBRepository extends AbstractRepository } return StorableIntrospector.examine(type).getAllProperties().get(name) != null; } + + @Override + public Backup startBackup() throws RepositoryException { + return startBackup(false); + } @Override - public Backup startBackup() throws RepositoryException { + public Backup startBackup(boolean deleteOldLogFiles) throws RepositoryException { + if (mLogInMemory) { + throw new IllegalStateException("Log files are only kept in memory and backups cannot be performed"); + } + synchronized (mBackupLock) { int count = mBackupCount; if (count == 0) { try { - enterBackupMode(); + enterBackupMode(deleteOldLogFiles); } catch (Exception e) { throw mExTransformer.toRepositoryException(e); } } mBackupCount = count + 1; - return new Backup() { - private boolean mDone; - - @Override - public void endBackup() throws RepositoryException { - synchronized (mBackupLock) { - if (mDone) { - return; - } - mDone = true; - - int count = mBackupCount - 1; - try { - if (count == 0) { - try { - exitBackupMode(); - } catch (Exception e) { - throw mExTransformer.toRepositoryException(e); - } - } - } finally { - mBackupCount = count; - } - } - } + return new FullBackup(); + } + } - @Override - public File[] getFiles() throws RepositoryException { - synchronized (mBackupLock) { - if (mDone) { - throw new IllegalStateException("Backup has ended"); - } + @Override + public Backup startIncrementalBackup(long lastLogNumber) + throws RepositoryException + { + return startIncrementalBackup(lastLogNumber, false); + } - try { - return backupFiles(); - } catch (Exception e) { - throw mExTransformer.toRepositoryException(e); - } - } - } - }; - } + @Override + public Backup startIncrementalBackup(long lastLogNumber, boolean deleteOldLogFiles) + throws RepositoryException + { + if (mLogInMemory) { + throw new IllegalStateException("Log files are only kept in memory and incremental backup cannot be performed"); + } + + if (lastLogNumber < 0) { + throw new IllegalArgumentException("The number of the last backup cannot be negative"); + } + synchronized (mBackupLock) { + try { + enterIncrementalBackupMode(lastLogNumber, deleteOldLogFiles); + ++mIncrementalBackupCount; + } catch (Exception e) { + throw mExTransformer.toRepositoryException(e); + } + } + return new IncrementalBackup(lastLogNumber); } /** @@ -665,17 +666,36 @@ abstract class BDBRepository extends AbstractRepository /** * Called only the first time a backup is started. */ - abstract void enterBackupMode() throws Exception; + abstract void enterBackupMode(boolean deleteOldLogFiles) throws Exception; /** * Called only after the last backup ends. */ abstract void exitBackupMode() throws Exception; + /** + * Called only when an incremental backup is started. + */ + abstract void enterIncrementalBackupMode(long lastLogNumber, boolean deleteOldLogFiles) throws Exception; + + /** + * Called only after incremental backup ends. + */ + abstract void exitIncrementalBackupMode() throws Exception; + /** * Called only if in backup mode. + * + * @param newLastLogNum reference to last log number at [0] + */ + abstract File[] backupFiles(long[] newLastLogNum) throws Exception; + + /** + * Called only if in incremental backup mode. + * + * @param newLastLogNum reference to last log number at [0] */ - abstract File[] backupFiles() throws Exception; + abstract File[] incrementalBackup(long lastLogNumber, long[] newLastLogNum) throws Exception; FetchException toFetchException(Throwable e) { return mExTransformer.toFetchException(e); @@ -892,4 +912,102 @@ abstract class BDBRepository extends AbstractRepository } } } -} + + abstract class AbstractBackup implements Backup { + boolean mDone; + long mFinalLogNumber; + + AbstractBackup() { + mFinalLogNumber = -1; + } + + @Override + public void endBackup() throws RepositoryException { + synchronized (mBackupLock) { + if (mDone) { + return; + } + mDone = true; + finishBackup(); + } + } + + @Override + public File[] getFiles() throws RepositoryException { + synchronized (mBackupLock) { + if (mDone) { + throw new IllegalStateException("Backup has ended"); + } + + try { + long[] newLastLogNum = {-1}; + File[] toReturn = getBackupFiles(newLastLogNum); + mFinalLogNumber = newLastLogNum[0]; + return toReturn; + } catch (Exception e) { + throw mExTransformer.toRepositoryException(e); + } + } + } + + @Override + public long getLastLogNumber() throws RepositoryException { + if (mFinalLogNumber < 0) { + throw new IllegalStateException("Must get files prior to retrieving the last log number"); + } + return mFinalLogNumber; + } + + abstract void finishBackup() throws RepositoryException; + + abstract File[] getBackupFiles(long[] newLastLogNum) throws Exception; + } + + class IncrementalBackup extends AbstractBackup { + private final long mLastLogNumber; + + IncrementalBackup(long lastLogNumber) { + super(); + mLastLogNumber = lastLogNumber; + } + + @Override + public void finishBackup() throws RepositoryException { + --mIncrementalBackupCount; + + try { + exitIncrementalBackupMode(); + } catch (Exception e) { + throw mExTransformer.toRepositoryException(e); + } + } + + @Override + public File[] getBackupFiles(long[] newLastLogNum) throws Exception { + return incrementalBackup(mLastLogNumber, newLastLogNum); + } + } + + class FullBackup extends AbstractBackup { + @Override + void finishBackup() throws RepositoryException { + int count = mBackupCount - 1; + try { + if (count == 0) { + try { + exitBackupMode(); + } catch (Exception e) { + throw mExTransformer.toRepositoryException(e); + } + } + } finally { + mBackupCount = count; + } + } + + @Override + public File[] getBackupFiles(long[] newLastLogNum) throws Exception { + return backupFiles(newLastLogNum); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/HotBackupCapability.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/HotBackupCapability.java index 85011e9..76cfb7e 100644 --- a/src/main/java/com/amazon/carbonado/repo/sleepycat/HotBackupCapability.java +++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/HotBackupCapability.java @@ -29,31 +29,96 @@ import com.amazon.carbonado.capability.Capability; * from a hot backup, it is critical that a full recovery be * performed. BDB-JE does not require this, however. Pass true to {@link * BDBRepositoryBuilder#setRunFullRecovery(boolean)} to enable. + * + *

+ * If incremental backups are performed it is required that + * log file removal is disabled in the underlying database. * * @author Brian S O'Neill + * @author Olga Kuznetsova * @since 1.2.1 */ public interface HotBackupCapability extends Capability { +///TODO:Have log file deletion be queued after all backups are completed /** * Starts the backup by disabling log file deletion. Be sure to call * endBackup when done to resume log file cleanup. Concurrent backups are * supported. + * + *

+ * To perform incremental backups use the builder option of setLogInMemory(false) + * so that old log files are not deleted. Log files can be deleted in the + * future before starting a new backup (see method below). + * + * @throws IllegalStateException if log files are being removed (setLogInMemory(true)) */ Backup startBackup() throws RepositoryException; + /** + * Starts the backup by disabling log file deletion. Be sure to call + * endBackup when done to resume log file cleanup. Concurrent backups are supported. + * + *

+ * Caution should be observed when deleting old log files by force as log files as they may be required + * for future incremental backups (if concurrent backups are running). + * If any concurrent backups are occurring, log fail deletion will fail. + * + * @param deleteOldLogFiles deletes log files that are no longer in use and have been backed up. False by default. + * @throws IllegalStateException if log files are being removed (setLogInMemory(true)) + */ + Backup startBackup(boolean deleteOldLogFiles) throws RepositoryException; + + /** + * Starts an incremental backup. Log files that are newer than the lastLogNumber will be copied + * during the backup. Should only be run after performing a full backup. + * + * @param lastLogNumber number of the last log file that was copied in a previous backup. + * @throws IllegalArgumentException if lastLogNumber is negative + * @throws IllegalStateException if log files are being removed (setLogInMemory(true)) + */ + Backup startIncrementalBackup(long lastLogNumber) throws RepositoryException; + + /** + * Starts an incremental backup. Log files that are newer than the lastLogNumber will be copied + * during the backup. Can only be run after performing a full backup. + * + *

+ * Caution should be observed when deleting old log files by force as log files as they may be required + * for future incremental backups (if concurrent backups are running). + * If any concurrent backups are occurring, log fail deletion will fail. + * + * @param lastLogNumber number of the last log file that was copied in a previous backup. + * @param deleteOldLogFiles deletes log files that are no longer in use and have been backed up. False by default. + * @throws IllegalArgumentException if lastLogNumber is negative + * @throws IllegalStateException if log files are being removed (setLogInMemory(true)) + */ + Backup startIncrementalBackup(long lastLogNumber, boolean deleteOldLogFiles) throws RepositoryException; + public static interface Backup { /** - * End the backup and resume log file cleanup. + * Resume normal operation. */ void endBackup() throws RepositoryException; /** * Returns all the files to be copied, in the exact order in which they * must be copied. + * + *

+ * These files must be copied prior to calling endBackup(). * * @return ordered array of absolute files * @throws IllegalStateException if backup has ended */ File[] getFiles() throws RepositoryException; + + /** + * Can be called after a backup has been performed to find the last log file + * that has been backed up. + * + * @return the file number of the last file in the current backup set. + * This number is required to perform incremental backups. + */ + long getLastLogNumber() throws RepositoryException; } } -- cgit v1.2.3