Added Incremental Backup to HotBackupCapability

2 files changed, 229 insertions, 46 deletions

diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java
index be6549a..ed90278 100644
--- a/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java
+++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/BDBRepository.java
@@ -113,6 +113,7 @@ abstract class BDBRepository<Txn> extends AbstractRepository<Txn>
 

     final boolean mRunCheckpointer;

     final boolean mKeepOldLogFiles;

+    final boolean mLogInMemory;

     final boolean mRunDeadlockDetector;

 

     final File mDataHome;

@@ -122,6 +123,7 @@ abstract class BDBRepository<Txn> extends AbstractRepository<Txn>
 

     final Object mBackupLock = new Object();

     int mBackupCount = 0;

+    int mIncrementalBackupCount = 0;

 

     private LayoutFactory mLayoutFactory;

 

@@ -157,6 +159,7 @@ abstract class BDBRepository<Txn> extends AbstractRepository<Txn>
 

         mRunCheckpointer = !builder.getReadOnly() && builder.getRunCheckpointer();

         mKeepOldLogFiles = builder.getKeepOldLogFiles();

+	mLogInMemory = builder.getLogInMemory();

         mRunDeadlockDetector = builder.getRunDeadlockDetector();

         mStorableCodecFactory = builder.getStorableCodecFactory();

         mPreShutdownHook = builder.getPreShutdownHook();

@@ -215,62 +218,60 @@ abstract class BDBRepository<Txn> extends AbstractRepository<Txn>
         }

         return StorableIntrospector.examine(type).getAllProperties().get(name) != null;

     }

+    

+    @Override 

+    public Backup startBackup() throws RepositoryException {

+	return startBackup(false);

+    }

 

     @Override

-    public Backup startBackup() throws RepositoryException {

+    public Backup startBackup(boolean deleteOldLogFiles) throws RepositoryException {

+	if (mLogInMemory) {

+	    throw new IllegalStateException("Log files are only kept in memory and backups cannot be performed");

+	}

+

         synchronized (mBackupLock) {

             int count = mBackupCount;

             if (count == 0) {

                 try {

-                    enterBackupMode();

+                    enterBackupMode(deleteOldLogFiles);

                 } catch (Exception e) {

                     throw mExTransformer.toRepositoryException(e);

                 }

             }

             mBackupCount = count + 1;

 

-            return new Backup() {

-                private boolean mDone;

-

-                @Override

-                public void endBackup() throws RepositoryException {

-                    synchronized (mBackupLock) {

-                        if (mDone) {

-                            return;

-                        }

-                        mDone = true;

-

-                        int count = mBackupCount - 1;

-                        try {

-                            if (count == 0) {

-                                try {

-                                    exitBackupMode();

-                                } catch (Exception e) {

-                                    throw mExTransformer.toRepositoryException(e);

-                                }

-                            }

-                        } finally {

-                            mBackupCount = count;

-                        }

-                    }

-                }

+	    return new FullBackup();

+        }

+    }

 

-                @Override

-                public File[] getFiles() throws RepositoryException {

-                    synchronized (mBackupLock) {

-                        if (mDone) {

-                            throw new IllegalStateException("Backup has ended");

-                        }

+    @Override 

+    public Backup startIncrementalBackup(long lastLogNumber) 

+	throws RepositoryException

+    {

+	return startIncrementalBackup(lastLogNumber, false);

+    }

 

-                        try {

-                            return backupFiles();

-                        } catch (Exception e) {

-                            throw mExTransformer.toRepositoryException(e);

-                        }

-                    }

-                }

-            };

-        }

+    @Override

+    public Backup startIncrementalBackup(long lastLogNumber, boolean deleteOldLogFiles) 

+	throws RepositoryException

+    {

+	if (mLogInMemory) {

+	    throw new IllegalStateException("Log files are only kept in memory and incremental backup cannot be performed");

+	}

+

+	if (lastLogNumber < 0) {

+	    throw new IllegalArgumentException("The number of the last backup cannot be negative");

+	}

+	synchronized (mBackupLock) {

+	    try {

+		enterIncrementalBackupMode(lastLogNumber, deleteOldLogFiles);

+		++mIncrementalBackupCount;

+	    } catch (Exception e) {

+		throw mExTransformer.toRepositoryException(e);

+	    }

+	}

+	return new IncrementalBackup(lastLogNumber);

     }

 

     /**

@@ -665,7 +666,7 @@ abstract class BDBRepository<Txn> extends AbstractRepository<Txn>
     /**

      * Called only the first time a backup is started.

      */

-    abstract void enterBackupMode() throws Exception;

+    abstract void enterBackupMode(boolean deleteOldLogFiles) throws Exception;

 

     /**

      * Called only after the last backup ends.

@@ -673,9 +674,28 @@ abstract class BDBRepository<Txn> extends AbstractRepository<Txn>
     abstract void exitBackupMode() throws Exception;

 

     /**

+     * Called only when an incremental backup is started.

+     */

+    abstract void enterIncrementalBackupMode(long lastLogNumber, boolean deleteOldLogFiles) throws Exception;

+

+    /**

+     * Called only after incremental backup ends.

+     */

+    abstract void exitIncrementalBackupMode() throws Exception;

+

+    /**

      * Called only if in backup mode.

+     *

+     * @param newLastLogNum reference to last log number at [0]

+     */

+    abstract File[] backupFiles(long[] newLastLogNum) throws Exception;

+

+    /**

+     * Called only if in incremental backup mode.

+     *

+     * @param newLastLogNum reference to last log number at [0]

      */

-    abstract File[] backupFiles() throws Exception;

+    abstract File[] incrementalBackup(long lastLogNumber, long[] newLastLogNum) throws Exception;

 

     FetchException toFetchException(Throwable e) {

         return mExTransformer.toFetchException(e);

@@ -892,4 +912,102 @@ abstract class BDBRepository<Txn> extends AbstractRepository<Txn>
             }

         }

     }

-}

+

+    abstract class AbstractBackup implements Backup {

+	boolean mDone;

+	long mFinalLogNumber;

+

+	AbstractBackup() {

+	    mFinalLogNumber = -1;

+	}

+

+	@Override 

+	public void endBackup() throws RepositoryException {

+	    synchronized (mBackupLock) {

+		if (mDone) {

+		    return;

+		}

+		mDone = true;

+		finishBackup();

+	    }

+	}

+

+	@Override

+	public File[] getFiles() throws RepositoryException {

+	    synchronized (mBackupLock) {

+		if (mDone) {

+		    throw new IllegalStateException("Backup has ended");

+		}

+		

+		try {

+		    long[] newLastLogNum = {-1}; 

+		    File[] toReturn =  getBackupFiles(newLastLogNum);

+		    mFinalLogNumber = newLastLogNum[0];

+		    return toReturn;

+		} catch (Exception e) {

+		    throw mExTransformer.toRepositoryException(e);

+		}

+	    }

+	}

+	

+	@Override

+	public long getLastLogNumber() throws RepositoryException {

+	    if (mFinalLogNumber < 0) {

+		throw new IllegalStateException("Must get files prior to retrieving the last log number");

+	    }

+	    return mFinalLogNumber;

+	}	

+	

+	abstract void finishBackup() throws RepositoryException;

+

+	abstract File[] getBackupFiles(long[] newLastLogNum) throws Exception;

+    }

+

+    class IncrementalBackup extends AbstractBackup {

+	private final long mLastLogNumber;

+	

+	IncrementalBackup(long lastLogNumber) {

+	    super();

+	    mLastLogNumber = lastLogNumber;

+	}

+

+	@Override

+	public void finishBackup() throws RepositoryException {

+	    --mIncrementalBackupCount;

+	    

+	    try {

+		exitIncrementalBackupMode();

+	    } catch (Exception e) {

+		throw mExTransformer.toRepositoryException(e);

+	    }

+	}

+	

+	@Override

+	public File[] getBackupFiles(long[] newLastLogNum) throws Exception {

+	    return incrementalBackup(mLastLogNumber, newLastLogNum);

+	}

+    }

+

+    class FullBackup extends AbstractBackup {

+	@Override

+	void finishBackup() throws RepositoryException {

+	    int count = mBackupCount - 1;

+	    try {

+		if (count == 0) {

+		    try {

+			exitBackupMode();

+		    } catch (Exception e) {

+			throw mExTransformer.toRepositoryException(e);

+		    }

+		}

+	    } finally {

+		mBackupCount = count;

+	    }

+	}

+	

+	@Override

+	public File[] getBackupFiles(long[] newLastLogNum) throws Exception {

+	    return backupFiles(newLastLogNum);

+	}

+    }

+}
\ No newline at end of file
diff --git a/src/main/java/com/amazon/carbonado/repo/sleepycat/HotBackupCapability.java b/src/main/java/com/amazon/carbonado/repo/sleepycat/HotBackupCapability.java
index 85011e9..76cfb7e 100644
--- a/src/main/java/com/amazon/carbonado/repo/sleepycat/HotBackupCapability.java
+++ b/src/main/java/com/amazon/carbonado/repo/sleepycat/HotBackupCapability.java
@@ -29,31 +29,96 @@ import com.amazon.carbonado.capability.Capability;
  * from a hot backup, it is <b>critical</b> that a full recovery be

  * performed. BDB-JE does not require this, however. Pass true to {@link

  * BDBRepositoryBuilder#setRunFullRecovery(boolean)} to enable.

+ * 

+ * <p>

+ * If incremental backups are performed it is required that

+ * log file removal is disabled in the underlying database.

  *

  * @author Brian S O'Neill

+ * @author Olga Kuznetsova

  * @since 1.2.1

  */

 public interface HotBackupCapability extends Capability {

+///TODO:Have log file deletion be queued after all backups are completed

     /**

      * Starts the backup by disabling log file deletion. Be sure to call

      * endBackup when done to resume log file cleanup. Concurrent backups are

      * supported.

+     *

+     * <p>

+     * To perform incremental backups use the builder option of setLogInMemory(false) 

+     * so that old log files are not deleted. Log files can be deleted in the

+     * future before starting a new backup (see method below).

+     *

+     * @throws IllegalStateException if log files are being removed (setLogInMemory(true))

      */

     Backup startBackup() throws RepositoryException;

 

+    /**

+     * Starts the backup by disabling log file deletion. Be sure to call 

+     * endBackup when done to resume log file cleanup. Concurrent backups are supported.

+     *

+     * <p>

+     * Caution should be observed when deleting old log files by force as log files as they may be required

+     * for future incremental backups (if concurrent backups are running). 

+     * If any concurrent backups are occurring, log fail deletion will fail.

+     *

+     * @param deleteOldLogFiles deletes log files that are no longer in use and have been backed up. False by default.

+     * @throws IllegalStateException if log files are being removed (setLogInMemory(true))

+     */

+    Backup startBackup(boolean deleteOldLogFiles) throws RepositoryException;

+

+    /**

+     * Starts an incremental backup. Log files that are newer than the lastLogNumber will be copied

+     * during the backup. Should only be run after performing a full backup.

+     *

+     * @param lastLogNumber number of the last log file that was copied in a previous backup.

+     * @throws IllegalArgumentException if lastLogNumber is negative

+     * @throws IllegalStateException if log files are being removed (setLogInMemory(true))

+     */

+    Backup startIncrementalBackup(long lastLogNumber) throws RepositoryException;

+

+    /**

+     * Starts an incremental backup. Log files that are newer than the lastLogNumber will be copied

+     * during the backup. Can only be run after performing a full backup.

+     *

+     * <p>

+     * Caution should be observed when deleting old log files by force as log files as they may be required

+     * for future incremental backups (if concurrent backups are running). 

+     * If any concurrent backups are occurring, log fail deletion will fail.

+     *

+     * @param lastLogNumber number of the last log file that was copied in a previous backup.

+     * @param deleteOldLogFiles deletes log files that are no longer in use and have been backed up. False by default.

+     * @throws IllegalArgumentException if lastLogNumber is negative

+     * @throws IllegalStateException if log files are being removed (setLogInMemory(true))

+     */

+    Backup startIncrementalBackup(long lastLogNumber, boolean deleteOldLogFiles) throws RepositoryException;

+

     public static interface Backup {

         /**

-         * End the backup and resume log file cleanup.

+	 * Resume normal operation.

          */

         void endBackup() throws RepositoryException;

 

         /**

          * Returns all the files to be copied, in the exact order in which they

          * must be copied.

+	 *

+	 * <p>

+	 * These files must be copied prior to calling endBackup().

          *

          * @return ordered array of absolute files

          * @throws IllegalStateException if backup has ended

          */

         File[] getFiles() throws RepositoryException;

+

+	/**

+	 * Can be called after a backup has been performed to find the last log file

+	 * that has been backed up.

+	 *

+	 * @return the file number of the last file in the current backup set.

+	 * This number is required to perform incremental backups.

+	 */

+	long getLastLogNumber() throws RepositoryException;

     }

 }