From 742805682f6784ddd9c1b5b476305dbdcad6dd08 Mon Sep 17 00:00:00 2001
From: "Brian S. O'Neill" <bronee@gmail.com>
Date: Mon, 5 Nov 2012 00:11:55 +0000
Subject: Add layout metadata sync utility.
---
.../java/com/amazon/carbonado/layout/Layout.java | 63 +++--
.../com/amazon/carbonado/layout/LayoutSync.java | 297 +++++++++++++++++++++
2 files changed, 327 insertions(+), 33 deletions(-)
create mode 100644 src/main/java/com/amazon/carbonado/layout/LayoutSync.java
(limited to 'src/main/java/com/amazon/carbonado/layout')
diff --git a/src/main/java/com/amazon/carbonado/layout/Layout.java b/src/main/java/com/amazon/carbonado/layout/Layout.java
index ef7f74c..89806fe 100644
--- a/src/main/java/com/amazon/carbonado/layout/Layout.java
+++ b/src/main/java/com/amazon/carbonado/layout/Layout.java
@@ -176,19 +176,7 @@ public class Layout {
storedLayout.setLayoutID(layoutID);
storedLayout.setStorableTypeName(info.getStorableType().getName());
- storedLayout.setCreationTimestamp(System.currentTimeMillis());
- try {
- storedLayout.setCreationUser(System.getProperty("user.name"));
- } catch (SecurityException e) {
- // Can't get user, no big deal.
- }
- try {
- storedLayout.setCreationHost(InetAddress.getLocalHost().getHostName());
- } catch (UnknownHostException e) {
- // Can't get host, no big deal.
- } catch (SecurityException e) {
- // Can't get host, no big deal.
- }
+ fillInCreationInfo(storedLayout);
if (options == null) {
mOptions = null;
@@ -213,6 +201,25 @@ public class Layout {
mAllProperties = Collections.unmodifiableList(list);
}
+ static void fillInCreationInfo(StoredLayout storedLayout) {
+ storedLayout.setCreationTimestamp(System.currentTimeMillis());
+ try {
+ storedLayout.setCreationUser(System.getProperty("user.name"));
+ } catch (SecurityException e) {
+ // Can't get user, no big deal.
+ storedLayout.setCreationUser(null);
+ }
+ try {
+ storedLayout.setCreationHost(InetAddress.getLocalHost().getHostName());
+ } catch (UnknownHostException e) {
+ // Can't get host, no big deal.
+ storedLayout.setCreationHost(null);
+ } catch (SecurityException e) {
+ // Can't get host, no big deal.
+ storedLayout.setCreationHost(null);
+ }
+ }
+
/**
* Returns a unique identifier for this layout.
*/
@@ -315,30 +322,20 @@ public class Layout {
* @throws FetchNoneException if generation not found
*/
public Layout getGeneration(int generation) throws FetchNoneException, FetchException {
- StoredLayout layout;
try {
- layout = getStoredLayoutByGeneration(generation);
- } catch (FetchNoneException e) {
- try {
- Storage<StoredLayoutEquivalence> equivStorage =
- mLayoutFactory.mRepository.storageFor(StoredLayoutEquivalence.class);
- StoredLayoutEquivalence equiv = equivStorage.prepare();
- equiv.setStorableTypeName(getStorableTypeName());
- equiv.setGeneration(generation);
- if (equiv.tryLoad()) {
- layout = getStoredLayoutByGeneration(equiv.getMatchedGeneration());
- } else {
- throw e;
- }
- } catch (RepositoryException e2) {
- if (e2 != e) {
- LogFactory.getLog(Layout.class).error("Unable to determine equivalance: ", e2);
- }
- throw e;
+ Storage<StoredLayoutEquivalence> equivStorage =
+ mLayoutFactory.mRepository.storageFor(StoredLayoutEquivalence.class);
+ StoredLayoutEquivalence equiv = equivStorage.prepare();
+ equiv.setStorableTypeName(getStorableTypeName());
+ equiv.setGeneration(generation);
+ if (equiv.tryLoad()) {
+ generation = equiv.getMatchedGeneration();
}
+ } catch (RepositoryException e) {
+ throw e.toFetchException();
}
- return new Layout(mLayoutFactory, layout);
+ return new Layout(mLayoutFactory, getStoredLayoutByGeneration(generation));
}
private StoredLayout getStoredLayoutByGeneration(int generation)
diff --git a/src/main/java/com/amazon/carbonado/layout/LayoutSync.java b/src/main/java/com/amazon/carbonado/layout/LayoutSync.java
new file mode 100644
index 0000000..510dc82
--- /dev/null
+++ b/src/main/java/com/amazon/carbonado/layout/LayoutSync.java
@@ -0,0 +1,297 @@
+/*
+ * Copyright 2012 Amazon Technologies, Inc. or its affiliates.
+ * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks
+ * of Amazon Technologies, Inc. or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.amazon.carbonado.layout;
+
+import java.util.Arrays;
+
+import com.amazon.carbonado.Cursor;
+import com.amazon.carbonado.Query;
+import com.amazon.carbonado.Repository;
+import com.amazon.carbonado.RepositoryException;
+import com.amazon.carbonado.Storage;
+import com.amazon.carbonado.Transaction;
+
+import com.amazon.carbonado.cursor.FetchAheadCursor;
+import com.amazon.carbonado.cursor.FilteredCursor;
+
+/**
+ * Synchronizes layout metadata between two repositories. Both source and
+ * destination might be updated.
+ *
+ * @author Brian S O'Neill
+ */
+public class LayoutSync {
+ private final Repository mSource;
+ private final Repository mDestination;
+
+ public LayoutSync(Repository source, Repository destination) {
+ mSource = source;
+ mDestination = destination;
+ }
+
+ /**
+ * @return true if any changes to source were made
+ */
+ public boolean run() throws RepositoryException {
+ if (doRun()) {
+ // Second pass.
+ doRun();
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * @return true if a second pass should be run
+ */
+ private boolean doRun() throws RepositoryException {
+ Storage<StoredLayout> srcLayoutStorage = mSource.storageFor(StoredLayout.class);
+ Storage<StoredLayout> dstLayoutStorage = mDestination.storageFor(StoredLayout.class);
+
+ Cursor<StoredLayout> srcLayouts = srcLayoutStorage.query().orderBy("layoutID").fetch();
+ try {
+ Cursor<StoredLayout> dstLayouts = dstLayoutStorage.query().orderBy("layoutID").fetch();
+ try {
+ return doRun(srcLayouts, dstLayouts);
+ } finally {
+ dstLayouts.close();
+ }
+ } finally {
+ srcLayouts.close();
+ }
+ }
+
+ /**
+ * @return true if another pass should be run
+ */
+ private boolean doRun(Cursor<StoredLayout> srcLayouts, Cursor<StoredLayout> dstLayouts)
+ throws RepositoryException
+ {
+ // Fetch ahead to prevent BDB cursor deadlocks.
+ srcLayouts = new FetchAheadCursor<StoredLayout>(srcLayouts, 1000);
+ dstLayouts = new FetchAheadCursor<StoredLayout>(dstLayouts, 1000);
+
+ boolean doAgain = false;
+
+ StoredLayout src = null;
+ StoredLayout dst = null;
+
+ while (true) {
+ if (src == null) {
+ if (!srcLayouts.hasNext()) {
+ return doAgain;
+ }
+ src = srcLayouts.next();
+ }
+
+ if (dst == null && dstLayouts.hasNext()) {
+ dst = dstLayouts.next();
+ }
+
+ if (dst == null || src.getLayoutID() < dst.getLayoutID()) {
+ // Insert missing layout. If a generation conflict, do over in
+ // second pass after all source generations have been inserted.
+ doAgain |= !tryInsert(src);
+ src = null;
+ continue;
+ }
+
+ if (src.getLayoutID() > dst.getLayoutID()) {
+ // Skip layout which only exists in destination.
+ dst = null;
+ continue;
+ }
+
+ if (src.getGeneration() != dst.getGeneration()) {
+ // Same layouts, but with different generation. Create a new
+ // non-conflicting generation to replace both.
+ createNewGen(src, dst);
+ doAgain = true;
+ }
+
+ src = null;
+ dst = null;
+ }
+ }
+
+ /**
+ * @return false if generation conflict
+ */
+ private boolean tryInsert(StoredLayout src) throws RepositoryException {
+ Storage<StoredLayout> dstLayoutStorage = mDestination.storageFor(StoredLayout.class);
+
+ Storage<StoredLayoutProperty> srcPropStorage =
+ mSource.storageFor(StoredLayoutProperty.class);
+ Storage<StoredLayoutProperty> dstPropStorage =
+ mDestination.storageFor(StoredLayoutProperty.class);
+
+ Transaction txn = mDestination.enterTransaction();
+ try {
+ txn.setForUpdate(true);
+ StoredLayout dst = dstLayoutStorage.prepare();
+ src.copyAllProperties(dst);
+ if (!dst.tryInsert()) {
+ return false;
+ }
+
+ Cursor<StoredLayoutProperty> srcProps =
+ srcPropStorage.query("layoutID = ?").with(src.getLayoutID()).fetch();
+ while (srcProps.hasNext()) {
+ StoredLayoutProperty srcProp = srcProps.next();
+ StoredLayoutProperty dstProp = dstPropStorage.prepare();
+ srcProp.copyAllProperties(dstProp);
+ if (!dstProp.tryInsert()) {
+ dstProp.tryDelete();
+ dstProp.insert();
+ }
+ }
+
+ txn.commit();
+ } finally {
+ txn.exit();
+ }
+
+ return true;
+ }
+
+ private void createNewGen(StoredLayout src, StoredLayout dst) throws RepositoryException {
+ long layoutID = src.getLayoutID();
+ if (layoutID != dst.getLayoutID()) {
+ throw new AssertionError();
+ }
+
+ String storableTypeName = src.getStorableTypeName();
+ if (!storableTypeName.equals(dst.getStorableTypeName())) {
+ throw new AssertionError();
+ }
+
+ Transaction dstTxn = mDestination.enterTransaction();
+ try {
+ dstTxn.setForUpdate(true);
+
+ Transaction srcTxn = mSource.enterTransaction();
+ try {
+ srcTxn.setForUpdate(true);
+
+ // New layout generation which is not used by src or dst.
+ int newGen = nextGen(storableTypeName);
+
+ int oldSrcGen = src.getGeneration();
+ int oldDstGen = dst.getGeneration();
+
+ long now = System.currentTimeMillis();
+
+ // Remap source layout to new generation.
+ doCreateNewGen(now, mSource, layoutID, oldSrcGen, newGen);
+
+ // Remap destination layout to new generation.
+ doCreateNewGen(now, mDestination, layoutID, oldDstGen, newGen);
+
+ // Check if source has a layout generation matching the
+ // conflicting destination layout. It cannot be used anymore.
+ Cursor<StoredLayout> srcLayouts =
+ findLayouts(mSource, storableTypeName, oldDstGen);
+ while (srcLayouts.hasNext()) {
+ src = srcLayouts.next();
+ newGen = nextGen(storableTypeName);
+ doCreateNewGen(now, mSource, src.getLayoutID(), oldDstGen, newGen);
+ }
+
+ // Check if destination has a layout generation matching the
+ // conflicting source layout. It cannot be used anymore.
+ Cursor<StoredLayout> dstLayouts =
+ findLayouts(mDestination, storableTypeName, oldSrcGen);
+ while (dstLayouts.hasNext()) {
+ dst = dstLayouts.next();
+ newGen = nextGen(storableTypeName);
+ doCreateNewGen(now, mDestination, dst.getLayoutID(), oldSrcGen, newGen);
+ }
+
+ srcTxn.commit();
+ } finally {
+ srcTxn.exit();
+ }
+
+ dstTxn.commit();
+ } finally {
+ dstTxn.exit();
+ }
+ }
+
+ private void doCreateNewGen(long now, Repository repo, long layoutID, int oldGen, int newGen)
+ throws RepositoryException
+ {
+ Storage<StoredLayout> layoutStorage =
+ repo.storageFor(StoredLayout.class);
+ Storage<StoredLayoutEquivalence> equivStorage =
+ repo.storageFor(StoredLayoutEquivalence.class);
+
+ StoredLayout newLayout = layoutStorage.prepare();
+ newLayout.setLayoutID(layoutID);
+ newLayout.load();
+ Layout.fillInCreationInfo(newLayout);
+ newLayout.setGeneration(newGen);
+ // Consistent timestamp for all records.
+ newLayout.setCreationTimestamp(now);
+ newLayout.update();
+
+ StoredLayoutEquivalence equiv = equivStorage.prepare();
+ equiv.setStorableTypeName(newLayout.getStorableTypeName());
+ equiv.setGeneration(oldGen);
+ equiv.setMatchedGeneration(newGen);
+
+ equiv.insert();
+ }
+
+ private int nextGen(String storableTypeName) throws RepositoryException {
+ return Math.max(nextGen(mSource, storableTypeName),
+ nextGen(mDestination, storableTypeName));
+ }
+
+ private int nextGen(Repository repo, String storableTypeName)
+ throws RepositoryException
+ {
+ Cursor<StoredLayout> cursor =
+ repo.storageFor(StoredLayout.class)
+ .query("storableTypeName = ?").with(storableTypeName)
+ .orderBy("-generation")
+ .fetchSlice(0, 1L);
+
+ try {
+ // Must always return a result. Otherwise, no conflict existed in
+ // the first place, indicating a bug.
+ return cursor.next().getGeneration() + 1;
+ } finally {
+ cursor.close();
+ }
+
+ // TODO: also choose from highest equivalence
+ }
+
+ private Cursor<StoredLayout> findLayouts(Repository repo,
+ String storableTypeName, int generation)
+ throws RepositoryException
+ {
+ // Query without using the index, in case it's inconsistent.
+ return FilteredCursor.applyFilter
+ (repo.storageFor(StoredLayout.class).query().fetch(),
+ StoredLayout.class, "storableTypeName = ? & generation = ?",
+ storableTypeName, generation);
+ }
+}
--
cgit v1.2.3