From 742805682f6784ddd9c1b5b476305dbdcad6dd08 Mon Sep 17 00:00:00 2001 From: "Brian S. O'Neill" Date: Mon, 5 Nov 2012 00:11:55 +0000 Subject: Add layout metadata sync utility. --- .../java/com/amazon/carbonado/layout/Layout.java | 63 +++-- .../com/amazon/carbonado/layout/LayoutSync.java | 297 +++++++++++++++++++++ 2 files changed, 327 insertions(+), 33 deletions(-) create mode 100644 src/main/java/com/amazon/carbonado/layout/LayoutSync.java (limited to 'src/main/java') diff --git a/src/main/java/com/amazon/carbonado/layout/Layout.java b/src/main/java/com/amazon/carbonado/layout/Layout.java index ef7f74c..89806fe 100644 --- a/src/main/java/com/amazon/carbonado/layout/Layout.java +++ b/src/main/java/com/amazon/carbonado/layout/Layout.java @@ -176,19 +176,7 @@ public class Layout { storedLayout.setLayoutID(layoutID); storedLayout.setStorableTypeName(info.getStorableType().getName()); - storedLayout.setCreationTimestamp(System.currentTimeMillis()); - try { - storedLayout.setCreationUser(System.getProperty("user.name")); - } catch (SecurityException e) { - // Can't get user, no big deal. - } - try { - storedLayout.setCreationHost(InetAddress.getLocalHost().getHostName()); - } catch (UnknownHostException e) { - // Can't get host, no big deal. - } catch (SecurityException e) { - // Can't get host, no big deal. - } + fillInCreationInfo(storedLayout); if (options == null) { mOptions = null; @@ -213,6 +201,25 @@ public class Layout { mAllProperties = Collections.unmodifiableList(list); } + static void fillInCreationInfo(StoredLayout storedLayout) { + storedLayout.setCreationTimestamp(System.currentTimeMillis()); + try { + storedLayout.setCreationUser(System.getProperty("user.name")); + } catch (SecurityException e) { + // Can't get user, no big deal. + storedLayout.setCreationUser(null); + } + try { + storedLayout.setCreationHost(InetAddress.getLocalHost().getHostName()); + } catch (UnknownHostException e) { + // Can't get host, no big deal. + storedLayout.setCreationHost(null); + } catch (SecurityException e) { + // Can't get host, no big deal. + storedLayout.setCreationHost(null); + } + } + /** * Returns a unique identifier for this layout. */ @@ -315,30 +322,20 @@ public class Layout { * @throws FetchNoneException if generation not found */ public Layout getGeneration(int generation) throws FetchNoneException, FetchException { - StoredLayout layout; try { - layout = getStoredLayoutByGeneration(generation); - } catch (FetchNoneException e) { - try { - Storage equivStorage = - mLayoutFactory.mRepository.storageFor(StoredLayoutEquivalence.class); - StoredLayoutEquivalence equiv = equivStorage.prepare(); - equiv.setStorableTypeName(getStorableTypeName()); - equiv.setGeneration(generation); - if (equiv.tryLoad()) { - layout = getStoredLayoutByGeneration(equiv.getMatchedGeneration()); - } else { - throw e; - } - } catch (RepositoryException e2) { - if (e2 != e) { - LogFactory.getLog(Layout.class).error("Unable to determine equivalance: ", e2); - } - throw e; + Storage equivStorage = + mLayoutFactory.mRepository.storageFor(StoredLayoutEquivalence.class); + StoredLayoutEquivalence equiv = equivStorage.prepare(); + equiv.setStorableTypeName(getStorableTypeName()); + equiv.setGeneration(generation); + if (equiv.tryLoad()) { + generation = equiv.getMatchedGeneration(); } + } catch (RepositoryException e) { + throw e.toFetchException(); } - return new Layout(mLayoutFactory, layout); + return new Layout(mLayoutFactory, getStoredLayoutByGeneration(generation)); } private StoredLayout getStoredLayoutByGeneration(int generation) diff --git a/src/main/java/com/amazon/carbonado/layout/LayoutSync.java b/src/main/java/com/amazon/carbonado/layout/LayoutSync.java new file mode 100644 index 0000000..510dc82 --- /dev/null +++ b/src/main/java/com/amazon/carbonado/layout/LayoutSync.java @@ -0,0 +1,297 @@ +/* + * Copyright 2012 Amazon Technologies, Inc. or its affiliates. + * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks + * of Amazon Technologies, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.amazon.carbonado.layout; + +import java.util.Arrays; + +import com.amazon.carbonado.Cursor; +import com.amazon.carbonado.Query; +import com.amazon.carbonado.Repository; +import com.amazon.carbonado.RepositoryException; +import com.amazon.carbonado.Storage; +import com.amazon.carbonado.Transaction; + +import com.amazon.carbonado.cursor.FetchAheadCursor; +import com.amazon.carbonado.cursor.FilteredCursor; + +/** + * Synchronizes layout metadata between two repositories. Both source and + * destination might be updated. + * + * @author Brian S O'Neill + */ +public class LayoutSync { + private final Repository mSource; + private final Repository mDestination; + + public LayoutSync(Repository source, Repository destination) { + mSource = source; + mDestination = destination; + } + + /** + * @return true if any changes to source were made + */ + public boolean run() throws RepositoryException { + if (doRun()) { + // Second pass. + doRun(); + return true; + } + return false; + } + + /** + * @return true if a second pass should be run + */ + private boolean doRun() throws RepositoryException { + Storage srcLayoutStorage = mSource.storageFor(StoredLayout.class); + Storage dstLayoutStorage = mDestination.storageFor(StoredLayout.class); + + Cursor srcLayouts = srcLayoutStorage.query().orderBy("layoutID").fetch(); + try { + Cursor dstLayouts = dstLayoutStorage.query().orderBy("layoutID").fetch(); + try { + return doRun(srcLayouts, dstLayouts); + } finally { + dstLayouts.close(); + } + } finally { + srcLayouts.close(); + } + } + + /** + * @return true if another pass should be run + */ + private boolean doRun(Cursor srcLayouts, Cursor dstLayouts) + throws RepositoryException + { + // Fetch ahead to prevent BDB cursor deadlocks. + srcLayouts = new FetchAheadCursor(srcLayouts, 1000); + dstLayouts = new FetchAheadCursor(dstLayouts, 1000); + + boolean doAgain = false; + + StoredLayout src = null; + StoredLayout dst = null; + + while (true) { + if (src == null) { + if (!srcLayouts.hasNext()) { + return doAgain; + } + src = srcLayouts.next(); + } + + if (dst == null && dstLayouts.hasNext()) { + dst = dstLayouts.next(); + } + + if (dst == null || src.getLayoutID() < dst.getLayoutID()) { + // Insert missing layout. If a generation conflict, do over in + // second pass after all source generations have been inserted. + doAgain |= !tryInsert(src); + src = null; + continue; + } + + if (src.getLayoutID() > dst.getLayoutID()) { + // Skip layout which only exists in destination. + dst = null; + continue; + } + + if (src.getGeneration() != dst.getGeneration()) { + // Same layouts, but with different generation. Create a new + // non-conflicting generation to replace both. + createNewGen(src, dst); + doAgain = true; + } + + src = null; + dst = null; + } + } + + /** + * @return false if generation conflict + */ + private boolean tryInsert(StoredLayout src) throws RepositoryException { + Storage dstLayoutStorage = mDestination.storageFor(StoredLayout.class); + + Storage srcPropStorage = + mSource.storageFor(StoredLayoutProperty.class); + Storage dstPropStorage = + mDestination.storageFor(StoredLayoutProperty.class); + + Transaction txn = mDestination.enterTransaction(); + try { + txn.setForUpdate(true); + StoredLayout dst = dstLayoutStorage.prepare(); + src.copyAllProperties(dst); + if (!dst.tryInsert()) { + return false; + } + + Cursor srcProps = + srcPropStorage.query("layoutID = ?").with(src.getLayoutID()).fetch(); + while (srcProps.hasNext()) { + StoredLayoutProperty srcProp = srcProps.next(); + StoredLayoutProperty dstProp = dstPropStorage.prepare(); + srcProp.copyAllProperties(dstProp); + if (!dstProp.tryInsert()) { + dstProp.tryDelete(); + dstProp.insert(); + } + } + + txn.commit(); + } finally { + txn.exit(); + } + + return true; + } + + private void createNewGen(StoredLayout src, StoredLayout dst) throws RepositoryException { + long layoutID = src.getLayoutID(); + if (layoutID != dst.getLayoutID()) { + throw new AssertionError(); + } + + String storableTypeName = src.getStorableTypeName(); + if (!storableTypeName.equals(dst.getStorableTypeName())) { + throw new AssertionError(); + } + + Transaction dstTxn = mDestination.enterTransaction(); + try { + dstTxn.setForUpdate(true); + + Transaction srcTxn = mSource.enterTransaction(); + try { + srcTxn.setForUpdate(true); + + // New layout generation which is not used by src or dst. + int newGen = nextGen(storableTypeName); + + int oldSrcGen = src.getGeneration(); + int oldDstGen = dst.getGeneration(); + + long now = System.currentTimeMillis(); + + // Remap source layout to new generation. + doCreateNewGen(now, mSource, layoutID, oldSrcGen, newGen); + + // Remap destination layout to new generation. + doCreateNewGen(now, mDestination, layoutID, oldDstGen, newGen); + + // Check if source has a layout generation matching the + // conflicting destination layout. It cannot be used anymore. + Cursor srcLayouts = + findLayouts(mSource, storableTypeName, oldDstGen); + while (srcLayouts.hasNext()) { + src = srcLayouts.next(); + newGen = nextGen(storableTypeName); + doCreateNewGen(now, mSource, src.getLayoutID(), oldDstGen, newGen); + } + + // Check if destination has a layout generation matching the + // conflicting source layout. It cannot be used anymore. + Cursor dstLayouts = + findLayouts(mDestination, storableTypeName, oldSrcGen); + while (dstLayouts.hasNext()) { + dst = dstLayouts.next(); + newGen = nextGen(storableTypeName); + doCreateNewGen(now, mDestination, dst.getLayoutID(), oldSrcGen, newGen); + } + + srcTxn.commit(); + } finally { + srcTxn.exit(); + } + + dstTxn.commit(); + } finally { + dstTxn.exit(); + } + } + + private void doCreateNewGen(long now, Repository repo, long layoutID, int oldGen, int newGen) + throws RepositoryException + { + Storage layoutStorage = + repo.storageFor(StoredLayout.class); + Storage equivStorage = + repo.storageFor(StoredLayoutEquivalence.class); + + StoredLayout newLayout = layoutStorage.prepare(); + newLayout.setLayoutID(layoutID); + newLayout.load(); + Layout.fillInCreationInfo(newLayout); + newLayout.setGeneration(newGen); + // Consistent timestamp for all records. + newLayout.setCreationTimestamp(now); + newLayout.update(); + + StoredLayoutEquivalence equiv = equivStorage.prepare(); + equiv.setStorableTypeName(newLayout.getStorableTypeName()); + equiv.setGeneration(oldGen); + equiv.setMatchedGeneration(newGen); + + equiv.insert(); + } + + private int nextGen(String storableTypeName) throws RepositoryException { + return Math.max(nextGen(mSource, storableTypeName), + nextGen(mDestination, storableTypeName)); + } + + private int nextGen(Repository repo, String storableTypeName) + throws RepositoryException + { + Cursor cursor = + repo.storageFor(StoredLayout.class) + .query("storableTypeName = ?").with(storableTypeName) + .orderBy("-generation") + .fetchSlice(0, 1L); + + try { + // Must always return a result. Otherwise, no conflict existed in + // the first place, indicating a bug. + return cursor.next().getGeneration() + 1; + } finally { + cursor.close(); + } + + // TODO: also choose from highest equivalence + } + + private Cursor findLayouts(Repository repo, + String storableTypeName, int generation) + throws RepositoryException + { + // Query without using the index, in case it's inconsistent. + return FilteredCursor.applyFilter + (repo.storageFor(StoredLayout.class).query().fetch(), + StoredLayout.class, "storableTypeName = ? & generation = ?", + storableTypeName, generation); + } +} -- cgit v1.2.3