From d5e7a0f66f639d42934074a751aa9af9f76a3ceb Mon Sep 17 00:00:00 2001 From: "Brian S. O'Neill" Date: Sun, 17 Sep 2006 21:40:47 +0000 Subject: Finished replacing old query engine. --- .../carbonado/qe/FullScanIndexedQueryExecutor.java | 96 ---------------- .../amazon/carbonado/qe/FullScanQueryExecutor.java | 4 +- .../amazon/carbonado/qe/IndexedQueryAnalyzer.java | 26 +++-- .../amazon/carbonado/qe/IndexedQueryExecutor.java | 34 +++--- .../com/amazon/carbonado/qe/KeyQueryExecutor.java | 6 +- .../java/com/amazon/carbonado/qe/OrderingList.java | 13 +++ .../com/amazon/carbonado/qe/RepositoryAccess.java | 4 +- .../amazon/carbonado/qe/SortedQueryExecutor.java | 8 +- .../com/amazon/carbonado/qe/StorageAccess.java | 1 - .../amazon/carbonado/qe/UnionQueryAnalyzer.java | 123 +++++++++++++++------ .../amazon/carbonado/qe/UnionQueryExecutor.java | 27 ++++- 11 files changed, 169 insertions(+), 173 deletions(-) delete mode 100644 src/main/java/com/amazon/carbonado/qe/FullScanIndexedQueryExecutor.java (limited to 'src/main/java/com/amazon/carbonado/qe') diff --git a/src/main/java/com/amazon/carbonado/qe/FullScanIndexedQueryExecutor.java b/src/main/java/com/amazon/carbonado/qe/FullScanIndexedQueryExecutor.java deleted file mode 100644 index cfd012b..0000000 --- a/src/main/java/com/amazon/carbonado/qe/FullScanIndexedQueryExecutor.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2006 Amazon Technologies, Inc. or its affiliates. - * Amazon, Amazon.com and Carbonado are trademarks or registered trademarks - * of Amazon Technologies, Inc. or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.amazon.carbonado.qe; - -import java.io.IOException; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import com.amazon.carbonado.Cursor; -import com.amazon.carbonado.FetchException; -import com.amazon.carbonado.Storable; - -import com.amazon.carbonado.filter.Filter; -import com.amazon.carbonado.filter.FilterValues; - -import com.amazon.carbonado.info.OrderedProperty; -import com.amazon.carbonado.info.StorableIndex; - -/** - * QueryExecutor which fully scans all Storables of a given type, referencing - * an index. - * - * @author Brian S O'Neill - */ -public class FullScanIndexedQueryExecutor extends AbstractQueryExecutor { - private final Support mSupport; - private final StorableIndex mIndex; - - /** - * @param support support for full scan - * @param index index to use, which may be a primary key index - * @throws IllegalArgumentException if support or index is null - */ - public FullScanIndexedQueryExecutor(Support support, StorableIndex index) { - if (support == null || index == null) { - throw new IllegalArgumentException(); - } - mSupport = support; - mIndex = index; - } - - /** - * Returns an open filter. - */ - public Filter getFilter() { - return Filter.getOpenFilter(mIndex.getStorableType()); - } - - public Cursor fetch(FilterValues values) throws FetchException { - return mSupport.fetch(mIndex); - } - - /** - * Returns the natural order of the index. - */ - public OrderingList getOrdering() { - return OrderingList.get(mIndex.getOrderedProperties()); - } - - public boolean printPlan(Appendable app, int indentLevel, FilterValues values) - throws IOException - { - indent(app, indentLevel); - app.append("full index scan: "); - app.append(mIndex.getStorableType().getName()); - newline(app); - return true; - } - - public static interface Support { - /** - * Perform a full scan of all Storables referenced by an index. - * - * @param index index to scan, which may be a primary key index - */ - Cursor fetch(StorableIndex index) throws FetchException; - } -} diff --git a/src/main/java/com/amazon/carbonado/qe/FullScanQueryExecutor.java b/src/main/java/com/amazon/carbonado/qe/FullScanQueryExecutor.java index 2875542..dbf5690 100644 --- a/src/main/java/com/amazon/carbonado/qe/FullScanQueryExecutor.java +++ b/src/main/java/com/amazon/carbonado/qe/FullScanQueryExecutor.java @@ -59,7 +59,7 @@ public class FullScanQueryExecutor extends AbstractQueryExec } public Cursor fetch(FilterValues values) throws FetchException { - return mSupport.fetch(); + return mSupport.fetchAll(); } /** @@ -85,6 +85,6 @@ public class FullScanQueryExecutor extends AbstractQueryExec /** * Perform a full scan of all Storables. */ - Cursor fetch() throws FetchException; + Cursor fetchAll() throws FetchException; } } diff --git a/src/main/java/com/amazon/carbonado/qe/IndexedQueryAnalyzer.java b/src/main/java/com/amazon/carbonado/qe/IndexedQueryAnalyzer.java index 9b46227..76f0b52 100644 --- a/src/main/java/com/amazon/carbonado/qe/IndexedQueryAnalyzer.java +++ b/src/main/java/com/amazon/carbonado/qe/IndexedQueryAnalyzer.java @@ -83,8 +83,10 @@ public class IndexedQueryAnalyzer { * @param ordering optional properties which define desired ordering * @throws IllegalArgumentException if filter is not supported */ - public Result analyze(Filter filter, OrderingList ordering) { - if (!filter.isBound()) { + public Result analyze(Filter filter, OrderingList ordering) + throws SupportException, RepositoryException + { + if (filter != null && !filter.isBound()) { throw new IllegalArgumentException("Filter must be bound"); } @@ -151,7 +153,9 @@ public class IndexedQueryAnalyzer { /** * @return null if no foreign indexes for property */ - private synchronized ForeignIndexes getForeignIndexes(ChainedProperty chainedProp) { + private synchronized ForeignIndexes getForeignIndexes(ChainedProperty chainedProp) + throws SupportException, RepositoryException + { // Remove the last property as it is expected to be a simple storable // property instead of a joined Storable. chainedProp = chainedProp.trim(); @@ -198,7 +202,9 @@ public class IndexedQueryAnalyzer { * Checks if the property is a join and its internal properties are fully * indexed. */ - private boolean isProperJoin(StorableProperty property) { + private boolean isProperJoin(StorableProperty property) + throws SupportException, RepositoryException + { if (!property.isJoin() || property.isQuery()) { return false; } @@ -227,7 +233,9 @@ public class IndexedQueryAnalyzer { return false; } - private boolean simpleAnalyze(Filter filter) { + private boolean simpleAnalyze(Filter filter) + throws SupportException, RepositoryException + { Collection> indexes = indexesFor(filter.getStorableType()); if (indexes != null) { @@ -242,7 +250,9 @@ public class IndexedQueryAnalyzer { return false; } - private Collection> indexesFor(Class type) { + private Collection> indexesFor(Class type) + throws SupportException, RepositoryException + { return mRepoAccess.storageAccessFor(type).getAllIndexes(); } @@ -509,10 +519,6 @@ public class IndexedQueryAnalyzer { { CompositeScore score = getCompositeScore(); FilteringScore fScore = score.getFilteringScore(); - - if (!fScore.hasAnyMatches()) { - return new FullScanIndexedQueryExecutor(access, index); - } if (fScore.isKeyMatch()) { return new KeyQueryExecutor(access, index, fScore); } diff --git a/src/main/java/com/amazon/carbonado/qe/IndexedQueryExecutor.java b/src/main/java/com/amazon/carbonado/qe/IndexedQueryExecutor.java index f8d2b23..d87740b 100644 --- a/src/main/java/com/amazon/carbonado/qe/IndexedQueryExecutor.java +++ b/src/main/java/com/amazon/carbonado/qe/IndexedQueryExecutor.java @@ -52,6 +52,7 @@ public class IndexedQueryExecutor extends AbstractQueryExecu private final Support mSupport; private final StorableIndex mIndex; + private final int mIdentityCount; private final Filter mIdentityFilter; private final List> mExclusiveRangeStartFilters; private final List> mInclusiveRangeStartFilters; @@ -78,6 +79,7 @@ public class IndexedQueryExecutor extends AbstractQueryExecu FilteringScore fScore = score.getFilteringScore(); + mIdentityCount = fScore.getIdentityCount(); mIdentityFilter = fScore.getIdentityFilter(); mExclusiveRangeStartFilters = fScore.getExclusiveRangeStartFilters(); mInclusiveRangeStartFilters = fScore.getInclusiveRangeStartFilters(); @@ -152,11 +154,11 @@ public class IndexedQueryExecutor extends AbstractQueryExecu } } - return mSupport.fetch(mIndex, identityValues, - rangeStartBoundary, rangeStartValue, - rangeEndBoundary, rangeEndValue, - mReverseRange, - mReverseOrder); + return mSupport.fetchSubset(mIndex, identityValues, + rangeStartBoundary, rangeStartValue, + rangeEndBoundary, rangeEndValue, + mReverseRange, + mReverseOrder); } public Filter getFilter() { @@ -179,7 +181,11 @@ public class IndexedQueryExecutor extends AbstractQueryExecu } public OrderingList getOrdering() { - return OrderingList.get(mIndex.getOrderedProperties()); + OrderingList list = OrderingList.get(mIndex.getOrderedProperties()); + if (mIdentityCount > 0) { + list = OrderingList.get(list.subList(mIdentityCount, list.size())); + } + return list; } public boolean printPlan(Appendable app, int indentLevel, FilterValues values) @@ -261,14 +267,14 @@ public class IndexedQueryExecutor extends AbstractQueryExecu * @param reverseOrder when true, iteration should be reversed from its * natural order */ - Cursor fetch(StorableIndex index, - Object[] identityValues, - BoundaryType rangeStartBoundary, - Object rangeStartValue, - BoundaryType rangeEndBoundary, - Object rangeEndValue, - boolean reverseRange, - boolean reverseOrder) + Cursor fetchSubset(StorableIndex index, + Object[] identityValues, + BoundaryType rangeStartBoundary, + Object rangeStartValue, + BoundaryType rangeEndBoundary, + Object rangeEndValue, + boolean reverseRange, + boolean reverseOrder) throws FetchException; } } diff --git a/src/main/java/com/amazon/carbonado/qe/KeyQueryExecutor.java b/src/main/java/com/amazon/carbonado/qe/KeyQueryExecutor.java index acde1a8..d8436c0 100644 --- a/src/main/java/com/amazon/carbonado/qe/KeyQueryExecutor.java +++ b/src/main/java/com/amazon/carbonado/qe/KeyQueryExecutor.java @@ -69,7 +69,7 @@ public class KeyQueryExecutor extends AbstractQueryExecutor< } public Cursor fetch(FilterValues values) throws FetchException { - return mSupport.fetch(mIndex, values.getValuesFor(mKeyFilter)); + return mSupport.fetchOne(mIndex, values.getValuesFor(mKeyFilter)); } public Filter getFilter() { @@ -87,7 +87,7 @@ public class KeyQueryExecutor extends AbstractQueryExecutor< throws IOException { indent(app, indentLevel); - app.append("index key: "); + app.append("index key match: "); app.append(mIndex.getStorableType().getName()); newline(app); indent(app, indentLevel); @@ -110,7 +110,7 @@ public class KeyQueryExecutor extends AbstractQueryExecutor< * @param index index to open, which may be a primary key index * @param identityValues of exactly matching values to apply to index */ - Cursor fetch(StorableIndex index, Object[] identityValues) + Cursor fetchOne(StorableIndex index, Object[] identityValues) throws FetchException; } } diff --git a/src/main/java/com/amazon/carbonado/qe/OrderingList.java b/src/main/java/com/amazon/carbonado/qe/OrderingList.java index 08a39f5..f6366e5 100644 --- a/src/main/java/com/amazon/carbonado/qe/OrderingList.java +++ b/src/main/java/com/amazon/carbonado/qe/OrderingList.java @@ -97,6 +97,19 @@ public class OrderingList extends AbstractList OrderingList get(List> orderings) { + OrderingList list = emptyList(); + if (orderings != null && orderings.size() > 0) { + for (OrderedProperty property : orderings) { + list = list.concat(property); + } + } + return list; + } + private static OrderingList getListHead(Class type) { OrderingList node; synchronized (cCache) { diff --git a/src/main/java/com/amazon/carbonado/qe/RepositoryAccess.java b/src/main/java/com/amazon/carbonado/qe/RepositoryAccess.java index 6e3c3de..cb25afd 100644 --- a/src/main/java/com/amazon/carbonado/qe/RepositoryAccess.java +++ b/src/main/java/com/amazon/carbonado/qe/RepositoryAccess.java @@ -20,6 +20,7 @@ package com.amazon.carbonado.qe; import com.amazon.carbonado.MalformedTypeException; import com.amazon.carbonado.Repository; +import com.amazon.carbonado.RepositoryException; import com.amazon.carbonado.Storable; import com.amazon.carbonado.SupportException; @@ -40,5 +41,6 @@ public interface RepositoryAccess { * @throws IllegalArgumentException if specified type is null * @throws MalformedTypeException if specified type is not suitable */ - StorageAccess storageAccessFor(Class type); + StorageAccess storageAccessFor(Class type) + throws SupportException, RepositoryException; } diff --git a/src/main/java/com/amazon/carbonado/qe/SortedQueryExecutor.java b/src/main/java/com/amazon/carbonado/qe/SortedQueryExecutor.java index 7b12e43..cae03d8 100644 --- a/src/main/java/com/amazon/carbonado/qe/SortedQueryExecutor.java +++ b/src/main/java/com/amazon/carbonado/qe/SortedQueryExecutor.java @@ -117,10 +117,10 @@ public class SortedQueryExecutor extends AbstractQueryExecut throws IOException { indent(app, indentLevel); - if (mHandledOrdering.size() == 0) { - app.append("full sort: "); - } else { - app.append("finish sort: "); + app.append("sort: "); + if (mHandledOrdering.size() > 0) { + app.append(mHandledOrdering.toString()); + app.append(", "); } app.append(mRemainderOrdering.toString()); newline(app); diff --git a/src/main/java/com/amazon/carbonado/qe/StorageAccess.java b/src/main/java/com/amazon/carbonado/qe/StorageAccess.java index 5f93952..2ab2313 100644 --- a/src/main/java/com/amazon/carbonado/qe/StorageAccess.java +++ b/src/main/java/com/amazon/carbonado/qe/StorageAccess.java @@ -37,7 +37,6 @@ import com.amazon.carbonado.info.StorableIndex; */ public interface StorageAccess extends FullScanQueryExecutor.Support, - FullScanIndexedQueryExecutor.Support, KeyQueryExecutor.Support, IndexedQueryExecutor.Support, SortedQueryExecutor.Support diff --git a/src/main/java/com/amazon/carbonado/qe/UnionQueryAnalyzer.java b/src/main/java/com/amazon/carbonado/qe/UnionQueryAnalyzer.java index e0d49c2..5933f6c 100644 --- a/src/main/java/com/amazon/carbonado/qe/UnionQueryAnalyzer.java +++ b/src/main/java/com/amazon/carbonado/qe/UnionQueryAnalyzer.java @@ -80,8 +80,10 @@ public class UnionQueryAnalyzer implements QueryExecutorFact * @param filter optional filter which must be {@link Filter#isBound bound} * @param ordering optional properties which define desired ordering */ - public Result analyze(Filter filter, OrderingList ordering) { - if (!filter.isBound()) { + public Result analyze(Filter filter, OrderingList ordering) + throws SupportException, RepositoryException + { + if (filter != null && !filter.isBound()) { throw new IllegalArgumentException("Filter must be bound"); } @@ -89,7 +91,7 @@ public class UnionQueryAnalyzer implements QueryExecutorFact ordering = OrderingList.emptyList(); } - return new Result(buildSubResults(filter, ordering)); + return buildResult(filter, ordering); } /** @@ -108,14 +110,19 @@ public class UnionQueryAnalyzer implements QueryExecutorFact * Splits the filter into sub-results, merges sub-results, and possibly * imposes a total ordering. */ - private List.Result> - buildSubResults(Filter filter, OrderingList ordering) + private Result buildResult(Filter filter, OrderingList ordering) + throws SupportException, RepositoryException { - List.Result> subResults = splitIntoSubResults(filter, ordering); + List.Result> subResults; + if (filter == null) { + subResults = Collections.singletonList(mIndexAnalyzer.analyze(filter, ordering)); + } else { + subResults = splitIntoSubResults(filter, ordering); + } if (subResults.size() <= 1) { // Total ordering not required. - return subResults; + return new Result(subResults); } // If any orderings have an unspecified direction, switch to ASCENDING @@ -141,7 +148,7 @@ public class UnionQueryAnalyzer implements QueryExecutorFact if (subResults.size() <= 1) { // Total ordering no longer required. - return subResults; + return new Result(subResults); } } @@ -155,7 +162,7 @@ public class UnionQueryAnalyzer implements QueryExecutorFact ChainedProperty property = op.getChainedProperty(); if (pruneKeys(keys, property)) { // Found a key which is fully covered, indicating total ordering. - return subResults; + return new Result(subResults, ordering); } } @@ -224,7 +231,7 @@ public class UnionQueryAnalyzer implements QueryExecutorFact } } - return subResults; + return new Result(subResults, ordering); } /** @@ -272,7 +279,7 @@ public class UnionQueryAnalyzer implements QueryExecutorFact private Tally bestTally(Iterable tallies) { Tally best = null; for (Tally tally : tallies) { - if (best == null || tally.compareTo(best) < 0) { + if (best == null || tally.compareTo(best) > 0) { best = tally; } } @@ -298,12 +305,16 @@ public class UnionQueryAnalyzer implements QueryExecutorFact */ private List.Result> splitIntoSubResults(Filter filter, OrderingList ordering) + throws SupportException, RepositoryException { // Required for split to work. Filter dnfFilter = filter.disjunctiveNormalForm(); Splitter splitter = new Splitter(ordering); - dnfFilter.accept(splitter, null); + RepositoryException e = dnfFilter.accept(splitter, null); + if (e != null) { + throw e; + } List.Result> subResults = splitter.mSubResults; @@ -373,7 +384,9 @@ public class UnionQueryAnalyzer implements QueryExecutorFact return mergedResults; } - Storage storageDelegate(IndexedQueryAnalyzer.Result result) { + Storage storageDelegate(IndexedQueryAnalyzer.Result result) + throws SupportException, RepositoryException + { StorableIndex localIndex = result.getLocalIndex(); StorageAccess localAccess = mRepoAccess.storageAccessFor(getStorableType()); @@ -389,12 +402,18 @@ public class UnionQueryAnalyzer implements QueryExecutorFact public class Result { private final List.Result> mSubResults; + private final OrderingList mTotalOrdering; Result(List.Result> subResults) { + this(subResults, null); + } + + Result(List.Result> subResults, OrderingList totalOrdering) { if (subResults.size() < 1) { throw new IllegalArgumentException(); } mSubResults = Collections.unmodifiableList(subResults); + mTotalOrdering = totalOrdering; } /** @@ -405,6 +424,13 @@ public class UnionQueryAnalyzer implements QueryExecutorFact return mSubResults; } + /** + * Returns a total ordering, if one was imposed. Otherwise, null is returned. + */ + public OrderingList getTotalOrdering() { + return mTotalOrdering; + } + /** * Creates a QueryExecutor based on this result. */ @@ -423,7 +449,7 @@ public class UnionQueryAnalyzer implements QueryExecutorFact executors.add(subResults.get(i).createExecutor()); } - return new UnionQueryExecutor(executors); + return new UnionQueryExecutor(executors, mTotalOrdering); } } @@ -484,7 +510,7 @@ public class UnionQueryAnalyzer implements QueryExecutorFact } /** - * Returns -1 if this tally is better. + * Returns -1 if this tally is worse. */ public int compareTo(Tally other) { int thisBest = getBestCount(); @@ -497,13 +523,20 @@ public class UnionQueryAnalyzer implements QueryExecutorFact } return 0; } + + public String toString() { + return "Tally: {property=" + mProperty + + ", asc=" + mAscendingCount + + ", desc=" + mDescendingCount + + '}'; + } } /** * Analyzes a disjunctive normal filter into sub-results over filters that * only contain 'and' operations. */ - private class Splitter extends Visitor { + private class Splitter extends Visitor { private final OrderingList mOrdering; final List.Result> mSubResults; @@ -514,37 +547,55 @@ public class UnionQueryAnalyzer implements QueryExecutorFact } @Override - public Object visit(OrFilter filter, Object param) { - Filter left = filter.getLeftFilter(); - if (!(left instanceof OrFilter)) { - subAnalyze(left); - } else { - left.accept(this, param); - } - Filter right = filter.getRightFilter(); - if (!(right instanceof OrFilter)) { - subAnalyze(right); - } else { - right.accept(this, param); + public RepositoryException visit(OrFilter filter, Object param) { + try { + Filter left = filter.getLeftFilter(); + if (!(left instanceof OrFilter)) { + subAnalyze(left); + } else { + RepositoryException e = left.accept(this, param); + if (e != null) { + return e; + } + } + Filter right = filter.getRightFilter(); + if (!(right instanceof OrFilter)) { + subAnalyze(right); + } else { + RepositoryException e = right.accept(this, param); + if (e != null) { + return e; + } + } + return null; + } catch (RepositoryException e) { + return e; } - return null; } // This method should only be called if root filter has no 'or' operators. @Override - public Object visit(AndFilter filter, Object param) { - subAnalyze(filter); - return null; + public RepositoryException visit(AndFilter filter, Object param) { + try { + subAnalyze(filter); + return null; + } catch (RepositoryException e) { + return e; + } } // This method should only be called if root filter has no logical operators. @Override - public Object visit(PropertyFilter filter, Object param) { - subAnalyze(filter); - return null; + public RepositoryException visit(PropertyFilter filter, Object param) { + try { + subAnalyze(filter); + return null; + } catch (RepositoryException e) { + return e; + } } - private void subAnalyze(Filter subFilter) { + private void subAnalyze(Filter subFilter) throws SupportException, RepositoryException { IndexedQueryAnalyzer.Result subResult = mIndexAnalyzer.analyze(subFilter, mOrdering); diff --git a/src/main/java/com/amazon/carbonado/qe/UnionQueryExecutor.java b/src/main/java/com/amazon/carbonado/qe/UnionQueryExecutor.java index 5d2a5c6..47672d1 100644 --- a/src/main/java/com/amazon/carbonado/qe/UnionQueryExecutor.java +++ b/src/main/java/com/amazon/carbonado/qe/UnionQueryExecutor.java @@ -64,19 +64,34 @@ public class UnionQueryExecutor extends AbstractQueryExecuto /** * @param executors executors to wrap, each must have the exact same total ordering - * @throws IllegalArgumentException if any parameter is null or if ordering doesn't match + * @throws IllegalArgumentException if any executors is null or if ordering doesn't match */ public UnionQueryExecutor(List> executors) { + this(executors, null); + } + + /** + * @param executors executors to wrap, each must have the exact same total ordering + * @param totalOrdering effective total ordering of executors + * @throws IllegalArgumentException if executors is null + */ + public UnionQueryExecutor(List> executors, OrderingList totalOrdering) { if (executors == null || executors.size() == 0) { throw new IllegalArgumentException(); } - OrderingList totalOrdering = executors.get(0).getOrdering(); - // Compare for consistency. - for (int i=1; i