diff options
| author | Jesse Morgan <jesse@jesterpm.net> | 2014-05-23 22:40:39 -0700 | 
|---|---|---|
| committer | Jesse Morgan <jesse@jesterpm.net> | 2014-05-23 22:40:39 -0700 | 
| commit | 347589a78ba6db05bb4fe62ad92a0b73e4a57c3e (patch) | |
| tree | 9d6c9db0d4b5f7c222857620978e5b16588f51a5 | |
| parent | 255b10334bcf190cf56c0d57791e39d5e808497b (diff) | |
Adding dynamo support.
| -rw-r--r-- | ivy.xml | 1 | ||||
| -rw-r--r-- | src/com/p4square/grow/GrowProcessComponent.java | 23 | ||||
| -rw-r--r-- | src/com/p4square/grow/backend/DynamoGrowData.java | 196 | ||||
| -rw-r--r-- | src/com/p4square/grow/backend/GrowBackend.java | 2 | ||||
| -rw-r--r-- | src/com/p4square/grow/backend/db/CassandraProviderImpl.java | 10 | ||||
| -rw-r--r-- | src/com/p4square/grow/backend/dynamo/DbTool.java | 341 | ||||
| -rw-r--r-- | src/com/p4square/grow/backend/dynamo/DynamoCollectionProviderImpl.java | 109 | ||||
| -rw-r--r-- | src/com/p4square/grow/backend/dynamo/DynamoDatabase.java | 235 | ||||
| -rw-r--r-- | src/com/p4square/grow/backend/dynamo/DynamoKey.java | 56 | ||||
| -rw-r--r-- | src/com/p4square/grow/backend/dynamo/DynamoProviderImpl.java | 37 | ||||
| -rw-r--r-- | src/com/p4square/grow/frontend/SurveyPageResource.java | 9 | ||||
| -rw-r--r-- | src/com/p4square/grow/provider/DelegateCollectionProvider.java | 4 | ||||
| -rw-r--r-- | src/com/p4square/grow/provider/JsonEncodedProvider.java | 8 | 
13 files changed, 1007 insertions, 24 deletions
@@ -31,5 +31,6 @@          <dependency org="com.netflix.astyanax" name="astyanax-core" rev="[1.0,)" conf="default" />          <dependency org="com.netflix.astyanax" name="astyanax-thrift" rev="[1.0,)" conf="default" />          <dependency org="com.netflix.astyanax" name="astyanax-cassandra" rev="[1.0,)" conf="default" /> +        <dependency org="com.amazonaws" name="aws-java-sdk" rev="[1.7.8,)" conf="default" />      </dependencies>  </ivy-module> diff --git a/src/com/p4square/grow/GrowProcessComponent.java b/src/com/p4square/grow/GrowProcessComponent.java index 29da766..7d0938e 100644 --- a/src/com/p4square/grow/GrowProcessComponent.java +++ b/src/com/p4square/grow/GrowProcessComponent.java @@ -38,13 +38,17 @@ public class GrowProcessComponent extends Component {       * Create a new Grow Process website component combining a frontend and backend.       */      public GrowProcessComponent() throws Exception { +        this(new Config()); +    } + +    public GrowProcessComponent(Config config) {          // Clients          getClients().add(Protocol.FILE);          getClients().add(Protocol.HTTP);          getClients().add(Protocol.HTTPS);          // Prepare mConfig -        mConfig = new Config(); +        mConfig = config;          // Frontend          GrowFrontend frontend = new GrowFrontend(mConfig); @@ -62,6 +66,7 @@ public class GrowProcessComponent extends Component {          getDefaultHost().attach("/backend", auth);      } +      @Override      public void start() throws Exception {          // Load mConfigs @@ -84,10 +89,16 @@ public class GrowProcessComponent extends Component {       * Stand-alone main for testing.       */      public static void main(String[] args) throws Exception { +        // Load an optional config file from the first argument. +        Config config = new Config(); +        config.setDomain("dev"); +        if (args.length == 1) { +            config.updateConfig(args[0]); +        } +          // Start the HTTP Server -        final GrowProcessComponent component = new GrowProcessComponent(); +        final GrowProcessComponent component = new GrowProcessComponent(config);          component.getServers().add(Protocol.HTTP, 8085); -        //component.getClients().add(new Client(null, Arrays.asList(Protocol.HTTPS), "org.restlet.ext.httpclient.HttpClientHelper"));          // Static content          try { @@ -102,12 +113,6 @@ public class GrowProcessComponent extends Component {                      + e.getMessage(), e);          } -        // Load an optional config file from the first argument. -        component.mConfig.setDomain("dev"); -        if (args.length == 1) { -            component.mConfig.updateConfig(args[0]); -        } -          // Setup shutdown hook          Runtime.getRuntime().addShutdownHook(new Thread() {              public void run() { diff --git a/src/com/p4square/grow/backend/DynamoGrowData.java b/src/com/p4square/grow/backend/DynamoGrowData.java new file mode 100644 index 0000000..4123999 --- /dev/null +++ b/src/com/p4square/grow/backend/DynamoGrowData.java @@ -0,0 +1,196 @@ +/* + * Copyright 2014 Jesse Morgan + */ + +package com.p4square.grow.backend; + +import java.io.IOException; + +import com.amazonaws.auth.AWSCredentials; + +import com.p4square.grow.backend.dynamo.DynamoDatabase; +import com.p4square.grow.backend.dynamo.DynamoKey; +import com.p4square.grow.backend.dynamo.DynamoProviderImpl; +import com.p4square.grow.backend.dynamo.DynamoCollectionProviderImpl; + +import com.p4square.grow.config.Config; + +import com.p4square.grow.model.Message; +import com.p4square.grow.model.MessageThread; +import com.p4square.grow.model.Playlist; +import com.p4square.grow.model.Question; +import com.p4square.grow.model.TrainingRecord; +import com.p4square.grow.model.UserRecord; + +import com.p4square.grow.provider.CollectionProvider; +import com.p4square.grow.provider.DelegateCollectionProvider; +import com.p4square.grow.provider.DelegateProvider; +import com.p4square.grow.provider.Provider; +import com.p4square.grow.provider.JsonEncodedProvider; + +/** + *  + * @author Jesse Morgan <jesse@jesterpm.net> + */ +class DynamoGrowData implements GrowData { +    private static final String DEFAULT_COLUMN = "value"; +    private static final String DEFAULT_PLAYLIST_KEY = "/training/defaultplaylist"; + +    private final Config mConfig; +    private final DynamoDatabase mDatabase; + +    private final Provider<String, UserRecord> mUserRecordProvider; + +    private final Provider<String, Question> mQuestionProvider; +    private final Provider<String, TrainingRecord> mTrainingRecordProvider; +    private final CollectionProvider<String, String, String> mVideoProvider; + +    private final CollectionProvider<String, String, MessageThread> mFeedThreadProvider; +    private final CollectionProvider<String, String, Message> mFeedMessageProvider; + +    private final Provider<String, String> mStringProvider; + +    private final CollectionProvider<String, String, String> mAnswerProvider; + +    public DynamoGrowData(final Config config) { +        mConfig = config; + +        AWSCredentials creds = new AWSCredentials() { +            @Override +            public String getAWSAccessKeyId() { +                return config.getString("awsAccessKey"); +            } +            @Override +            public String getAWSSecretKey() { +                return config.getString("awsSecretKey"); +            } +        }; + +        String endpoint = config.getString("dynamoEndpoint"); +        if (endpoint != null) { +            mDatabase = new DynamoDatabase(creds, endpoint); +        } else { +            mDatabase = new DynamoDatabase(creds); +        } + +        mUserRecordProvider = new DelegateProvider<String, DynamoKey, UserRecord>( +                new DynamoProviderImpl<UserRecord>(mDatabase, UserRecord.class)) { +            @Override +            public DynamoKey makeKey(String userid) { +                return DynamoKey.newAttributeKey("accounts", userid, DEFAULT_COLUMN); +            } +        }; + +        mQuestionProvider = new DelegateProvider<String, DynamoKey, Question>( +                new DynamoProviderImpl<Question>(mDatabase, Question.class)) { +            @Override +            public DynamoKey makeKey(String questionId) { +                return DynamoKey.newAttributeKey("strings", +                                                 "/questions/" + questionId, +                                                 DEFAULT_COLUMN); +            } +        }; + +        mFeedThreadProvider = new DynamoCollectionProviderImpl<MessageThread>( +                mDatabase, "feedthreads", MessageThread.class); +        mFeedMessageProvider = new DynamoCollectionProviderImpl<Message>( +                mDatabase, "feedmessages", Message.class); + +        mTrainingRecordProvider = new DelegateProvider<String, DynamoKey, TrainingRecord>( +                new DynamoProviderImpl<TrainingRecord>(mDatabase, TrainingRecord.class)) { +            @Override +            public DynamoKey makeKey(String userId) { +                return DynamoKey.newAttributeKey("training", +                                                 userId, +                                                 DEFAULT_COLUMN); +            } +        }; + +        mVideoProvider = new DelegateCollectionProvider<String, String, String, String, String>( +                new DynamoCollectionProviderImpl<String>(mDatabase, "strings", String.class)) { +            @Override +            public String makeCollectionKey(String key) { +                return "/training/" + key; +            } + +            @Override +            public String makeKey(String key) { +                return key; +            } + +            @Override +            public String unmakeKey(String key) { +                return key; +            } +        }; + +        mStringProvider = new DelegateProvider<String, DynamoKey, String>( +                new DynamoProviderImpl<String>(mDatabase, String.class)) { +            @Override +            public DynamoKey makeKey(String id) { +                return DynamoKey.newAttributeKey("strings", id, DEFAULT_COLUMN); +            } +        }; + +        mAnswerProvider = new DynamoCollectionProviderImpl<String>( +                mDatabase, "assessments", String.class); +    } + +    @Override +    public void start() throws Exception { +    } + +    @Override +    public void stop() throws Exception { +    } + +    @Override +    public Provider<String, UserRecord> getUserRecordProvider() { +        return mUserRecordProvider; +    } + +    @Override +    public Provider<String, Question> getQuestionProvider() { +        return mQuestionProvider; +    } + +    @Override +    public Provider<String, TrainingRecord> getTrainingRecordProvider() { +        return mTrainingRecordProvider; +    } + +    @Override +    public CollectionProvider<String, String, String> getVideoProvider() { +        return mVideoProvider; +    } + +    @Override +    public Playlist getDefaultPlaylist() throws IOException { +        String blob = mStringProvider.get(DEFAULT_PLAYLIST_KEY); +        if (blob == null) { +            return null; +        } + +        return JsonEncodedProvider.MAPPER.readValue(blob, Playlist.class); +    } + +    @Override +    public CollectionProvider<String, String, MessageThread> getThreadProvider() { +        return mFeedThreadProvider; +    } + +    @Override +    public CollectionProvider<String, String, Message> getMessageProvider() { +        return mFeedMessageProvider; +    } + +    @Override +    public Provider<String, String> getStringProvider() { +        return mStringProvider; +    } + +    @Override +    public CollectionProvider<String, String, String> getAnswerProvider() { +        return mAnswerProvider; +    } +} diff --git a/src/com/p4square/grow/backend/GrowBackend.java b/src/com/p4square/grow/backend/GrowBackend.java index 49d064c..e73ad38 100644 --- a/src/com/p4square/grow/backend/GrowBackend.java +++ b/src/com/p4square/grow/backend/GrowBackend.java @@ -61,7 +61,7 @@ public class GrowBackend extends Application implements GrowData {      public GrowBackend(Config config) {          mConfig = config; -        mGrowData = new CassandraGrowData(config); +        mGrowData = new DynamoGrowData(config);      }      @Override diff --git a/src/com/p4square/grow/backend/db/CassandraProviderImpl.java b/src/com/p4square/grow/backend/db/CassandraProviderImpl.java index 9d896e7..da5a9f2 100644 --- a/src/com/p4square/grow/backend/db/CassandraProviderImpl.java +++ b/src/com/p4square/grow/backend/db/CassandraProviderImpl.java @@ -26,20 +26,12 @@ public class CassandraProviderImpl<V> extends JsonEncodedProvider<V> implements      @Override      public V get(CassandraKey key) throws IOException {          String blob = mDb.getKey(key.getColumnFamily(), key.getId(), key.getColumn()); -        if (mClazz == String.class) { -            return (V) blob; -        }          return decode(blob);      }      @Override      public void put(CassandraKey key, V obj) throws IOException { -        String blob; -        if (mClazz == String.class) { -            blob = (String) obj; -        } else { -            blob = encode(obj); -        } +        String blob = encode(obj);          mDb.putKey(key.getColumnFamily(), key.getId(), key.getColumn(), blob);      }  } diff --git a/src/com/p4square/grow/backend/dynamo/DbTool.java b/src/com/p4square/grow/backend/dynamo/DbTool.java new file mode 100644 index 0000000..5784f3e --- /dev/null +++ b/src/com/p4square/grow/backend/dynamo/DbTool.java @@ -0,0 +1,341 @@ +/* + * Copyright 2014 Jesse Morgan + */ + +package com.p4square.grow.backend.dynamo; + +import java.util.Arrays; +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + +import com.amazonaws.auth.AWSCredentials; + +import com.p4square.grow.backend.dynamo.DynamoDatabase; +import com.p4square.grow.backend.dynamo.DynamoKey; +import com.p4square.grow.config.Config; +import com.p4square.grow.model.UserRecord; +import com.p4square.grow.provider.Provider; + +/** + * + * @author Jesse Morgan <jesse@jesterpm.net> + */ +public class DbTool { +    private static final FilenameFilter JSON_FILTER = new JsonFilter(); + +    private static Config mConfig; +    private static DynamoDatabase mDatabase; + +    public static void usage() { +        System.out.println("java com.p4square.grow.backend.dynamo.DbTool <command>...\n"); +        System.out.println("Commands:"); +        System.out.println("\t--domain <domain>                           Set config domain"); +        System.out.println("\t--dev                                       Set config domain to dev"); +        System.out.println("\t--config <file>                             Merge in config file"); +        System.out.println("\t--list                                      List all tables"); +        System.out.println("\t--create <table> <reads> <writes>           Create a table"); +        System.out.println("\t--update <table> <reads> <writes>           Update table throughput"); +        System.out.println("\t--drop   <table>                            Delete a table"); +        System.out.println("\t--get    <table> <key> <attribute>          Get a value"); +        System.out.println("\t--put    <table> <key> <attribute> <value>  Put a value"); +        System.out.println(); +        System.out.println("Bootstrap Commands:"); +        System.out.println("\t--bootstrap <data>          Create all tables and import all data"); +        System.out.println("\t--loadStrings <data>        Load all videos and questions"); +        System.out.println("\t--destroy                   Drop all tables"); +        System.out.println("\t--addadmin <user> <pass>    Add a backend account"); +    } + +    public static void main(String... args) { +        if (args.length == 0) { +            usage(); +            System.exit(1); +        } + +        mConfig = new Config(); + +        try { +            int offset = 0; +            while (offset < args.length) { +                if ("--domain".equals(args[offset])) { +                    mConfig.setDomain(args[offset + 1]); +                    mDatabase = null; +                    offset += 2; + +                } else if ("--dev".equals(args[offset])) { +                    mConfig.setDomain("dev"); +                    mDatabase = null; +                    offset += 1; + +                } else if ("--config".equals(args[offset])) { +                    mConfig.updateConfig(args[offset + 1]); +                    mDatabase = null; +                    offset += 2; + +                } else if ("--list".equals(args[offset])) { +                    //offset = list(args, ++offset); + +                } else if ("--create".equals(args[offset])) { +                    offset = create(args, ++offset); + +                } else if ("--update".equals(args[offset])) { +                    offset = update(args, ++offset); + +                } else if ("--drop".equals(args[offset])) { +                    offset = drop(args, ++offset); + +                } else if ("--get".equals(args[offset])) { +                    offset = get(args, ++offset); + +                } else if ("--put".equals(args[offset])) { +                    offset = put(args, ++offset); + +                /* Bootstrap Commands */ +                } else if ("--bootstrap".equals(args[offset])) { +                    offset = bootstrapTables(args, ++offset); +                    offset = loadStrings(args, offset); + +                } else if ("--loadStrings".equals(args[offset])) { +                    offset = loadStrings(args, ++offset); + +                } else if ("--destroy".equals(args[offset])) { +                    offset = destroy(args, ++offset); + +                } else if ("--addadmin".equals(args[offset])) { +                    offset = addAdmin(args, ++offset); + +                } else { +                    throw new IllegalArgumentException("Unknown command " + args[offset]); +                } +            } +        } catch (Exception e) { +            e.printStackTrace(); +            System.exit(2); +        } +    } + +    private static DynamoDatabase getDatabase() { +        if (mDatabase == null) { +            AWSCredentials creds = new AWSCredentials() { +                @Override +                public String getAWSAccessKeyId() { +                    return mConfig.getString("awsAccessKey"); +                } +                @Override +                public String getAWSSecretKey() { +                    return mConfig.getString("awsSecretKey"); +                } +            }; + +            String endpoint = mConfig.getString("dynamoEndpoint"); +            if (endpoint != null) { +                mDatabase = new DynamoDatabase(creds, endpoint); +            } else { +                mDatabase = new DynamoDatabase(creds); +            } +        } + +        return mDatabase; +    } + +    private static int create(String[] args, int offset) { +        String name = args[offset++]; +        long reads  = Long.parseLong(args[offset++]); +        long writes = Long.parseLong(args[offset++]); + +        DynamoDatabase db = getDatabase(); + +        db.createTable(name, reads, writes); + +        return offset; +    } + +    private static int update(String[] args, int offset) { +        String name = args[offset++]; +        long reads  = Long.parseLong(args[offset++]); +        long writes = Long.parseLong(args[offset++]); + +        DynamoDatabase db = getDatabase(); + +        db.updateTable(name, reads, writes); + +        return offset; +    } + +    private static int drop(String[] args, int offset) { +        String name = args[offset++]; + +        DynamoDatabase db = getDatabase(); + +        db.deleteTable(name); + +        return offset; +    } + +    private static int get(String[] args, int offset) { +        String table     = args[offset++]; +        String key       = args[offset++]; +        String attribute = args[offset++]; + +        DynamoDatabase db = getDatabase(); + +        String value = db.getAttribute(DynamoKey.newAttributeKey(table, key, attribute)); + +        if (value == null) { +            value = "<null>"; +        } + +        System.out.printf("%s %s:%s\n%s\n\n", table, key, attribute, value); + +        return offset; +    } + +    private static int put(String[] args, int offset) { +        String table     = args[offset++]; +        String key       = args[offset++]; +        String attribute = args[offset++]; +        String value     = args[offset++]; + +        DynamoDatabase db = getDatabase(); + +        db.putAttribute(DynamoKey.newAttributeKey(table, key, attribute), value); + +        return offset; +    } + +    private static int bootstrapTables(String[] args, int offset) { +        DynamoDatabase db = getDatabase(); + +        db.createTable("strings",      10,  1); +        db.createTable("accounts",     10,  1); +        db.createTable("assessments",  10,  5); +        db.createTable("training",     10,  5); +        db.createTable("feedthreads",  10,  1); +        db.createTable("feedmessages", 10,  1); + +        return offset; +    } + +    private static int loadStrings(String[] args, int offset) throws IOException { +        String data = args[offset++]; +        File baseDir = new File(data); + +        DynamoDatabase db = getDatabase(); + +        insertQuestions(baseDir); +        insertVideos(baseDir); +        insertDefaultPlaylist(baseDir); + +        return offset; +    } + +    private static int destroy(String[] args, int offset) { +        DynamoDatabase db = getDatabase(); + +        final String[] tables = { "strings", +                                  "accounts", +                                  "assessments", +                                  "training", +                                  "feedthreads", +                                  "feedmessages" +                                }; + +        for (String table : tables) { +            try { +                db.deleteTable(table); +            } catch (Exception e) { +                System.err.println("Deleting " + table + ": " + e.getMessage()); +            } +        } + +        return offset; +    } + +    private static int addAdmin(String[] args, int offset) throws IOException { +        String user = args[offset++]; +        String pass = args[offset++]; + +        DynamoDatabase db = getDatabase(); + +        UserRecord record = new UserRecord(); +        record.setId(user); +        record.setBackendPassword(pass); + +        Provider<DynamoKey, UserRecord> provider = new DynamoProviderImpl(db, UserRecord.class); +        provider.put(DynamoKey.newAttributeKey("accounts", user, "value"), record); + +        return offset; +    } + +    private static void insertQuestions(File baseDir) throws IOException { +        DynamoDatabase db = getDatabase(); +        File questions = new File(baseDir, "questions"); + +        File[] files = questions.listFiles(JSON_FILTER); +        Arrays.sort(files); + +        for (File file : files) { +            String filename = file.getName(); +            String questionId = filename.substring(0, filename.lastIndexOf('.')); + +            byte[] encoded = Files.readAllBytes(file.toPath()); +            String value = new String(encoded, StandardCharsets.UTF_8); +            db.putAttribute(DynamoKey.newAttributeKey("strings", +                        "/questions/" + questionId, "value"), value); +            System.out.println("Inserted /questions/" + questionId); +        } + +        String filename = files[0].getName(); +        String first    = filename.substring(0, filename.lastIndexOf('.')); +        int    count    = files.length; +        String summary  = "{\"first\": \"" + first + "\", \"count\": " + count + "}"; +        db.putAttribute(DynamoKey.newAttributeKey("strings", "/questions", "value"), summary); +        System.out.println("Inserted /questions"); +    } + +    private static void insertVideos(File baseDir) throws IOException { +        DynamoDatabase db = getDatabase(); +        File videos = new File(baseDir, "videos"); + +        for (File topic : videos.listFiles()) { +            if (!topic.isDirectory()) { +                continue; +            } + +            String topicName = topic.getName(); + +            File[] files = topic.listFiles(JSON_FILTER); +            for (File file : files) { +                String filename = file.getName(); +                String videoId = filename.substring(0, filename.lastIndexOf('.')); + +                byte[] encoded = Files.readAllBytes(file.toPath()); +                String value = new String(encoded, StandardCharsets.UTF_8); +                db.putAttribute(DynamoKey.newAttributeKey("strings", +                            "/training/" + topicName, videoId), value); +                System.out.println("Inserted /training/" + topicName + ":" + videoId); +            } +        } +    } + +    private static void insertDefaultPlaylist(File baseDir) throws IOException { +        DynamoDatabase db = getDatabase(); +        File file = new File(baseDir, "videos/playlist.json"); + +        byte[] encoded = Files.readAllBytes(file.toPath()); +        String value = new String(encoded, StandardCharsets.UTF_8); +        db.putAttribute(DynamoKey.newAttributeKey("strings", +                    "/training/defaultplaylist", "value"), value); +        System.out.println("Inserted /training/defaultplaylist"); +    } + +    private static class JsonFilter implements FilenameFilter { +        @Override +        public boolean accept(File dir, String name) { +            return name.endsWith(".json"); +        } +    } +} diff --git a/src/com/p4square/grow/backend/dynamo/DynamoCollectionProviderImpl.java b/src/com/p4square/grow/backend/dynamo/DynamoCollectionProviderImpl.java new file mode 100644 index 0000000..b53e9f7 --- /dev/null +++ b/src/com/p4square/grow/backend/dynamo/DynamoCollectionProviderImpl.java @@ -0,0 +1,109 @@ +/* + * Copyright 2014 Jesse Morgan + */ + +package com.p4square.grow.backend.dynamo; + +import java.io.IOException; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import com.p4square.grow.provider.CollectionProvider; +import com.p4square.grow.provider.JsonEncodedProvider; + +/** + *  + * @author Jesse Morgan <jesse@jesterpm.net> + */ +public class DynamoCollectionProviderImpl<V> implements CollectionProvider<String, String, V> { +    private final DynamoDatabase mDb; +    private final String mTable; +    private final Class<V> mClazz; + +    public DynamoCollectionProviderImpl(DynamoDatabase db, String table, Class<V> clazz) { +        mDb = db; +        mTable = table; +        mClazz = clazz; +    } + +    @Override +    public V get(String collection, String key) throws IOException { +        String blob = mDb.getAttribute(DynamoKey.newAttributeKey(mTable, collection, key)); +        return decode(blob); +    } + +    @Override +    public Map<String, V> query(String collection) throws IOException { +        return query(collection, -1); +    } + +    @Override +    public Map<String, V> query(String collection, int limit) throws IOException { +        Map<String, V> result = new LinkedHashMap<>(); + +        Map<String, String> row = mDb.getKey(DynamoKey.newKey(mTable, collection)); +        if (row.size() > 0) { +            int count = 0; +            for (Map.Entry<String, String> c : row.entrySet()) { +                if (limit >= 0 && ++count > limit) { +                    break; // Limit reached. +                } + +                String key = c.getKey(); +                String blob = c.getValue(); +                V obj = decode(blob); + +                result.put(key, obj); +            } +        } + +        return Collections.unmodifiableMap(result); +    } + +    @Override +    public void put(String collection, String key, V obj) throws IOException { +        if (obj == null) { +            mDb.deleteAttribute(DynamoKey.newAttributeKey(mTable, collection, key)); +        } else { +            String blob = encode(obj); +            mDb.putAttribute(DynamoKey.newAttributeKey(mTable, collection, key), blob); +        } +    } + +    /** +     * Encode the object as JSON. +     * +     * @param obj The object to encode. +     * @return The JSON encoding of obj. +     * @throws IOException if the object cannot be encoded. +     */ +    protected String encode(V obj) throws IOException { +        if (mClazz == String.class) { +            return (String) obj; +        } else { +            return JsonEncodedProvider.MAPPER.writeValueAsString(obj); +        } +    } + +    /** +     * Decode the JSON string as an object. +     * +     * @param blob The JSON data to decode. +     * @return The decoded object or null if blob is null. +     * @throws IOException If an object cannot be decoded. +     */ +    protected V decode(String blob) throws IOException { +        if (blob == null) { +            return null; +        } + +        if (mClazz == String.class) { +            return (V) blob; +        } + +        V obj = JsonEncodedProvider.MAPPER.readValue(blob, mClazz); +        return obj; +    } +} diff --git a/src/com/p4square/grow/backend/dynamo/DynamoDatabase.java b/src/com/p4square/grow/backend/dynamo/DynamoDatabase.java new file mode 100644 index 0000000..076844f --- /dev/null +++ b/src/com/p4square/grow/backend/dynamo/DynamoDatabase.java @@ -0,0 +1,235 @@ +/* + * Copyright 2014 Jesse Morgan + */ + +package com.p4square.grow.backend.dynamo; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; +import com.amazonaws.services.dynamodbv2.model.AttributeAction; +import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; +import com.amazonaws.services.dynamodbv2.model.AttributeValue; +import com.amazonaws.services.dynamodbv2.model.AttributeValueUpdate; +import com.amazonaws.services.dynamodbv2.model.CreateTableRequest; +import com.amazonaws.services.dynamodbv2.model.CreateTableResult; +import com.amazonaws.services.dynamodbv2.model.DeleteItemRequest; +import com.amazonaws.services.dynamodbv2.model.DeleteItemResult; +import com.amazonaws.services.dynamodbv2.model.DeleteTableRequest; +import com.amazonaws.services.dynamodbv2.model.DeleteTableResult; +import com.amazonaws.services.dynamodbv2.model.GetItemRequest; +import com.amazonaws.services.dynamodbv2.model.GetItemResult; +import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; +import com.amazonaws.services.dynamodbv2.model.KeyType; +import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput; +import com.amazonaws.services.dynamodbv2.model.PutItemRequest; +import com.amazonaws.services.dynamodbv2.model.PutItemResult; +import com.amazonaws.services.dynamodbv2.model.UpdateItemRequest; +import com.amazonaws.services.dynamodbv2.model.UpdateItemResult; +import com.amazonaws.services.dynamodbv2.model.UpdateTableRequest; +import com.amazonaws.services.dynamodbv2.model.UpdateTableResult; + +/** + * A wrapper around the Dynamo API. + */ +public class DynamoDatabase { +    private final AmazonDynamoDBClient mClient; + +    public DynamoDatabase(AWSCredentials awsCreds) { +        mClient = new AmazonDynamoDBClient(awsCreds); +    } + +    public DynamoDatabase(AWSCredentials awsCreds, String endpoint) { +        this(awsCreds); +        mClient.setEndpoint(endpoint); +    } + +    public void createTable(String name, long reads, long writes) { +        ArrayList<AttributeDefinition> attributeDefinitions = new ArrayList<>(); +        attributeDefinitions.add(new AttributeDefinition() +                .withAttributeName("id") +                .withAttributeType("S")); + +        ArrayList<KeySchemaElement> ks = new ArrayList<>(); +        ks.add(new KeySchemaElement().withAttributeName("id").withKeyType(KeyType.HASH)); + +        ProvisionedThroughput provisionedThroughput = new ProvisionedThroughput() +            .withReadCapacityUnits(reads) +            .withWriteCapacityUnits(writes); + +        CreateTableRequest request = new CreateTableRequest() +            .withTableName(name) +            .withAttributeDefinitions(attributeDefinitions) +            .withKeySchema(ks) +            .withProvisionedThroughput(provisionedThroughput); + +        CreateTableResult result = mClient.createTable(request); +    } + +    public void updateTable(String name, long reads, long writes) { +        ProvisionedThroughput provisionedThroughput = new ProvisionedThroughput() +            .withReadCapacityUnits(reads) +            .withWriteCapacityUnits(writes); + +        UpdateTableRequest request = new UpdateTableRequest() +            .withTableName(name) +            .withProvisionedThroughput(provisionedThroughput); + +        UpdateTableResult result = mClient.updateTable(request); +    } + +    public void deleteTable(String name) { +        DeleteTableRequest deleteTableRequest = new DeleteTableRequest() +            .withTableName(name); + +        DeleteTableResult result = mClient.deleteTable(deleteTableRequest); +    } + +    public Map<String, String> getKey(final DynamoKey key) { +        GetItemRequest getItemRequest = new GetItemRequest() +            .withTableName(key.getTable()) +            .withKey(generateKey(key)); + +        GetItemResult getItemResult = mClient.getItem(getItemRequest); +        Map<String, AttributeValue> map = getItemResult.getItem(); + +        Map<String, String> result = new LinkedHashMap<>(); +        if (map != null) { +            for (Map.Entry<String, AttributeValue> entry : map.entrySet()) { +                if (!"id".equals(entry.getKey())) { +                    result.put(entry.getKey(), entry.getValue().getS()); +                } +            } +        } + +        return result; +    } + +    public String getAttribute(final DynamoKey key) { +        checkAttributeKey(key); + +        GetItemRequest getItemRequest = new GetItemRequest() +            .withTableName(key.getTable()) +            .withKey(generateKey(key)) +            .withAttributesToGet(key.getAttribute()); + +        GetItemResult result = mClient.getItem(getItemRequest); +        Map<String, AttributeValue> map = result.getItem(); + +        if (map == null) { +            return null; +        } + +        AttributeValue value = map.get(key.getAttribute()); +        if (value != null) { +            return value.getS(); + +        } else { +            return null; +        } +    } + +    /** +     * Set all attributes for the given key. +     * +     * @param key The key. +     * @param values Map of attributes to values. +     */ +    public void putKey(final DynamoKey key, final Map<String, String> values) { +        Map<String, AttributeValue> item = new HashMap<>(); +        for (Map.Entry<String, String> entry : values.entrySet()) { +            item.put(entry.getKey(), new AttributeValue().withS(entry.getValue())); +        } + +        // Set the Key +        item.putAll(generateKey(key)); + +        PutItemRequest putItemRequest = new PutItemRequest() +            .withTableName(key.getTable()) +            .withItem(item); + +        PutItemResult result = mClient.putItem(putItemRequest); +    } + +    /** +     * Set the particular attributes of the given key. +     * +     * @param key The key. +     * @param value The new value. +     */ +    public void putAttribute(final DynamoKey key, final String value) { +        checkAttributeKey(key); + +        Map<String, AttributeValueUpdate> updateItem = new HashMap<>(); +        updateItem.put(key.getAttribute(), +                new AttributeValueUpdate() +                .withAction(AttributeAction.PUT) +                .withValue(new AttributeValue().withS(value))); + +        UpdateItemRequest updateItemRequest = new UpdateItemRequest() +            .withTableName(key.getTable()) +            .withKey(generateKey(key)) +            .withAttributeUpdates(updateItem); +        // TODO: Check conditions. + +        UpdateItemResult result = mClient.updateItem(updateItemRequest); +    } + +    /** +     * Delete the given key. +     * +     * @param key The key. +     */ +    public void deleteKey(final DynamoKey key) { +        DeleteItemRequest deleteItemRequest = new DeleteItemRequest() +            .withTableName(key.getTable()) +            .withKey(generateKey(key)); + +        DeleteItemResult result = mClient.deleteItem(deleteItemRequest); +    } + +    /** +     * Delete an attribute from the given key. +     * +     * @param key The key. +     */ +    public void deleteAttribute(final DynamoKey key) { +        checkAttributeKey(key); + +        Map<String, AttributeValueUpdate> updateItem = new HashMap<>(); +        updateItem.put(key.getAttribute(), +                new AttributeValueUpdate().withAction(AttributeAction.DELETE)); + +        UpdateItemRequest updateItemRequest = new UpdateItemRequest() +            .withTableName(key.getTable()) +            .withKey(generateKey(key)) +            .withAttributeUpdates(updateItem); + +        UpdateItemResult result = mClient.updateItem(updateItemRequest); +    } + +    /** +     * Generate a DynamoDB Key Map from the DynamoKey. +     */ +    private Map<String, AttributeValue> generateKey(final DynamoKey key) { +        HashMap<String, AttributeValue> keyMap = new HashMap<>(); +        keyMap.put("id", new AttributeValue().withS(key.getHashKey())); + +        String range = key.getRangeKey(); +        if (range != null) { +            keyMap.put("range", new AttributeValue().withS(range)); +        } + +        return keyMap; +    } + +    private void checkAttributeKey(DynamoKey key) { +        if (null == key.getAttribute()) { +            throw new IllegalArgumentException("Attribute must be non-null"); +        } +    } +} diff --git a/src/com/p4square/grow/backend/dynamo/DynamoKey.java b/src/com/p4square/grow/backend/dynamo/DynamoKey.java new file mode 100644 index 0000000..5cdbacd --- /dev/null +++ b/src/com/p4square/grow/backend/dynamo/DynamoKey.java @@ -0,0 +1,56 @@ +/* + * Copyright 2014 Jesse Morgan + */ + +package com.p4square.grow.backend.dynamo; + +/** + * DynamoKey represents a table, hash key, and range key tupl. + */ +public class DynamoKey { +    private final String mTable; +    private final String mHashKey; +    private final String mRangeKey; +    private final String mAttribute; + +    public static DynamoKey newKey(final String table, final String hashKey) { +        return new DynamoKey(table, hashKey, null, null); +    } + +    public static DynamoKey newRangeKey(final String table, final String hashKey, +            final String rangeKey) { + +        return new DynamoKey(table, hashKey, rangeKey, null); +    } + +    public static DynamoKey newAttributeKey(final String table, final String hashKey, +            final String attribute) { + +        return new DynamoKey(table, hashKey, null, attribute); +    } + +    public DynamoKey(final String table, final String hashKey, final String rangeKey, +            final String attribute) { + +        mTable = table; +        mHashKey = hashKey; +        mRangeKey = rangeKey; +        mAttribute = attribute; +    } + +    public String getTable() { +        return mTable; +    } + +    public String getHashKey() { +        return mHashKey; +    } + +    public String getRangeKey() { +        return mRangeKey; +    } + +    public String getAttribute() { +        return mAttribute; +    } +} diff --git a/src/com/p4square/grow/backend/dynamo/DynamoProviderImpl.java b/src/com/p4square/grow/backend/dynamo/DynamoProviderImpl.java new file mode 100644 index 0000000..93a535f --- /dev/null +++ b/src/com/p4square/grow/backend/dynamo/DynamoProviderImpl.java @@ -0,0 +1,37 @@ +/* + * Copyright 2013 Jesse Morgan + */ + +package com.p4square.grow.backend.dynamo; + +import java.io.IOException; + +import com.p4square.grow.provider.Provider; +import com.p4square.grow.provider.JsonEncodedProvider; + +/** + * Provider implementation backed by a DynamoDB Table. + * + * @author Jesse Morgan <jesse@jesterpm.net> + */ +public class DynamoProviderImpl<V> extends JsonEncodedProvider<V> implements Provider<DynamoKey, V> { +    private final DynamoDatabase mDb; + +    public DynamoProviderImpl(DynamoDatabase db, Class<V> clazz) { +        super(clazz); + +        mDb = db; +    } + +    @Override +    public V get(DynamoKey key) throws IOException { +        String blob = mDb.getAttribute(key); +        return decode(blob); +    } + +    @Override +    public void put(DynamoKey key, V obj) throws IOException { +        String blob = encode(obj); +        mDb.putAttribute(key, blob); +    } +} diff --git a/src/com/p4square/grow/frontend/SurveyPageResource.java b/src/com/p4square/grow/frontend/SurveyPageResource.java index 313fb7b..cfc4eed 100644 --- a/src/com/p4square/grow/frontend/SurveyPageResource.java +++ b/src/com/p4square/grow/frontend/SurveyPageResource.java @@ -249,9 +249,12 @@ public class SurveyPageResource extends FreeMarkerPageResource {          if (nextQuestionId == null) {              // Just finished the last question. Update the user's account              try { -                UserRecord account = mUserRecordProvider.get(mUserId); -                if (account == null) { -                    account = new UserRecord(); +                UserRecord account = null; +                try { +                    account = mUserRecordProvider.get(mUserId); +                } catch (NotFoundException e) { +                    // User record doesn't exist, so create a new one. +                    account = new UserRecord(getRequest().getClientInfo().getUser());                  }                  account.setLanding("training");                  mUserRecordProvider.put(mUserId, account); diff --git a/src/com/p4square/grow/provider/DelegateCollectionProvider.java b/src/com/p4square/grow/provider/DelegateCollectionProvider.java index e17af87..cf697ba 100644 --- a/src/com/p4square/grow/provider/DelegateCollectionProvider.java +++ b/src/com/p4square/grow/provider/DelegateCollectionProvider.java @@ -5,7 +5,7 @@  package com.p4square.grow.provider;  import java.io.IOException; -import java.util.HashMap; +import java.util.LinkedHashMap;  import java.util.Map;  /** @@ -31,7 +31,7 @@ public abstract class DelegateCollectionProvider<C, DC, K, DK, V>      public Map<K, V> query(C collection, int limit) throws IOException {          Map<DK, V> delegateResult =  mProvider.query(makeCollectionKey(collection), limit); -        Map<K, V> result = new HashMap<>(); +        Map<K, V> result = new LinkedHashMap<>();          for (Map.Entry<DK, V> entry : delegateResult.entrySet()) {              result.put(unmakeKey(entry.getKey()), entry.getValue());          } diff --git a/src/com/p4square/grow/provider/JsonEncodedProvider.java b/src/com/p4square/grow/provider/JsonEncodedProvider.java index 7651443..500f761 100644 --- a/src/com/p4square/grow/provider/JsonEncodedProvider.java +++ b/src/com/p4square/grow/provider/JsonEncodedProvider.java @@ -46,6 +46,10 @@ public abstract class JsonEncodedProvider<V> {       * @throws IOException if the object cannot be encoded.       */      protected String encode(V obj) throws IOException { +        if (mClazz == String.class) { +            return (String) obj; +        } +          return MAPPER.writeValueAsString(obj);      } @@ -61,6 +65,10 @@ public abstract class JsonEncodedProvider<V> {              return null;          } +        if (mClazz == String.class) { +            return (V) blob; +        } +          V obj;          if (mClazz != null) {              obj = MAPPER.readValue(blob, mClazz);  | 
