Dropping ElasticSearch in favor a single data file

The entire dataset is fairly small (116 kB), so I'm dropping the
ElasticSearch cluster in favor of just writing the whole dataset into a
file in S3. This lambda will run every 15 minutes and the client side
will pull it down and filter the results.

6 files changed, 92 insertions, 265 deletions

diff --git a/src/main/java/com/p4square/groupsindexer/GroupsSearch.java b/src/main/java/com/p4square/groupsindexer/GroupsSearch.java
deleted file mode 100644
index 2cbc7e5..0000000
--- a/src/main/java/com/p4square/groupsindexer/GroupsSearch.java
+++ /dev/null
@@ -1,144 +0,0 @@
-package com.p4square.groupsindexer;
-
-import com.amazonaws.auth.AWS4Signer;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
-import com.amazonaws.http.AWSRequestSigningApacheInterceptor;
-import com.amazonaws.services.lambda.runtime.Context;
-import com.amazonaws.services.lambda.runtime.RequestHandler;
-import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent;
-import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.p4square.groupsindexer.model.ErrorResponse;
-import com.p4square.groupsindexer.model.GroupSearchDocument;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpRequestInterceptor;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * GroupsSearch is an API Gateway Proxy Lambda which executes a search and returns the results.
- *
- * Required (custom) environment variables:
- *
- * <ul>
- *  <li>ES_URL</li>
- *  <li>IMAGE_URL_PREFIX</li>
- * </ul>
- */
-public class GroupsSearch implements RequestHandler<APIGatewayProxyRequestEvent, APIGatewayProxyResponseEvent> {
-
-    private static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
-
-    private static final Logger LOG = LogManager.getLogger(GroupsSearch.class);
-    private static final ObjectMapper MAPPER = new ObjectMapper();
-
-    private final RestHighLevelClient esClient;
-    private final String imageUrlPrefix;
-
-    public GroupsSearch() throws Exception {
-        // Prefix to prepend to image urls.
-        imageUrlPrefix = System.getenv("IMAGE_URL_PREFIX");
-
-        // Setup ElasticSearch client
-        final String ES_URL = System.getenv("ES_URL");
-        AWS4Signer signer = new AWS4Signer();
-        signer.setServiceName("es");
-        signer.setRegionName(System.getenv("AWS_DEFAULT_REGION"));
-        HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(signer.getServiceName(), signer, credentialsProvider);
-
-        esClient = new RestHighLevelClient(RestClient
-                .builder(HttpHost.create(ES_URL))
-                .setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor)));
-    }
-
-    @Override
-    public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent event, Context context) {
-        final APIGatewayProxyResponseEvent response = new APIGatewayProxyResponseEvent();
-
-        try {
-            final Map<String, String> params = event.getQueryStringParameters();
-            if (params == null) {
-                response.setStatusCode(400);
-                response.setBody(MAPPER.writeValueAsString(new ErrorResponse("Request must contain a query.")));
-                return response;
-            }
-
-            final BoolQueryBuilder query = QueryBuilders.boolQuery();
-
-            if (params.containsKey("q")) {
-                query.must(QueryBuilders.simpleQueryStringQuery(params.get("q")));
-            }
-
-            if (params.containsKey("groupTypeId")) {
-                query.filter(QueryBuilders.termQuery("groupType.id", params.get("groupTypeId")));
-            }
-
-            if (params.containsKey("campusId")) {
-                query.filter(QueryBuilders.termQuery("campus.id", params.get("campusId")));
-            }
-
-            if (params.containsKey("meetingDayId")) {
-                query.filter(QueryBuilders.termQuery("meetingDay.id", params.get("meetingDayId")));
-            }
-
-            if (params.containsKey("childcare")) {
-                query.filter(QueryBuilders.termQuery("childcare", Boolean.parseBoolean(params.get("childcare"))));
-            }
-
-            params.entrySet()
-                    .stream()
-                    .filter(entry -> entry.getKey().startsWith("udf_"))
-                    .map(entry -> QueryBuilders.termQuery("udf." + entry.getKey() + ".id", entry.getValue()))
-                    .forEach(query::filter);
-
-            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-            searchSourceBuilder.query(query);
-            searchSourceBuilder.size(20);
-
-            SearchRequest searchRequest = new SearchRequest("groups");
-            searchRequest.types("group");
-            searchRequest.source(searchSourceBuilder);
-
-            SearchResponse searchResponse = esClient.search(searchRequest);
-
-            List<GroupSearchDocument> docs = new ArrayList<>();
-            for (final SearchHit hit : searchResponse.getHits().getHits()) {
-                GroupSearchDocument doc = MAPPER.readValue(hit.getSourceAsString(), GroupSearchDocument.class);
-                // Sanitize the output
-                doc.setLeaderEmail(null);
-                if (doc.getImageUrl() != null) {
-                    doc.setImageUrl(imageUrlPrefix + "/" + doc.getImageUrl());
-                }
-                docs.add(doc);
-            }
-
-            response.setStatusCode(200);
-            response.setBody(MAPPER.writeValueAsString(docs));
-
-        } catch (Exception e) {
-            LOG.error(e.getMessage());
-            response.setStatusCode(500);
-            try {
-                response.setBody(MAPPER.writeValueAsString(new ErrorResponse(e.getMessage())));
-            } catch (JsonProcessingException _) {
-                // Unexpected.
-            }
-        }
-
-        return response;
-    }
-}
diff --git a/src/main/java/com/p4square/groupsindexer/SearchFields.java b/src/main/java/com/p4square/groupsindexer/SearchFieldsCache.java
index c82781c..d5b4610 100644
--- a/src/main/java/com/p4square/groupsindexer/SearchFields.java
+++ b/src/main/java/com/p4square/groupsindexer/SearchFieldsCache.java
@@ -1,84 +1,34 @@
 package com.p4square.groupsindexer;
 
-import com.amazonaws.services.lambda.runtime.Context;
-import com.amazonaws.services.lambda.runtime.RequestHandler;
-import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent;
-import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent;
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.p4square.ccbapi.CCBAPI;
-import com.p4square.ccbapi.CCBAPIClient;
 import com.p4square.ccbapi.model.*;
-import com.p4square.groupsindexer.model.ErrorResponse;
 import com.p4square.groupsindexer.model.SearchField;
 import com.p4square.groupsindexer.model.StringPair;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.io.IOException;
-import java.net.URI;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.stream.Collectors;
 
-/**
- * SearchFields is an API Gateway Proxy which returns the searchable dropdown fields and their choices.
- *
- * Required (custom) environment variables:
- * <ul>
- *  <li>CCBAPIURL</li>
- *  <li>CCBAPIUser</li>
- *  <li>CCBAPIPassword</li>
- * </ul>
- *
- */
-public class SearchFields implements RequestHandler<APIGatewayProxyRequestEvent, APIGatewayProxyResponseEvent> {
-
+public class SearchFieldsCache {
     private static final long REFRESH_INTERVAL_MS = 15 * 60 * 1000;
 
-    private static final Logger LOG = LogManager.getLogger(GroupsSearch.class);
-    private static final ObjectMapper MAPPER = new ObjectMapper();
+    private static final Logger LOG = LogManager.getLogger(SearchFieldsCache.class);
 
     private final CCBAPI ccbClient;
 
     private List<SearchField> cachedFields;
     private long lastRefresh;
 
-    public SearchFields() throws Exception {
-        // Setup CCB Client
-        final String CCBAPIURL = System.getenv("CCBAPIURL");
-        final String CCBAPIUser = System.getenv("CCBAPIUser");
-        final String CCBAPIPassword = System.getenv("CCBAPIPassword");
-        ccbClient = new CCBAPIClient(new URI(CCBAPIURL), CCBAPIUser, CCBAPIPassword);
+    public SearchFieldsCache(CCBAPI ccbClient) {
+        this.ccbClient = ccbClient;
     }
 
-    @Override
-    public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent event, Context context) {
-        final APIGatewayProxyResponseEvent response = new APIGatewayProxyResponseEvent();
-        try {
-            final List<SearchField> fields = getFields();
-            if (fields == null) {
-                response.setStatusCode(500);
-                response.setBody(MAPPER.writeValueAsString(new ErrorResponse("Unable to get search fields.")));
-                return response;
-            }
-
-            response.setStatusCode(200);
-            response.setBody(MAPPER.writeValueAsString(fields));
-
-        } catch (Exception e) {
-            LOG.error(e.getMessage());
-            response.setStatusCode(500);
-            try {
-                response.setBody(MAPPER.writeValueAsString(new ErrorResponse(e.getMessage())));
-            } catch (JsonProcessingException _) {
-                // Unexpected.
-            }
-        }
-
-        return response;
-    }
-
-    private synchronized List<SearchField> getFields() {
+    public synchronized List<SearchField> getSearchFields() {
         if (System.currentTimeMillis() - lastRefresh < REFRESH_INTERVAL_MS) {
             LOG.debug("Using cached CCB fields");
             return cachedFields;
@@ -149,5 +99,4 @@ public class SearchFields implements RequestHandler<APIGatewayProxyRequestEvent,
                 .map(entry -> StringPair.of(String.valueOf(entry.getId()), entry.getName()))
                 .collect(Collectors.toList());
     }
-
 }
diff --git a/src/main/java/com/p4square/groupsindexer/UpdateIndexes.java b/src/main/java/com/p4square/groupsindexer/SyncGroups.java
index e4937e1..f81cb98 100644
--- a/src/main/java/com/p4square/groupsindexer/UpdateIndexes.java
+++ b/src/main/java/com/p4square/groupsindexer/SyncGroups.java
@@ -1,95 +1,92 @@
 package com.p4square.groupsindexer;
 
-import com.amazonaws.auth.AWS4Signer;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
-import com.amazonaws.http.AWSRequestSigningApacheInterceptor;
 import com.amazonaws.services.lambda.runtime.Context;
 import com.amazonaws.services.lambda.runtime.RequestHandler;
 import com.amazonaws.services.lambda.runtime.events.ScheduledEvent;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.p4square.ccbapi.CCBAPI;
 import com.p4square.ccbapi.CCBAPIClient;
-import com.p4square.ccbapi.model.*;
+import com.p4square.ccbapi.model.GetGroupProfilesRequest;
+import com.p4square.ccbapi.model.GetGroupProfilesResponse;
+import com.p4square.ccbapi.model.GroupProfile;
+import com.p4square.ccbapi.model.InteractionType;
 import com.p4square.groupsindexer.model.GroupSearchDocument;
 import com.p4square.groupsindexer.model.GroupSearchDocumentAdapter;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpRequestInterceptor;
+import com.p4square.groupsindexer.model.GroupsCollection;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.net.URL;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
- * UpdateIndexes is a scheduled lambda which populates the groups search index.
+ * SyncGroups is a scheduled lambda which syncs groups data from CCB.
  *
  * Required (custom) environment variables:
  * <ul>
  *  <li>CCBAPIURL</li>
  *  <li>CCBAPIUser</li>
  *  <li>CCBAPIPassword</li>
- *  <li>ES_URL</li>
- *  <li>IMAGE_BUCKET</li>
+ *  <li>OUTPUT_BUCKET</li>
  * </ul>
  *
  */
-public class UpdateIndexes implements RequestHandler<ScheduledEvent, String> {
+public class SyncGroups implements RequestHandler<ScheduledEvent, String> {
 
-    private static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
-
-    private static final Logger LOG = LogManager.getLogger(UpdateIndexes.class);
+    private static final Logger LOG = LogManager.getLogger(SyncGroups.class);
     private static final GroupSearchDocumentAdapter ADAPTER = new GroupSearchDocumentAdapter();
     private static final ObjectMapper MAPPER = new ObjectMapper();
+    static {
+        MAPPER.registerModule(new JavaTimeModule());
+        MAPPER.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+    }
 
-    private final String imageBucket;
+    private final String outputBucket;
+    private final String baseUrl;
 
     private final CCBAPI ccbClient;
-    private final RestHighLevelClient esClient;
     private final AmazonS3 s3Client;
 
-    public UpdateIndexes() throws Exception {
+    private final SearchFieldsCache searchFieldsCache;
+
+    public SyncGroups() throws Exception {
         // Setup CCB Client
         final String CCBAPIURL = System.getenv("CCBAPIURL");
         final String CCBAPIUser = System.getenv("CCBAPIUser");
         final String CCBAPIPassword = System.getenv("CCBAPIPassword");
         ccbClient = new CCBAPIClient(new URI(CCBAPIURL), CCBAPIUser, CCBAPIPassword);
-
-        // Setup ElasticSearch client
-        final String ES_URL = System.getenv("ES_URL");
-        AWS4Signer signer = new AWS4Signer();
-        signer.setServiceName("es");
-        signer.setRegionName(System.getenv("AWS_DEFAULT_REGION"));
-        HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(signer.getServiceName(), signer, credentialsProvider);
-
-        esClient = new RestHighLevelClient(RestClient
-                        .builder(HttpHost.create(ES_URL))
-                        .setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor)));
+        searchFieldsCache = new SearchFieldsCache(ccbClient);
 
         // Setup S3 Client
-        imageBucket = System.getenv("IMAGE_BUCKET");
+        outputBucket = System.getenv("OUTPUT_BUCKET");
         s3Client = AmazonS3ClientBuilder.defaultClient();
+
+        // Prefix to prepend to image urls.
+        baseUrl = System.getenv("BASE_URL");
     }
 
     @Override
     public String handleRequest(ScheduledEvent s3Event, Context context) {
         try {
-            GetGroupProfilesResponse response = ccbClient.getGroupProfiles(
+            final GroupsCollection groupsCollection = new GroupsCollection();
+            groupsCollection.setLastUpdated(Instant.now());
+            groupsCollection.setSearchFields(searchFieldsCache.getSearchFields());
+
+            final GetGroupProfilesResponse response = ccbClient.getGroupProfiles(
                     new GetGroupProfilesRequest()
                             .withIncludeImageUrl(true)
                             .withIncludeParticipants(false));
 
-            final BulkRequest indexRequest = new BulkRequest();
+            final List<GroupSearchDocument> groups = new ArrayList<>();
 
             for (GroupProfile profile : response.getGroups()) {
                 if (!profile.isActive() ||
@@ -110,8 +107,8 @@ public class UpdateIndexes implements RequestHandler<ScheduledEvent, String> {
                     try {
                         final URL imageUrl = new URL(profile.getImageUrl());
                         in = imageUrl.openStream();
-                        s3Client.putObject(imageBucket, imageKey, in, null);
-                        document.setImageUrl(imageKey);
+                        s3Client.putObject(outputBucket, imageKey, in, null);
+                        document.setImageUrl(baseUrl + "/" + imageKey);
                     } catch (Exception e) {
                         LOG.error("Failed to upload image for group " + profile.getId(), e);
                     } finally {
@@ -125,19 +122,12 @@ public class UpdateIndexes implements RequestHandler<ScheduledEvent, String> {
                     }
                 }
 
-                // Add request to batch.
-                indexRequest.add(Requests
-                        .indexRequest("groups")
-                        .type("group")
-                        .id(String.valueOf(document.getId()))
-                        .source(MAPPER.writeValueAsString(document), XContentType.JSON));
+                groups.add(document);
             }
 
-            BulkResponse esResponse = esClient.bulk(indexRequest);
-
-            if (esResponse.hasFailures()) {
-                throw new RuntimeException(esResponse.buildFailureMessage());
-            }
+            // Save the groups data
+            groupsCollection.setGroups(groups);
+            s3Client.putObject(outputBucket, "data/groups-data.json", MAPPER.writeValueAsString(groupsCollection));
 
             LOG.info("Updated search index. Found " + response.getGroups().size() + " groups.");
             return "ok";
diff --git a/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java b/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java
index e336cb8..5708c4f 100644
--- a/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java
+++ b/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java
@@ -29,9 +29,6 @@ public class GroupSearchDocument {
     @JsonProperty("leader-name")
     private String leaderName;
 
-    @JsonProperty("leader-email")
-    private String leaderEmail;
-
     @JsonProperty("member-count")
     private int currentMembers;
 
@@ -104,14 +101,6 @@ public class GroupSearchDocument {
         this.leaderName = leaderName;
     }
 
-    public String getLeaderEmail() {
-        return leaderEmail;
-    }
-
-    public void setLeaderEmail(String leaderEmail) {
-        this.leaderEmail = leaderEmail;
-    }
-
     public String getImageUrl() {
         return imageUrl;
     }
diff --git a/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java b/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java
index ebe852d..2a9a089 100644
--- a/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java
+++ b/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java
@@ -19,7 +19,6 @@ public class GroupSearchDocumentAdapter implements Function<GroupProfile, GroupS
         doc.setImageUrl(groupProfile.getImageUrl());
         doc.setLeaderId(groupProfile.getMainLeader().getId());
         doc.setLeaderName(groupProfile.getMainLeader().getFullName());
-        doc.setLeaderEmail(groupProfile.getMainLeader().getEmail());
         doc.setCurrentMembers(groupProfile.getCurrentMembers());
         doc.setGroupCapacity(groupProfile.getGroupCapacity());
         doc.setChildcareProvided(groupProfile.isChildcareProvided());
diff --git a/src/main/java/com/p4square/groupsindexer/model/GroupsCollection.java b/src/main/java/com/p4square/groupsindexer/model/GroupsCollection.java
new file mode 100644
index 0000000..de14239
--- /dev/null
+++ b/src/main/java/com/p4square/groupsindexer/model/GroupsCollection.java
@@ -0,0 +1,44 @@
+package com.p4square.groupsindexer.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.Instant;
+import java.util.List;
+
+/**
+ * A list of groups and some metadata.
+ */
+public class GroupsCollection {
+    @JsonProperty("last-updated")
+    private Instant lastUpdated;
+
+    @JsonProperty("groups")
+    private List<GroupSearchDocument> groups;
+
+    @JsonProperty("search-fields")
+    private List<SearchField> searchFields;
+
+    public Instant getLastUpdated() {
+        return lastUpdated;
+    }
+
+    public void setLastUpdated(Instant lastUpdated) {
+        this.lastUpdated = lastUpdated;
+    }
+
+    public List<GroupSearchDocument> getGroups() {
+        return groups;
+    }
+
+    public void setGroups(List<GroupSearchDocument> groups) {
+        this.groups = groups;
+    }
+
+    public List<SearchField> getSearchFields() {
+        return searchFields;
+    }
+
+    public void setSearchFields(List<SearchField> searchFields) {
+        this.searchFields = searchFields;
+    }
+}