diff options
author | Jesse Morgan <jesse@jesterpm.net> | 2018-08-04 16:23:03 -0700 |
---|---|---|
committer | Jesse Morgan <jesse@jesterpm.net> | 2018-08-04 16:23:03 -0700 |
commit | 86665e59f0269d1e41ac88fba41a9099c454be6a (patch) | |
tree | 19caf3a8e0c36230dfbe09f4be049d3f5b1dd308 /src/main | |
parent | 672c4709bba91cd58caacb47b006fb19bab646f6 (diff) |
Dropping ElasticSearch in favor a single data file
The entire dataset is fairly small (116 kB), so I'm dropping the
ElasticSearch cluster in favor of just writing the whole dataset into a
file in S3. This lambda will run every 15 minutes and the client side
will pull it down and filter the results.
Diffstat (limited to 'src/main')
-rw-r--r-- | src/main/java/com/p4square/groupsindexer/GroupsSearch.java | 144 | ||||
-rw-r--r-- | src/main/java/com/p4square/groupsindexer/SearchFieldsCache.java (renamed from src/main/java/com/p4square/groupsindexer/SearchFields.java) | 67 | ||||
-rw-r--r-- | src/main/java/com/p4square/groupsindexer/SyncGroups.java (renamed from src/main/java/com/p4square/groupsindexer/UpdateIndexes.java) | 90 | ||||
-rw-r--r-- | src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java | 11 | ||||
-rw-r--r-- | src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java | 1 | ||||
-rw-r--r-- | src/main/java/com/p4square/groupsindexer/model/GroupsCollection.java | 44 |
6 files changed, 92 insertions, 265 deletions
diff --git a/src/main/java/com/p4square/groupsindexer/GroupsSearch.java b/src/main/java/com/p4square/groupsindexer/GroupsSearch.java deleted file mode 100644 index 2cbc7e5..0000000 --- a/src/main/java/com/p4square/groupsindexer/GroupsSearch.java +++ /dev/null @@ -1,144 +0,0 @@ -package com.p4square.groupsindexer; - -import com.amazonaws.auth.AWS4Signer; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.http.AWSRequestSigningApacheInterceptor; -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; -import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.p4square.groupsindexer.model.ErrorResponse; -import com.p4square.groupsindexer.model.GroupSearchDocument; -import org.apache.http.HttpHost; -import org.apache.http.HttpRequestInterceptor; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * GroupsSearch is an API Gateway Proxy Lambda which executes a search and returns the results. - * - * Required (custom) environment variables: - * - * <ul> - * <li>ES_URL</li> - * <li>IMAGE_URL_PREFIX</li> - * </ul> - */ -public class GroupsSearch implements RequestHandler<APIGatewayProxyRequestEvent, APIGatewayProxyResponseEvent> { - - private static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); - - private static final Logger LOG = LogManager.getLogger(GroupsSearch.class); - private static final ObjectMapper MAPPER = new ObjectMapper(); - - private final RestHighLevelClient esClient; - private final String imageUrlPrefix; - - public GroupsSearch() throws Exception { - // Prefix to prepend to image urls. - imageUrlPrefix = System.getenv("IMAGE_URL_PREFIX"); - - // Setup ElasticSearch client - final String ES_URL = System.getenv("ES_URL"); - AWS4Signer signer = new AWS4Signer(); - signer.setServiceName("es"); - signer.setRegionName(System.getenv("AWS_DEFAULT_REGION")); - HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(signer.getServiceName(), signer, credentialsProvider); - - esClient = new RestHighLevelClient(RestClient - .builder(HttpHost.create(ES_URL)) - .setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor))); - } - - @Override - public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent event, Context context) { - final APIGatewayProxyResponseEvent response = new APIGatewayProxyResponseEvent(); - - try { - final Map<String, String> params = event.getQueryStringParameters(); - if (params == null) { - response.setStatusCode(400); - response.setBody(MAPPER.writeValueAsString(new ErrorResponse("Request must contain a query."))); - return response; - } - - final BoolQueryBuilder query = QueryBuilders.boolQuery(); - - if (params.containsKey("q")) { - query.must(QueryBuilders.simpleQueryStringQuery(params.get("q"))); - } - - if (params.containsKey("groupTypeId")) { - query.filter(QueryBuilders.termQuery("groupType.id", params.get("groupTypeId"))); - } - - if (params.containsKey("campusId")) { - query.filter(QueryBuilders.termQuery("campus.id", params.get("campusId"))); - } - - if (params.containsKey("meetingDayId")) { - query.filter(QueryBuilders.termQuery("meetingDay.id", params.get("meetingDayId"))); - } - - if (params.containsKey("childcare")) { - query.filter(QueryBuilders.termQuery("childcare", Boolean.parseBoolean(params.get("childcare")))); - } - - params.entrySet() - .stream() - .filter(entry -> entry.getKey().startsWith("udf_")) - .map(entry -> QueryBuilders.termQuery("udf." + entry.getKey() + ".id", entry.getValue())) - .forEach(query::filter); - - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(query); - searchSourceBuilder.size(20); - - SearchRequest searchRequest = new SearchRequest("groups"); - searchRequest.types("group"); - searchRequest.source(searchSourceBuilder); - - SearchResponse searchResponse = esClient.search(searchRequest); - - List<GroupSearchDocument> docs = new ArrayList<>(); - for (final SearchHit hit : searchResponse.getHits().getHits()) { - GroupSearchDocument doc = MAPPER.readValue(hit.getSourceAsString(), GroupSearchDocument.class); - // Sanitize the output - doc.setLeaderEmail(null); - if (doc.getImageUrl() != null) { - doc.setImageUrl(imageUrlPrefix + "/" + doc.getImageUrl()); - } - docs.add(doc); - } - - response.setStatusCode(200); - response.setBody(MAPPER.writeValueAsString(docs)); - - } catch (Exception e) { - LOG.error(e.getMessage()); - response.setStatusCode(500); - try { - response.setBody(MAPPER.writeValueAsString(new ErrorResponse(e.getMessage()))); - } catch (JsonProcessingException _) { - // Unexpected. - } - } - - return response; - } -} diff --git a/src/main/java/com/p4square/groupsindexer/SearchFields.java b/src/main/java/com/p4square/groupsindexer/SearchFieldsCache.java index c82781c..d5b4610 100644 --- a/src/main/java/com/p4square/groupsindexer/SearchFields.java +++ b/src/main/java/com/p4square/groupsindexer/SearchFieldsCache.java @@ -1,84 +1,34 @@ package com.p4square.groupsindexer; -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; -import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.p4square.ccbapi.CCBAPI; -import com.p4square.ccbapi.CCBAPIClient; import com.p4square.ccbapi.model.*; -import com.p4square.groupsindexer.model.ErrorResponse; import com.p4square.groupsindexer.model.SearchField; import com.p4square.groupsindexer.model.StringPair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.IOException; -import java.net.URI; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.stream.Collectors; -/** - * SearchFields is an API Gateway Proxy which returns the searchable dropdown fields and their choices. - * - * Required (custom) environment variables: - * <ul> - * <li>CCBAPIURL</li> - * <li>CCBAPIUser</li> - * <li>CCBAPIPassword</li> - * </ul> - * - */ -public class SearchFields implements RequestHandler<APIGatewayProxyRequestEvent, APIGatewayProxyResponseEvent> { - +public class SearchFieldsCache { private static final long REFRESH_INTERVAL_MS = 15 * 60 * 1000; - private static final Logger LOG = LogManager.getLogger(GroupsSearch.class); - private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final Logger LOG = LogManager.getLogger(SearchFieldsCache.class); private final CCBAPI ccbClient; private List<SearchField> cachedFields; private long lastRefresh; - public SearchFields() throws Exception { - // Setup CCB Client - final String CCBAPIURL = System.getenv("CCBAPIURL"); - final String CCBAPIUser = System.getenv("CCBAPIUser"); - final String CCBAPIPassword = System.getenv("CCBAPIPassword"); - ccbClient = new CCBAPIClient(new URI(CCBAPIURL), CCBAPIUser, CCBAPIPassword); + public SearchFieldsCache(CCBAPI ccbClient) { + this.ccbClient = ccbClient; } - @Override - public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent event, Context context) { - final APIGatewayProxyResponseEvent response = new APIGatewayProxyResponseEvent(); - try { - final List<SearchField> fields = getFields(); - if (fields == null) { - response.setStatusCode(500); - response.setBody(MAPPER.writeValueAsString(new ErrorResponse("Unable to get search fields."))); - return response; - } - - response.setStatusCode(200); - response.setBody(MAPPER.writeValueAsString(fields)); - - } catch (Exception e) { - LOG.error(e.getMessage()); - response.setStatusCode(500); - try { - response.setBody(MAPPER.writeValueAsString(new ErrorResponse(e.getMessage()))); - } catch (JsonProcessingException _) { - // Unexpected. - } - } - - return response; - } - - private synchronized List<SearchField> getFields() { + public synchronized List<SearchField> getSearchFields() { if (System.currentTimeMillis() - lastRefresh < REFRESH_INTERVAL_MS) { LOG.debug("Using cached CCB fields"); return cachedFields; @@ -149,5 +99,4 @@ public class SearchFields implements RequestHandler<APIGatewayProxyRequestEvent, .map(entry -> StringPair.of(String.valueOf(entry.getId()), entry.getName())) .collect(Collectors.toList()); } - } diff --git a/src/main/java/com/p4square/groupsindexer/UpdateIndexes.java b/src/main/java/com/p4square/groupsindexer/SyncGroups.java index e4937e1..f81cb98 100644 --- a/src/main/java/com/p4square/groupsindexer/UpdateIndexes.java +++ b/src/main/java/com/p4square/groupsindexer/SyncGroups.java @@ -1,95 +1,92 @@ package com.p4square.groupsindexer; -import com.amazonaws.auth.AWS4Signer; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.http.AWSRequestSigningApacheInterceptor; import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.ScheduledEvent; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.p4square.ccbapi.CCBAPI; import com.p4square.ccbapi.CCBAPIClient; -import com.p4square.ccbapi.model.*; +import com.p4square.ccbapi.model.GetGroupProfilesRequest; +import com.p4square.ccbapi.model.GetGroupProfilesResponse; +import com.p4square.ccbapi.model.GroupProfile; +import com.p4square.ccbapi.model.InteractionType; import com.p4square.groupsindexer.model.GroupSearchDocument; import com.p4square.groupsindexer.model.GroupSearchDocumentAdapter; -import org.apache.http.HttpHost; -import org.apache.http.HttpRequestInterceptor; +import com.p4square.groupsindexer.model.GroupsCollection; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.client.Requests; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URL; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; /** - * UpdateIndexes is a scheduled lambda which populates the groups search index. + * SyncGroups is a scheduled lambda which syncs groups data from CCB. * * Required (custom) environment variables: * <ul> * <li>CCBAPIURL</li> * <li>CCBAPIUser</li> * <li>CCBAPIPassword</li> - * <li>ES_URL</li> - * <li>IMAGE_BUCKET</li> + * <li>OUTPUT_BUCKET</li> * </ul> * */ -public class UpdateIndexes implements RequestHandler<ScheduledEvent, String> { +public class SyncGroups implements RequestHandler<ScheduledEvent, String> { - private static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); - - private static final Logger LOG = LogManager.getLogger(UpdateIndexes.class); + private static final Logger LOG = LogManager.getLogger(SyncGroups.class); private static final GroupSearchDocumentAdapter ADAPTER = new GroupSearchDocumentAdapter(); private static final ObjectMapper MAPPER = new ObjectMapper(); + static { + MAPPER.registerModule(new JavaTimeModule()); + MAPPER.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + } - private final String imageBucket; + private final String outputBucket; + private final String baseUrl; private final CCBAPI ccbClient; - private final RestHighLevelClient esClient; private final AmazonS3 s3Client; - public UpdateIndexes() throws Exception { + private final SearchFieldsCache searchFieldsCache; + + public SyncGroups() throws Exception { // Setup CCB Client final String CCBAPIURL = System.getenv("CCBAPIURL"); final String CCBAPIUser = System.getenv("CCBAPIUser"); final String CCBAPIPassword = System.getenv("CCBAPIPassword"); ccbClient = new CCBAPIClient(new URI(CCBAPIURL), CCBAPIUser, CCBAPIPassword); - - // Setup ElasticSearch client - final String ES_URL = System.getenv("ES_URL"); - AWS4Signer signer = new AWS4Signer(); - signer.setServiceName("es"); - signer.setRegionName(System.getenv("AWS_DEFAULT_REGION")); - HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(signer.getServiceName(), signer, credentialsProvider); - - esClient = new RestHighLevelClient(RestClient - .builder(HttpHost.create(ES_URL)) - .setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor))); + searchFieldsCache = new SearchFieldsCache(ccbClient); // Setup S3 Client - imageBucket = System.getenv("IMAGE_BUCKET"); + outputBucket = System.getenv("OUTPUT_BUCKET"); s3Client = AmazonS3ClientBuilder.defaultClient(); + + // Prefix to prepend to image urls. + baseUrl = System.getenv("BASE_URL"); } @Override public String handleRequest(ScheduledEvent s3Event, Context context) { try { - GetGroupProfilesResponse response = ccbClient.getGroupProfiles( + final GroupsCollection groupsCollection = new GroupsCollection(); + groupsCollection.setLastUpdated(Instant.now()); + groupsCollection.setSearchFields(searchFieldsCache.getSearchFields()); + + final GetGroupProfilesResponse response = ccbClient.getGroupProfiles( new GetGroupProfilesRequest() .withIncludeImageUrl(true) .withIncludeParticipants(false)); - final BulkRequest indexRequest = new BulkRequest(); + final List<GroupSearchDocument> groups = new ArrayList<>(); for (GroupProfile profile : response.getGroups()) { if (!profile.isActive() || @@ -110,8 +107,8 @@ public class UpdateIndexes implements RequestHandler<ScheduledEvent, String> { try { final URL imageUrl = new URL(profile.getImageUrl()); in = imageUrl.openStream(); - s3Client.putObject(imageBucket, imageKey, in, null); - document.setImageUrl(imageKey); + s3Client.putObject(outputBucket, imageKey, in, null); + document.setImageUrl(baseUrl + "/" + imageKey); } catch (Exception e) { LOG.error("Failed to upload image for group " + profile.getId(), e); } finally { @@ -125,19 +122,12 @@ public class UpdateIndexes implements RequestHandler<ScheduledEvent, String> { } } - // Add request to batch. - indexRequest.add(Requests - .indexRequest("groups") - .type("group") - .id(String.valueOf(document.getId())) - .source(MAPPER.writeValueAsString(document), XContentType.JSON)); + groups.add(document); } - BulkResponse esResponse = esClient.bulk(indexRequest); - - if (esResponse.hasFailures()) { - throw new RuntimeException(esResponse.buildFailureMessage()); - } + // Save the groups data + groupsCollection.setGroups(groups); + s3Client.putObject(outputBucket, "data/groups-data.json", MAPPER.writeValueAsString(groupsCollection)); LOG.info("Updated search index. Found " + response.getGroups().size() + " groups."); return "ok"; diff --git a/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java b/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java index e336cb8..5708c4f 100644 --- a/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java +++ b/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java @@ -29,9 +29,6 @@ public class GroupSearchDocument { @JsonProperty("leader-name") private String leaderName; - @JsonProperty("leader-email") - private String leaderEmail; - @JsonProperty("member-count") private int currentMembers; @@ -104,14 +101,6 @@ public class GroupSearchDocument { this.leaderName = leaderName; } - public String getLeaderEmail() { - return leaderEmail; - } - - public void setLeaderEmail(String leaderEmail) { - this.leaderEmail = leaderEmail; - } - public String getImageUrl() { return imageUrl; } diff --git a/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java b/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java index ebe852d..2a9a089 100644 --- a/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java +++ b/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java @@ -19,7 +19,6 @@ public class GroupSearchDocumentAdapter implements Function<GroupProfile, GroupS doc.setImageUrl(groupProfile.getImageUrl()); doc.setLeaderId(groupProfile.getMainLeader().getId()); doc.setLeaderName(groupProfile.getMainLeader().getFullName()); - doc.setLeaderEmail(groupProfile.getMainLeader().getEmail()); doc.setCurrentMembers(groupProfile.getCurrentMembers()); doc.setGroupCapacity(groupProfile.getGroupCapacity()); doc.setChildcareProvided(groupProfile.isChildcareProvided()); diff --git a/src/main/java/com/p4square/groupsindexer/model/GroupsCollection.java b/src/main/java/com/p4square/groupsindexer/model/GroupsCollection.java new file mode 100644 index 0000000..de14239 --- /dev/null +++ b/src/main/java/com/p4square/groupsindexer/model/GroupsCollection.java @@ -0,0 +1,44 @@ +package com.p4square.groupsindexer.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.util.List; + +/** + * A list of groups and some metadata. + */ +public class GroupsCollection { + @JsonProperty("last-updated") + private Instant lastUpdated; + + @JsonProperty("groups") + private List<GroupSearchDocument> groups; + + @JsonProperty("search-fields") + private List<SearchField> searchFields; + + public Instant getLastUpdated() { + return lastUpdated; + } + + public void setLastUpdated(Instant lastUpdated) { + this.lastUpdated = lastUpdated; + } + + public List<GroupSearchDocument> getGroups() { + return groups; + } + + public void setGroups(List<GroupSearchDocument> groups) { + this.groups = groups; + } + + public List<SearchField> getSearchFields() { + return searchFields; + } + + public void setSearchFields(List<SearchField> searchFields) { + this.searchFields = searchFields; + } +} |