From 86665e59f0269d1e41ac88fba41a9099c454be6a Mon Sep 17 00:00:00 2001 From: Jesse Morgan Date: Sat, 4 Aug 2018 16:23:03 -0700 Subject: Dropping ElasticSearch in favor a single data file The entire dataset is fairly small (116 kB), so I'm dropping the ElasticSearch cluster in favor of just writing the whole dataset into a file in S3. This lambda will run every 15 minutes and the client side will pull it down and filter the results. --- pom.xml | 44 ++---- .../com/p4square/groupsindexer/GroupsSearch.java | 144 ------------------- .../com/p4square/groupsindexer/SearchFields.java | 153 --------------------- .../p4square/groupsindexer/SearchFieldsCache.java | 102 ++++++++++++++ .../com/p4square/groupsindexer/SyncGroups.java | 140 +++++++++++++++++++ .../com/p4square/groupsindexer/UpdateIndexes.java | 150 -------------------- .../groupsindexer/model/GroupSearchDocument.java | 11 -- .../model/GroupSearchDocumentAdapter.java | 1 - .../groupsindexer/model/GroupsCollection.java | 44 ++++++ 9 files changed, 301 insertions(+), 488 deletions(-) delete mode 100644 src/main/java/com/p4square/groupsindexer/GroupsSearch.java delete mode 100644 src/main/java/com/p4square/groupsindexer/SearchFields.java create mode 100644 src/main/java/com/p4square/groupsindexer/SearchFieldsCache.java create mode 100644 src/main/java/com/p4square/groupsindexer/SyncGroups.java delete mode 100644 src/main/java/com/p4square/groupsindexer/UpdateIndexes.java create mode 100644 src/main/java/com/p4square/groupsindexer/model/GroupsCollection.java diff --git a/pom.xml b/pom.xml index 31e4e13..6239111 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,23 @@ - - - jitpack.io - https://jitpack.io - - + + + + com.fasterxml.jackson + jackson-bom + 2.9.0 + pom + import + + + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + com.amazonaws aws-lambda-java-core @@ -49,35 +58,12 @@ 1.1.0 compile - - com.amazonaws - aws-java-sdk-elasticsearch - [1.11,1.12) - compile - com.amazonaws aws-java-sdk-s3 [1.11,1.12) compile - - org.elasticsearch.client - elasticsearch-rest-high-level-client - [6.3,6.4) - compile - - - org.apache.httpcomponents - httpclient - 4.5.5 - - - com.github.awslabs - aws-request-signing-apache-interceptor - b3772780da33a9b50d806636dc90f8fc6b74b8dc - compile - com.p4square ccbapi diff --git a/src/main/java/com/p4square/groupsindexer/GroupsSearch.java b/src/main/java/com/p4square/groupsindexer/GroupsSearch.java deleted file mode 100644 index 2cbc7e5..0000000 --- a/src/main/java/com/p4square/groupsindexer/GroupsSearch.java +++ /dev/null @@ -1,144 +0,0 @@ -package com.p4square.groupsindexer; - -import com.amazonaws.auth.AWS4Signer; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.http.AWSRequestSigningApacheInterceptor; -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; -import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.p4square.groupsindexer.model.ErrorResponse; -import com.p4square.groupsindexer.model.GroupSearchDocument; -import org.apache.http.HttpHost; -import org.apache.http.HttpRequestInterceptor; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * GroupsSearch is an API Gateway Proxy Lambda which executes a search and returns the results. - * - * Required (custom) environment variables: - * - *
    - *
  • ES_URL
    • - *
  • IMAGE_URL_PREFIX
    • - *
- */ -public class GroupsSearch implements RequestHandler { - - private static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); - - private static final Logger LOG = LogManager.getLogger(GroupsSearch.class); - private static final ObjectMapper MAPPER = new ObjectMapper(); - - private final RestHighLevelClient esClient; - private final String imageUrlPrefix; - - public GroupsSearch() throws Exception { - // Prefix to prepend to image urls. - imageUrlPrefix = System.getenv("IMAGE_URL_PREFIX"); - - // Setup ElasticSearch client - final String ES_URL = System.getenv("ES_URL"); - AWS4Signer signer = new AWS4Signer(); - signer.setServiceName("es"); - signer.setRegionName(System.getenv("AWS_DEFAULT_REGION")); - HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(signer.getServiceName(), signer, credentialsProvider); - - esClient = new RestHighLevelClient(RestClient - .builder(HttpHost.create(ES_URL)) - .setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor))); - } - - @Override - public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent event, Context context) { - final APIGatewayProxyResponseEvent response = new APIGatewayProxyResponseEvent(); - - try { - final Map params = event.getQueryStringParameters(); - if (params == null) { - response.setStatusCode(400); - response.setBody(MAPPER.writeValueAsString(new ErrorResponse("Request must contain a query."))); - return response; - } - - final BoolQueryBuilder query = QueryBuilders.boolQuery(); - - if (params.containsKey("q")) { - query.must(QueryBuilders.simpleQueryStringQuery(params.get("q"))); - } - - if (params.containsKey("groupTypeId")) { - query.filter(QueryBuilders.termQuery("groupType.id", params.get("groupTypeId"))); - } - - if (params.containsKey("campusId")) { - query.filter(QueryBuilders.termQuery("campus.id", params.get("campusId"))); - } - - if (params.containsKey("meetingDayId")) { - query.filter(QueryBuilders.termQuery("meetingDay.id", params.get("meetingDayId"))); - } - - if (params.containsKey("childcare")) { - query.filter(QueryBuilders.termQuery("childcare", Boolean.parseBoolean(params.get("childcare")))); - } - - params.entrySet() - .stream() - .filter(entry -> entry.getKey().startsWith("udf_")) - .map(entry -> QueryBuilders.termQuery("udf." + entry.getKey() + ".id", entry.getValue())) - .forEach(query::filter); - - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(query); - searchSourceBuilder.size(20); - - SearchRequest searchRequest = new SearchRequest("groups"); - searchRequest.types("group"); - searchRequest.source(searchSourceBuilder); - - SearchResponse searchResponse = esClient.search(searchRequest); - - List docs = new ArrayList<>(); - for (final SearchHit hit : searchResponse.getHits().getHits()) { - GroupSearchDocument doc = MAPPER.readValue(hit.getSourceAsString(), GroupSearchDocument.class); - // Sanitize the output - doc.setLeaderEmail(null); - if (doc.getImageUrl() != null) { - doc.setImageUrl(imageUrlPrefix + "/" + doc.getImageUrl()); - } - docs.add(doc); - } - - response.setStatusCode(200); - response.setBody(MAPPER.writeValueAsString(docs)); - - } catch (Exception e) { - LOG.error(e.getMessage()); - response.setStatusCode(500); - try { - response.setBody(MAPPER.writeValueAsString(new ErrorResponse(e.getMessage()))); - } catch (JsonProcessingException _) { - // Unexpected. - } - } - - return response; - } -} diff --git a/src/main/java/com/p4square/groupsindexer/SearchFields.java b/src/main/java/com/p4square/groupsindexer/SearchFields.java deleted file mode 100644 index c82781c..0000000 --- a/src/main/java/com/p4square/groupsindexer/SearchFields.java +++ /dev/null @@ -1,153 +0,0 @@ -package com.p4square.groupsindexer; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; -import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.p4square.ccbapi.CCBAPI; -import com.p4square.ccbapi.CCBAPIClient; -import com.p4square.ccbapi.model.*; -import com.p4square.groupsindexer.model.ErrorResponse; -import com.p4square.groupsindexer.model.SearchField; -import com.p4square.groupsindexer.model.StringPair; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.net.URI; -import java.util.*; -import java.util.stream.Collectors; - -/** - * SearchFields is an API Gateway Proxy which returns the searchable dropdown fields and their choices. - * - * Required (custom) environment variables: - *
    - *
  • CCBAPIURL
    • - *
  • CCBAPIUser
    • - *
  • CCBAPIPassword
    • - *
- * - */ -public class SearchFields implements RequestHandler { - - private static final long REFRESH_INTERVAL_MS = 15 * 60 * 1000; - - private static final Logger LOG = LogManager.getLogger(GroupsSearch.class); - private static final ObjectMapper MAPPER = new ObjectMapper(); - - private final CCBAPI ccbClient; - - private List cachedFields; - private long lastRefresh; - - public SearchFields() throws Exception { - // Setup CCB Client - final String CCBAPIURL = System.getenv("CCBAPIURL"); - final String CCBAPIUser = System.getenv("CCBAPIUser"); - final String CCBAPIPassword = System.getenv("CCBAPIPassword"); - ccbClient = new CCBAPIClient(new URI(CCBAPIURL), CCBAPIUser, CCBAPIPassword); - } - - @Override - public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent event, Context context) { - final APIGatewayProxyResponseEvent response = new APIGatewayProxyResponseEvent(); - try { - final List fields = getFields(); - if (fields == null) { - response.setStatusCode(500); - response.setBody(MAPPER.writeValueAsString(new ErrorResponse("Unable to get search fields."))); - return response; - } - - response.setStatusCode(200); - response.setBody(MAPPER.writeValueAsString(fields)); - - } catch (Exception e) { - LOG.error(e.getMessage()); - response.setStatusCode(500); - try { - response.setBody(MAPPER.writeValueAsString(new ErrorResponse(e.getMessage()))); - } catch (JsonProcessingException _) { - // Unexpected. - } - } - - return response; - } - - private synchronized List getFields() { - if (System.currentTimeMillis() - lastRefresh < REFRESH_INTERVAL_MS) { - LOG.debug("Using cached CCB fields"); - return cachedFields; - } - - try { - LOG.info("Fetching fields from CCB"); - - cachedFields = new ArrayList<>(); - final GetCustomFieldLabelsResponse labels = ccbClient.getCustomFieldLabels(); - - cachedFields.add(new SearchField("groupType", "Group Type", getValues(LookupTableType.GROUP_TYPE))); - // TODO fields.add(new SearchField("campusId", "Campus", ...)); - cachedFields.add(new SearchField("meetingDay", "Day", getValues(LookupTableType.MEET_DAY))); - - - for (final CustomField field : labels.getCustomFields()) { - final LookupTableType type = getTypeFromString(field.getName()); - if (type != null) { - cachedFields.add(new SearchField(getSearchFieldIdForType(type), field.getLabel(), getValues(type))); - } - } - - cachedFields.add(new SearchField("childcare", "Childcare", - Arrays.asList(StringPair.of("true", "Yes"), StringPair.of("false", "No")))); - - lastRefresh = System.currentTimeMillis(); - return cachedFields; - - } catch (Exception e) { - LOG.error(e.getMessage()); - return null; - } - } - - private LookupTableType getTypeFromString(String name) { - switch (name) { - case "udf_grp_pulldown_1": - return LookupTableType.UDF_GRP_PULLDOWN_1; - case "udf_grp_pulldown_2": - return LookupTableType.UDF_GRP_PULLDOWN_2; - case "udf_grp_pulldown_3": - return LookupTableType.UDF_GRP_PULLDOWN_3; - default: - return null; - } - } - - private String getSearchFieldIdForType(LookupTableType type) { - switch (type) { - case UDF_GRP_PULLDOWN_1: - return "udf_1"; - case UDF_GRP_PULLDOWN_2: - return "udf_2"; - case UDF_GRP_PULLDOWN_3: - return "udf_3"; - default: - throw new IllegalArgumentException(); - } - } - - private List getValues(LookupTableType type) throws IOException { - final GetLookupTableRequest lookupTableRequest = new GetLookupTableRequest().withType(type); - final GetLookupTableResponse lookupTable = ccbClient.getLookupTable(lookupTableRequest); - - return lookupTable.getItems() - .stream() - .map(entry -> StringPair.of(String.valueOf(entry.getId()), entry.getName())) - .collect(Collectors.toList()); - } - -} diff --git a/src/main/java/com/p4square/groupsindexer/SearchFieldsCache.java b/src/main/java/com/p4square/groupsindexer/SearchFieldsCache.java new file mode 100644 index 0000000..d5b4610 --- /dev/null +++ b/src/main/java/com/p4square/groupsindexer/SearchFieldsCache.java @@ -0,0 +1,102 @@ +package com.p4square.groupsindexer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.p4square.ccbapi.CCBAPI; +import com.p4square.ccbapi.model.*; +import com.p4square.groupsindexer.model.SearchField; +import com.p4square.groupsindexer.model.StringPair; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class SearchFieldsCache { + private static final long REFRESH_INTERVAL_MS = 15 * 60 * 1000; + + private static final Logger LOG = LogManager.getLogger(SearchFieldsCache.class); + + private final CCBAPI ccbClient; + + private List cachedFields; + private long lastRefresh; + + public SearchFieldsCache(CCBAPI ccbClient) { + this.ccbClient = ccbClient; + } + + public synchronized List getSearchFields() { + if (System.currentTimeMillis() - lastRefresh < REFRESH_INTERVAL_MS) { + LOG.debug("Using cached CCB fields"); + return cachedFields; + } + + try { + LOG.info("Fetching fields from CCB"); + + cachedFields = new ArrayList<>(); + final GetCustomFieldLabelsResponse labels = ccbClient.getCustomFieldLabels(); + + cachedFields.add(new SearchField("groupType", "Group Type", getValues(LookupTableType.GROUP_TYPE))); + // TODO fields.add(new SearchField("campusId", "Campus", ...)); + cachedFields.add(new SearchField("meetingDay", "Day", getValues(LookupTableType.MEET_DAY))); + + + for (final CustomField field : labels.getCustomFields()) { + final LookupTableType type = getTypeFromString(field.getName()); + if (type != null) { + cachedFields.add(new SearchField(getSearchFieldIdForType(type), field.getLabel(), getValues(type))); + } + } + + cachedFields.add(new SearchField("childcare", "Childcare", + Arrays.asList(StringPair.of("true", "Yes"), StringPair.of("false", "No")))); + + lastRefresh = System.currentTimeMillis(); + return cachedFields; + + } catch (Exception e) { + LOG.error(e.getMessage()); + return null; + } + } + + private LookupTableType getTypeFromString(String name) { + switch (name) { + case "udf_grp_pulldown_1": + return LookupTableType.UDF_GRP_PULLDOWN_1; + case "udf_grp_pulldown_2": + return LookupTableType.UDF_GRP_PULLDOWN_2; + case "udf_grp_pulldown_3": + return LookupTableType.UDF_GRP_PULLDOWN_3; + default: + return null; + } + } + + private String getSearchFieldIdForType(LookupTableType type) { + switch (type) { + case UDF_GRP_PULLDOWN_1: + return "udf_1"; + case UDF_GRP_PULLDOWN_2: + return "udf_2"; + case UDF_GRP_PULLDOWN_3: + return "udf_3"; + default: + throw new IllegalArgumentException(); + } + } + + private List getValues(LookupTableType type) throws IOException { + final GetLookupTableRequest lookupTableRequest = new GetLookupTableRequest().withType(type); + final GetLookupTableResponse lookupTable = ccbClient.getLookupTable(lookupTableRequest); + + return lookupTable.getItems() + .stream() + .map(entry -> StringPair.of(String.valueOf(entry.getId()), entry.getName())) + .collect(Collectors.toList()); + } +} diff --git a/src/main/java/com/p4square/groupsindexer/SyncGroups.java b/src/main/java/com/p4square/groupsindexer/SyncGroups.java new file mode 100644 index 0000000..f81cb98 --- /dev/null +++ b/src/main/java/com/p4square/groupsindexer/SyncGroups.java @@ -0,0 +1,140 @@ +package com.p4square.groupsindexer; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.ScheduledEvent; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.p4square.ccbapi.CCBAPI; +import com.p4square.ccbapi.CCBAPIClient; +import com.p4square.ccbapi.model.GetGroupProfilesRequest; +import com.p4square.ccbapi.model.GetGroupProfilesResponse; +import com.p4square.ccbapi.model.GroupProfile; +import com.p4square.ccbapi.model.InteractionType; +import com.p4square.groupsindexer.model.GroupSearchDocument; +import com.p4square.groupsindexer.model.GroupSearchDocumentAdapter; +import com.p4square.groupsindexer.model.GroupsCollection; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URL; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +/** + * SyncGroups is a scheduled lambda which syncs groups data from CCB. + * + * Required (custom) environment variables: + *
    + *
  • CCBAPIURL
    • + *
  • CCBAPIUser
    • + *
  • CCBAPIPassword
    • + *
  • OUTPUT_BUCKET
    • + *
+ * + */ +public class SyncGroups implements RequestHandler { + + private static final Logger LOG = LogManager.getLogger(SyncGroups.class); + private static final GroupSearchDocumentAdapter ADAPTER = new GroupSearchDocumentAdapter(); + private static final ObjectMapper MAPPER = new ObjectMapper(); + static { + MAPPER.registerModule(new JavaTimeModule()); + MAPPER.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + } + + private final String outputBucket; + private final String baseUrl; + + private final CCBAPI ccbClient; + private final AmazonS3 s3Client; + + private final SearchFieldsCache searchFieldsCache; + + public SyncGroups() throws Exception { + // Setup CCB Client + final String CCBAPIURL = System.getenv("CCBAPIURL"); + final String CCBAPIUser = System.getenv("CCBAPIUser"); + final String CCBAPIPassword = System.getenv("CCBAPIPassword"); + ccbClient = new CCBAPIClient(new URI(CCBAPIURL), CCBAPIUser, CCBAPIPassword); + searchFieldsCache = new SearchFieldsCache(ccbClient); + + // Setup S3 Client + outputBucket = System.getenv("OUTPUT_BUCKET"); + s3Client = AmazonS3ClientBuilder.defaultClient(); + + // Prefix to prepend to image urls. + baseUrl = System.getenv("BASE_URL"); + } + + @Override + public String handleRequest(ScheduledEvent s3Event, Context context) { + try { + final GroupsCollection groupsCollection = new GroupsCollection(); + groupsCollection.setLastUpdated(Instant.now()); + groupsCollection.setSearchFields(searchFieldsCache.getSearchFields()); + + final GetGroupProfilesResponse response = ccbClient.getGroupProfiles( + new GetGroupProfilesRequest() + .withIncludeImageUrl(true) + .withIncludeParticipants(false)); + + final List groups = new ArrayList<>(); + + for (GroupProfile profile : response.getGroups()) { + if (!profile.isActive() || + !profile.isPublicSearchListed() || + profile.getInteractionType() != InteractionType.MEMBERS_INTERACT) { + LOG.info("Skipping inactive/unlisted group " + profile.getName()); + continue; + } + + // Transform GroupProfile to Search Document. + final GroupSearchDocument document = ADAPTER.apply(profile); + + // Save GroupProfile image. + document.setImageUrl(null); + if (profile.getImageUrl() != null && !profile.getImageUrl().isEmpty()) { + final String imageKey = "group-images/group-" + profile.getId(); + InputStream in = null; + try { + final URL imageUrl = new URL(profile.getImageUrl()); + in = imageUrl.openStream(); + s3Client.putObject(outputBucket, imageKey, in, null); + document.setImageUrl(baseUrl + "/" + imageKey); + } catch (Exception e) { + LOG.error("Failed to upload image for group " + profile.getId(), e); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + // Ignore + } + } + } + } + + groups.add(document); + } + + // Save the groups data + groupsCollection.setGroups(groups); + s3Client.putObject(outputBucket, "data/groups-data.json", MAPPER.writeValueAsString(groupsCollection)); + + LOG.info("Updated search index. Found " + response.getGroups().size() + " groups."); + return "ok"; + + } catch (IOException e) { + LOG.error("Unexpected Exception: " + e.getMessage(), e); + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/com/p4square/groupsindexer/UpdateIndexes.java b/src/main/java/com/p4square/groupsindexer/UpdateIndexes.java deleted file mode 100644 index e4937e1..0000000 --- a/src/main/java/com/p4square/groupsindexer/UpdateIndexes.java +++ /dev/null @@ -1,150 +0,0 @@ -package com.p4square.groupsindexer; - -import com.amazonaws.auth.AWS4Signer; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.http.AWSRequestSigningApacheInterceptor; -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.ScheduledEvent; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.p4square.ccbapi.CCBAPI; -import com.p4square.ccbapi.CCBAPIClient; -import com.p4square.ccbapi.model.*; -import com.p4square.groupsindexer.model.GroupSearchDocument; -import com.p4square.groupsindexer.model.GroupSearchDocumentAdapter; -import org.apache.http.HttpHost; -import org.apache.http.HttpRequestInterceptor; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.client.Requests; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.xcontent.XContentType; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URL; - -/** - * UpdateIndexes is a scheduled lambda which populates the groups search index. - * - * Required (custom) environment variables: - *
    - *
  • CCBAPIURL
    • - *
  • CCBAPIUser
    • - *
  • CCBAPIPassword
    • - *
  • ES_URL
    • - *
  • IMAGE_BUCKET
    • - *
- * - */ -public class UpdateIndexes implements RequestHandler { - - private static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); - - private static final Logger LOG = LogManager.getLogger(UpdateIndexes.class); - private static final GroupSearchDocumentAdapter ADAPTER = new GroupSearchDocumentAdapter(); - private static final ObjectMapper MAPPER = new ObjectMapper(); - - private final String imageBucket; - - private final CCBAPI ccbClient; - private final RestHighLevelClient esClient; - private final AmazonS3 s3Client; - - public UpdateIndexes() throws Exception { - // Setup CCB Client - final String CCBAPIURL = System.getenv("CCBAPIURL"); - final String CCBAPIUser = System.getenv("CCBAPIUser"); - final String CCBAPIPassword = System.getenv("CCBAPIPassword"); - ccbClient = new CCBAPIClient(new URI(CCBAPIURL), CCBAPIUser, CCBAPIPassword); - - // Setup ElasticSearch client - final String ES_URL = System.getenv("ES_URL"); - AWS4Signer signer = new AWS4Signer(); - signer.setServiceName("es"); - signer.setRegionName(System.getenv("AWS_DEFAULT_REGION")); - HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(signer.getServiceName(), signer, credentialsProvider); - - esClient = new RestHighLevelClient(RestClient - .builder(HttpHost.create(ES_URL)) - .setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor))); - - // Setup S3 Client - imageBucket = System.getenv("IMAGE_BUCKET"); - s3Client = AmazonS3ClientBuilder.defaultClient(); - } - - @Override - public String handleRequest(ScheduledEvent s3Event, Context context) { - try { - GetGroupProfilesResponse response = ccbClient.getGroupProfiles( - new GetGroupProfilesRequest() - .withIncludeImageUrl(true) - .withIncludeParticipants(false)); - - final BulkRequest indexRequest = new BulkRequest(); - - for (GroupProfile profile : response.getGroups()) { - if (!profile.isActive() || - !profile.isPublicSearchListed() || - profile.getInteractionType() != InteractionType.MEMBERS_INTERACT) { - LOG.info("Skipping inactive/unlisted group " + profile.getName()); - continue; - } - - // Transform GroupProfile to Search Document. - final GroupSearchDocument document = ADAPTER.apply(profile); - - // Save GroupProfile image. - document.setImageUrl(null); - if (profile.getImageUrl() != null && !profile.getImageUrl().isEmpty()) { - final String imageKey = "group-images/group-" + profile.getId(); - InputStream in = null; - try { - final URL imageUrl = new URL(profile.getImageUrl()); - in = imageUrl.openStream(); - s3Client.putObject(imageBucket, imageKey, in, null); - document.setImageUrl(imageKey); - } catch (Exception e) { - LOG.error("Failed to upload image for group " + profile.getId(), e); - } finally { - if (in != null) { - try { - in.close(); - } catch (IOException e) { - // Ignore - } - } - } - } - - // Add request to batch. - indexRequest.add(Requests - .indexRequest("groups") - .type("group") - .id(String.valueOf(document.getId())) - .source(MAPPER.writeValueAsString(document), XContentType.JSON)); - } - - BulkResponse esResponse = esClient.bulk(indexRequest); - - if (esResponse.hasFailures()) { - throw new RuntimeException(esResponse.buildFailureMessage()); - } - - LOG.info("Updated search index. Found " + response.getGroups().size() + " groups."); - return "ok"; - - } catch (IOException e) { - LOG.error("Unexpected Exception: " + e.getMessage(), e); - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java b/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java index e336cb8..5708c4f 100644 --- a/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java +++ b/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocument.java @@ -29,9 +29,6 @@ public class GroupSearchDocument { @JsonProperty("leader-name") private String leaderName; - @JsonProperty("leader-email") - private String leaderEmail; - @JsonProperty("member-count") private int currentMembers; @@ -104,14 +101,6 @@ public class GroupSearchDocument { this.leaderName = leaderName; } - public String getLeaderEmail() { - return leaderEmail; - } - - public void setLeaderEmail(String leaderEmail) { - this.leaderEmail = leaderEmail; - } - public String getImageUrl() { return imageUrl; } diff --git a/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java b/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java index ebe852d..2a9a089 100644 --- a/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java +++ b/src/main/java/com/p4square/groupsindexer/model/GroupSearchDocumentAdapter.java @@ -19,7 +19,6 @@ public class GroupSearchDocumentAdapter implements Function groups; + + @JsonProperty("search-fields") + private List searchFields; + + public Instant getLastUpdated() { + return lastUpdated; + } + + public void setLastUpdated(Instant lastUpdated) { + this.lastUpdated = lastUpdated; + } + + public List getGroups() { + return groups; + } + + public void setGroups(List groups) { + this.groups = groups; + } + + public List getSearchFields() { + return searchFields; + } + + public void setSearchFields(List searchFields) { + this.searchFields = searchFields; + } +} -- cgit v1.2.3