Intermediate commit: Fixed a bug with mapping definition during index

creation

src/main/java/at/ac/uibk/gitsearch/config/CacheConfiguration.java

@@ -52,6 +52,7 @@ public class CacheConfiguration {
             createCache(cm, at.ac.uibk.gitsearch.repository.jpa.UserRepository.USERS_BY_EMAIL_CACHE);
             createCache(cm, EditorialPagesService.PAGES_CACHE);
             createCache(cm, MetaDataRepository.EXERCISE_BY_ID_CACHE);
+            createCache(cm, MetaDataRepository.EXERCISECHILDREN_BY_ID_CACHE);
 
             createCache(cm, at.ac.uibk.gitsearch.domain.User.class.getName());
             createCache(cm, at.ac.uibk.gitsearch.domain.Authority.class.getName());

src/main/java/at/ac/uibk/gitsearch/repository/search/ElasticSearchRepository.java

@@ -4,7 +4,6 @@ import at.ac.uibk.gitsearch.repository.jpa.StatisticsRepository;
 import at.ac.uibk.gitsearch.service.dto.StatisticsDTO;
 import co.elastic.clients.elasticsearch.ElasticsearchClient;
 import co.elastic.clients.elasticsearch._types.ElasticsearchException;
-import co.elastic.clients.elasticsearch._types.mapping.TypeMapping;
 import co.elastic.clients.elasticsearch._types.query_dsl.FieldAndFormat;
 import co.elastic.clients.elasticsearch._types.query_dsl.MatchPhrasePrefixQuery;
 import co.elastic.clients.elasticsearch._types.query_dsl.MatchQuery;
@@ -180,18 +179,7 @@ public class ElasticSearchRepository {
     }
 
     /**
-     * create an Index
-     *
-     * @param indexName the name of the index
-     * @param mapping   the type mapping
-     * @throws IOException
-     */
-    public void createIndex(final String indexName, final TypeMapping mapping) throws IOException {
-        this.elasticsearchAPIClient.indices().create(createIndexRequest -> createIndexRequest.index(indexName).mappings(mapping));
-    }
-
-    /**
-     * adds/updates searchstatistics to the current doc, if it is existing
+     * adds/updates search statistics to the current doc, if it is existing
      *
      * @param doc
      */
@@ -250,11 +238,13 @@ public class ElasticSearchRepository {
      * create an Index
      *
      * @param indexName the name of the index
-     * @param mapping   the type mapping
      * @throws IOException
      */
     public void createMetadataIndex(final String indexName) throws IOException {
-        createIndex(indexName, getMetaDataConfigDefinition());
+        try (InputStream metaDataStream = getMetadataIndexMapping()) {
+            this.elasticsearchAPIClient.indices()
+                .create(createIndexRequest -> createIndexRequest.index(indexName).withJson(metaDataStream));
+        }
     }
 
     /**
@@ -266,14 +256,6 @@ public class ElasticSearchRepository {
         return ElasticSearchRepository.class.getResourceAsStream("./es_metadata.schema.json");
     }
 
-    private TypeMapping getMetaDataConfigDefinition() throws IOException {
-        try (InputStream metaDataStream = getMetadataIndexMapping()) {
-            var mapper = this.elasticsearchAPIClient._transport().jsonpMapper();
-            var parser = mapper.jsonProvider().createParser(metaDataStream);
-            return TypeMapping._DESERIALIZER.deserialize(parser, mapper);
-        }
-    }
-
     public String createNewMetaDataIndex() throws IOException {
         final List<IndicesRecord> indices = getIndices();
 

src/main/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepository.java

@@ -4,11 +4,14 @@ import at.ac.uibk.gitsearch.domain.Authority;
 import at.ac.uibk.gitsearch.domain.Statistics;
 import at.ac.uibk.gitsearch.domain.util.TreeNode;
 import at.ac.uibk.gitsearch.repository.jpa.StatisticsRepository;
+import at.ac.uibk.gitsearch.service.SearchService;
+import at.ac.uibk.gitsearch.service.SearchService.ChildInfo;
 import at.ac.uibk.gitsearch.service.dto.AutoCompleteEntry;
 import at.ac.uibk.gitsearch.service.dto.StatisticsDTO;
 import co.elastic.clients.elasticsearch.ElasticsearchClient;
 import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
 import co.elastic.clients.elasticsearch._types.query_dsl.ExistsQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.FieldAndFormat;
 import co.elastic.clients.elasticsearch._types.query_dsl.MatchPhrasePrefixQuery;
 import co.elastic.clients.elasticsearch._types.query_dsl.MatchQuery;
 import co.elastic.clients.elasticsearch._types.query_dsl.MultiMatchQuery;
@@ -24,6 +27,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import java.io.IOException;
 import java.net.ConnectException;
@@ -101,6 +105,9 @@ public class MetaDataRepository {
 
     public static final String EXERCISE_BY_ID_CACHE = "at.ac.uibk.gitsearch.repository.search.MetaDataRepository.ExerciseByIdCache";
 
+    public static final String EXERCISECHILDREN_BY_ID_CACHE =
+        "at.ac.uibk.gitsearch.repository.search.MetaDataRepository.ExerciseChildrenByIdCache";
+
     private static final String SCOPE = "SCOPE";
 
     @SuppressWarnings("PMD.UnusedPrivateField") // just in case we want to reenable highlighter
@@ -1041,6 +1048,72 @@ public class MetaDataRepository {
         return getExerciseById(exerciseId.toString(), user);
     }
 
+    /**
+     * Searches for an exercise by id just a convenience method
+     *
+     * @param exerciseId the parsed exercise id
+     * @param user       the user (if authenticated)
+     * @return search result
+     * @throws NotFoundException if not found
+     */
+    @Cacheable(value = EXERCISECHILDREN_BY_ID_CACHE, key = "((#user.present)?#user.get().username:'') + #exerciseId.toString()")
+    public List<SearchService.ChildInfo> getChildInfos(ExerciseId exerciseId, Optional<User> user) {
+        String parentId = exerciseId.toString();
+
+        BoolQuery.Builder queryBuilder = new BoolQuery.Builder();
+
+        queryBuilder.filter(fq -> fq.term(tq -> tq.field(SearchRepositoryConstants.FILE_PARENTID).value(parentId)));
+
+        //        queryBuilder.must(q -> q.matchPhrase(mq -> mq.field(SearchRepositoryConstants.FILE_PARENTID).query(parentId)));
+        //        queryBuilder.must(q -> q.match(mq -> mq.field(SearchRepositoryConstants.FILE_PARENTID).query(parentId)));
+
+        //        addAuthorizationQueryWithJavaApi(user, queryBuilder);
+
+        try {
+            final BoolQuery query = queryBuilder.build();
+            var relevantFields = List.of(FieldAndFormat.of(ff -> ff.field(SearchRepositoryConstants.METADATA_TITLE)));
+            LOGGER.info("ElasticSearch Query using Java Client API:\n{}", query);
+            co.elastic.clients.elasticsearch.core.SearchResponse<ObjectNode> searchResponse = elasticsearchAPIClient.search(
+                search ->
+                    search
+                        .index(SearchRepositoryConstants.INDEX_METADATA)
+                        .fields(relevantFields)
+                        .query(q -> q.bool(query))
+                        .from(0)
+                        .size(100),
+                ObjectNode.class
+            );
+            return searchResponse.hits().hits().stream().map(this::toChildInfo).collect(Collectors.toList());
+        } catch (ElasticsearchException ese) {
+            LOGGER.warn("Exception in elastic search", ese);
+            throw ese;
+        } catch (IOException ce) {
+            @SuppressWarnings("PMD.CloseResource")
+            final RestClientTransport esTransport = (RestClientTransport) elasticsearchAPIClient._transport();
+            final String nodes = esTransport.restClient().getNodes().stream().map(Object::toString).collect(Collectors.joining(", "));
+            LOGGER.warn(CONNECTION_EXCEPTION_LOGTEMPLATE, nodes, ce);
+            throw new NotFoundException("Childrens from parent id " + exerciseId + " not found", ce);
+        }
+    }
+
+    /**
+     * just manually converts the elasticSearch hit to child info
+     *
+     * @param hit
+     * @return
+     */
+    private ChildInfo toChildInfo(Hit<ObjectNode> hit) {
+        try {
+            return new SearchService.ChildInfo(
+                ExerciseId.fromString(hit.id()),
+                hit.fields().get(SearchRepositoryConstants.METADATA_TITLE).to(String[].class)[0]
+            );
+        } catch (ParseException | IndexOutOfBoundsException e) {
+            LOGGER.warn("Cannot parse child info for {}", hit.id());
+            return null;
+        }
+    }
+
     /**
      * calculates recursively the list of elements in the collection. TODO make
      * more efficient by looking up all children in one go.

src/main/java/at/ac/uibk/gitsearch/service/SearchService.java

@@ -270,6 +270,41 @@ public class SearchService {
         }
     }
 
+    /**
+     * holds id and title for each child of an exercise
+     */
+    public static class ChildInfo {
+
+        ExerciseId childId;
+        String title;
+
+        public ChildInfo(ExerciseId childId, String title) {
+            super();
+            this.childId = childId;
+            this.title = title;
+        }
+
+        public ExerciseId getChildId() {
+            return childId;
+        }
+
+        public void setChildId(ExerciseId childId) {
+            this.childId = childId;
+        }
+
+        public String getTitle() {
+            return title;
+        }
+
+        public void setTitle(String title) {
+            this.title = title;
+        }
+    }
+
+    public List<ChildInfo> findChildrenByExerciseId(ExerciseId exerciseId) {
+        return metaDataRepository.getChildInfos(exerciseId, tokenProvider.getCurrentPrincipal());
+    }
+
     /**
      * Method used to filter out the Artemis export action for
      * non-authorized users having only partial access to the data.

src/test/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepositoryIT.java

 package at.ac.uibk.gitsearch.repository.search;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.hasProperty;
 import static org.hamcrest.Matchers.is;
@@ -10,6 +11,7 @@ import static org.junit.Assert.assertTrue;
 import at.ac.uibk.gitsearch.GitsearchApp;
 import at.ac.uibk.gitsearch.domain.util.TreeNode;
 import at.ac.uibk.gitsearch.repository.search.testESService.ElasticSearchTestConfiguration;
+import at.ac.uibk.gitsearch.service.SearchService.ChildInfo;
 import at.ac.uibk.gitsearch.service.dto.AutoCompleteEntry;
 import java.io.IOException;
 import java.text.ParseException;
@@ -194,7 +196,7 @@ public class MetaDataRepositoryIT {
     @Test
     @Timeout(value = 5, unit = TimeUnit.MINUTES) // just for debugging
     void getExerciseCollectionTree() throws IOException, ParseException {
-        String exerciseId = "[274]"; // should not exist
+        String exerciseId = "[274]"; // should exist and contains children
         ExerciseId id = ExerciseId.fromString(exerciseId);
 
         final TreeNode<SearchResultDTO> exerciseCollectionTree = metaDataRepository.getExerciseCollectionTree(id, Optional.empty());
@@ -209,4 +211,15 @@ public class MetaDataRepositoryIT {
             exerciseCollectionTree.getChildren().get(1).getChildren().size() == 2
         );
     }
+
+    @Test
+    @Timeout(value = 5, unit = TimeUnit.MINUTES) // just for debugging
+    void testGetChildInfos() throws ParseException {
+        String exerciseId = "[274]"; // should exist and contains children
+        ExerciseId id = ExerciseId.fromString(exerciseId);
+
+        final List<ChildInfo> exerciseCollectionTree = metaDataRepository.getChildInfos(id, Optional.empty());
+
+        assertThat(exerciseCollectionTree).isNotEmpty();
+    }
 }

src/test/java/at/ac/uibk/gitsearch/repository/search/testESService/ElasticSearchTestConfiguration.java

 package at.ac.uibk.gitsearch.repository.search.testESService;
 
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request;
+
 import at.ac.uibk.gitsearch.domain.util.TreeNode;
 import at.ac.uibk.gitsearch.repository.search.ElasticSearchRepository;
 import at.ac.uibk.gitsearch.repository.search.MetaDataRepository;
@@ -243,9 +245,6 @@ public class ElasticSearchTestConfiguration {
     }
 
     private void setUpMetaDataIndex(HttpHost hostURL) throws IOException {
-        HttpHeaders headers = new HttpHeaders();
-        headers.setContentType(MediaType.APPLICATION_JSON);
-
         String indexName = elasticSearchRepository.createNewMetaDataIndex();
         elasticSearchRepository.createAlias(indexName);
     }
@@ -257,10 +256,12 @@ public class ElasticSearchTestConfiguration {
         File rootData = new File(test.toURI());
         File metaDataTestDir = new File(rootData, "metaData");
 
+        List<SearchResultDTO> contentToIndex = new ArrayList<>();
+
         contentCount = 0;
         visitSubdirectories(
             metaDataTestDir,
-            (File metaDataFile, List<TreeNode<SearchResultDTO>> ancestors, String relativePath, GitProjectDTO gitProject) -> {
+            (File metaDataFile, List<TreeNode<SearchResultDTO>> descendants, String relativePath, GitProjectDTO gitProject) -> {
                 try {
                     final UserProvidedMetadataDTO userMetaData = parseMetaDataFile(metaDataFile);
 
@@ -292,12 +293,17 @@ public class ElasticSearchTestConfiguration {
                     file.setIndexing_date(Instant.now());
                     file.setCommit_id(gitProject.getCommit_id());
                     file.setChildren(
-                        ancestors.stream().map(node -> node.getData().getExerciseId()).collect(Collectors.toList()).toArray(new String[] {})
+                        descendants
+                            .stream()
+                            .map(node -> node.getData().getExerciseId())
+                            .collect(Collectors.toList())
+                            .toArray(new String[] {})
                     );
+                    descendants.forEach(node -> node.getData().getFile().setParentId(exerciseId.toString()));
                     toIndex.setFile(file);
 
                     toIndex.setExerciseId(exerciseId.toString());
-                    addContentToIndex(toIndex, exerciseId, elURL);
+                    contentToIndex.add(toIndex);
                     ElasticSearchTestConfiguration.this.contentCount++;
                     return toIndex;
                 } catch (IOException e) {
@@ -317,34 +323,34 @@ public class ElasticSearchTestConfiguration {
         //		    if(content==null) break;
         //	    }
 
-        waitForCleanStartUp(contentCount - 1);
-    }
+        indexSearchResults(contentToIndex, elURL);
 
-    private void addContentToIndex(SearchResultDTO content, ExerciseId id, HttpHost elURL)
-        throws JsonGenerationException, JsonMappingException, IOException {
-        ObjectMapper mapper = MetaDataRepository.getSearchResultObjectMapper();
-        StringWriter sw = new StringWriter();
-        mapper.writeValue(sw, content);
-        String contentString = sw.toString();
-        addContentToIndex(contentString, id.toString(), elURL);
+        elasticSearchRepository.waitForIndexingFinished();
+
+        waitForCleanStartUp(contentCount - 1); // TODO this is redundant now to waitForIndexingFinished
     }
 
     private static final Charset UTF8 = Charset.forName("UTF-8");
 
-    private void addContentToIndex(String content, String id, HttpHost elPortAndHost) {
-        RestTemplate restTemplate = new RestTemplate();
-        HttpHeaders headers = new HttpHeaders();
-        headers.setContentType(MediaType.APPLICATION_JSON);
-        HttpEntity<String> request = new HttpEntity<String>(content, headers);
-        String encodedId = URLEncoder.encode(id, UTF8);
-        encodedId = encodedId.replace("/", "%2F");
-        URI uri;
-        try {
-            uri = new URI(elPortAndHost.toURI() + "/metadata/_doc/" + encodedId);
-            restTemplate.put(uri, request);
-        } catch (URISyntaxException e) {
-            LOGGER.warn("Cannot add " + id, e);
-        }
+    private void indexSearchResults(List<SearchResultDTO> contents, HttpHost elPortAndHost) {
+        ObjectMapper mapper = MetaDataRepository.getSearchResultObjectMapper();
+        contents.forEach(sr -> {
+            try (StringWriter sw = new StringWriter()) {
+                mapper.writeValue(sw, sr);
+                String contentString = sw.toString();
+                RestTemplate restTemplate = new RestTemplate();
+                HttpHeaders headers = new HttpHeaders();
+                headers.setContentType(MediaType.APPLICATION_JSON);
+                HttpEntity<String> request = new HttpEntity<String>(contentString, headers);
+                String encodedId = URLEncoder.encode(sr.getExerciseId(), UTF8);
+                encodedId = encodedId.replace("/", "%2F");
+                URI uri;
+                uri = new URI(elPortAndHost.toURI() + "/metadata/_doc/" + encodedId);
+                restTemplate.put(uri, request);
+            } catch (URISyntaxException | IOException e) {
+                LOGGER.warn("Cannot add " + sr.getExerciseId(), e);
+            }
+        });
     }
 
     private ExerciseId calculateProjectId(String relativePath) {