This is the codeAbility Sharing Platform! Learn more about the codeAbility Sharing Platform.

Skip to content
Snippets Groups Projects
Commit 017316a7 authored by Michael Breu's avatar Michael Breu :speech_balloon:
Browse files

Erweiterung für Autocompletion

parent 5a92f116
Branches
Tags
2 merge requests!17Initial Merge to Prepare Release 1.0.0,!1Resolve "Metadaten konsolideren"
Showing
with 270 additions and 103 deletions
......@@ -2,7 +2,6 @@ package at.ac.uibk.gitsearch.repository.search;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
......@@ -11,15 +10,15 @@ import java.util.Set;
import java.util.StringTokenizer;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.springframework.stereotype.Repository;
......@@ -31,7 +30,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import at.ac.uibk.gitsearch.config.ApplicationProperties;
import at.ac.uibk.gitsearch.service.dto.GitFilesAggregationDTO;
import at.ac.uibk.gitsearch.service.dto.SearchInputDTO;
import at.ac.uibk.gitsearch.service.dto.SearchResultsDTO;
import at.ac.uibk.gitsearch.service.dto.SearchResultsDTO.Person;
......@@ -82,6 +80,17 @@ public class MetaDataRepository {
return getFieldAutoCompletion(keyWordPrefix, SearchRepositoryConstants.METADATA_KEYWORDS);
}
/**
* returns all programmingLanguage autocompletes for keyWord
*
* @param progammingLanguagePrefix
* @return
* @throws IOException
*/
public List<String> getProgrammingLanguageAutoComplete(String progammingLanguagePrefix) throws IOException {
return getFieldAutoCompletion(progammingLanguagePrefix, SearchRepositoryConstants.METADATA_PROGRAMMING_LANGUAGES);
}
/**
* returns all creator autocompletes
*
......@@ -139,9 +148,6 @@ public class MetaDataRepository {
for (Person creator : entry.getMetadata().getCreator())
cachedCompletions.get(SearchRepositoryConstants.METADATA_CREATOR)
.addAll(split(creator.getName()));
System.out.println(String.format("Title: %s, Description %s", entry.getMetadata().getTitle(),
entry.getMetadata().getDescription()));
}
}
}
......@@ -159,15 +165,37 @@ public class MetaDataRepository {
fillAutoCompletion();
SearchRequest searchRequest = new SearchRequest(SearchRepositoryConstants.INDEX_METADATA);
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery()
.must(QueryBuilders.simpleQueryStringQuery(searchInputDTO.getFulltextQuery())
.field(SearchRepositoryConstants.METADATA_DESCRIPTION)
.field(SearchRepositoryConstants.METADATA_KEYWORDS)
.field(SearchRepositoryConstants.METADATA_TITLE));
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery()
// .must(QueryBuilders.simpleQueryStringQuery(searchInputDTO.getFulltextQuery())
// .field(SearchRepositoryConstants.METADATA_DESCRIPTION)
// .field(SearchRepositoryConstants.METADATA_KEYWORDS)
// .field(SearchRepositoryConstants.METADATA_TITLE));
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
if(!StringUtils.isEmpty(searchInputDTO.getFulltextQuery()))
queryBuilder.must(QueryBuilders.multiMatchQuery(
searchInputDTO.getFulltextQuery(),
SearchRepositoryConstants.METADATA_DESCRIPTION,
SearchRepositoryConstants.METADATA_KEYWORDS,
SearchRepositoryConstants.METADATA_TITLE)
.type(MultiMatchQueryBuilder.Type.PHRASE_PREFIX)
);
if(!StringUtils.isEmpty(searchInputDTO.getMetadataProgrammingLanguage())) {
queryBuilder.must(QueryBuilders.simpleQueryStringQuery(searchInputDTO.getMetadataProgrammingLanguage()).field(SearchRepositoryConstants.METADATA_PROGRAMMING_LANGUAGES));
}
if(!StringUtils.isEmpty(searchInputDTO.getMetadataKeywords())) {
queryBuilder.must(QueryBuilders.simpleQueryStringQuery(searchInputDTO.getMetadataKeywords()).field(SearchRepositoryConstants.METADATA_KEYWORDS));
}
searchInputDTO.setSelectedRepository(new ArrayList<String>());
searchInputDTO.setSelectedFileFormat(new ArrayList<String>());
if(!StringUtils.isEmpty(searchInputDTO.getMetadataAuthor())) {
queryBuilder.must(QueryBuilders.simpleQueryStringQuery(searchInputDTO.getMetadataAuthor()).field(SearchRepositoryConstants.METADATA_CREATOR));
}
if(!StringUtils.isEmpty(searchInputDTO.getMetadataLicense())) {
queryBuilder.must(QueryBuilders.simpleQueryStringQuery(searchInputDTO.getMetadataLicense()).field(SearchRepositoryConstants.METADATA_LICENSE));
}
char[] boundaryChars = {'\n'};
......@@ -209,79 +237,4 @@ public class MetaDataRepository {
return results;
}
private static void addFulltextSelection(BoolQueryBuilder queryBuilder, List<String> selectedFileFormat,
String fileExtensionField) {
if (selectedFileFormat.isEmpty()) {
return;
}
queryBuilder.must(QueryBuilders.termsQuery(fileExtensionField, selectedFileFormat));
}
public GitFilesAggregationDTO aggregation(String query) throws IOException {
SearchRequest searchRequest = new SearchRequest(SearchRepositoryConstants.INDEX_FULLTEXT);
BoolQueryBuilder matchQueryBuilder = QueryBuilders.boolQuery()
.must(QueryBuilders.simpleQueryStringQuery(query).field(SearchRepositoryConstants.FULLTEXT_CONTENT));
TermsAggregationBuilder groupSubGroup = AggregationBuilders
.terms(SearchRepositoryConstants.FULLTEXT_SUB_GROUP_AGG)
.field(SearchRepositoryConstants.FULLTEXT_SUB_GROUP);
TermsAggregationBuilder groupFileExtension = AggregationBuilders
.terms(SearchRepositoryConstants.FULLTEXT_FILE_EXTENSION_AGG)
.field(SearchRepositoryConstants.FULLTEXT_FILE_EXTENSION);
TermsAggregationBuilder groupRepository = AggregationBuilders
.terms(SearchRepositoryConstants.FULLTEXT_REPOSITORY_AGG)
.field(SearchRepositoryConstants.FULLTEXT_REPOSITORY);
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(matchQueryBuilder).size(0)
.aggregation(groupSubGroup).aggregation(groupFileExtension).aggregation(groupRepository);
searchRequest.source(sourceBuilder);
SearchResponse searchResponse = elasticsearchClient.search(searchRequest, RequestOptions.DEFAULT);
return new GitFilesAggregationDTO(searchResponse.getAggregations());
}
private Collection<String> searchProjectsIDsMetadata(SearchInputDTO searchInputDTO) throws IOException {
if (!searchInputDTO.hasMetadataInput()) {
return null;
}
SearchRequest searchRequest = new SearchRequest(SearchRepositoryConstants.INDEX_METADATA);
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
addMetadataQueryShould(queryBuilder, searchInputDTO.getMetadataProgrammingLanguage(),
SearchRepositoryConstants.METADATA_PROGRAMMING_LANGUAGES);
addMetadataQueryShould(queryBuilder, searchInputDTO.getMetadataKeywords(),
SearchRepositoryConstants.METADATA_KEYWORDS);
addMetadataQueryMust(queryBuilder, searchInputDTO.getMetadataAuthor(),
SearchRepositoryConstants.METADATA_CREATOR);
addMetadataQueryMust(queryBuilder, searchInputDTO.getMetadataLicense(),
SearchRepositoryConstants.METADATA_LICENSE);
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(queryBuilder)
.fetchSource(SearchRepositoryConstants.METADATA_PROJECT_ID, null);
// TODO: Allow more than 10 entries.
searchRequest.source(sourceBuilder);
SearchResponse searchResponse = elasticsearchClient.search(searchRequest, RequestOptions.DEFAULT);
Collection<String> buckets = new ArrayList<>();
for (SearchHit searchHit : searchResponse.getHits().getHits()) {
buckets.add(searchHit.getId());
}
return buckets;
}
private static void addMetadataQueryShould(BoolQueryBuilder queryBuilder, String value, String field) {
if (value == null) {
return;
}
queryBuilder.should(QueryBuilders.termsQuery(field, value));
}
private static void addMetadataQueryMust(BoolQueryBuilder queryBuilder, String value, String field) {
if (value == null) {
return;
}
queryBuilder.must(QueryBuilders.simpleQueryStringQuery(value).analyzeWildcard(true).field(field));
}
}
......@@ -15,6 +15,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import at.ac.uibk.gitsearch.repository.search.MetaDataRepository;
import at.ac.uibk.gitsearch.repository.search.SearchRepositoryConstants;
import at.ac.uibk.gitsearch.service.dto.SearchInputDTO;
import at.ac.uibk.gitsearch.service.dto.SearchResultsDTO;
import at.ac.uibk.gitsearch.service.dto.SearchResultsDTO.SearchResultDTO;
......@@ -37,6 +38,50 @@ public class SearchService {
public SearchService() {
}
/**
* returns all keyword autocompletes for keyWord
*
* @param keyWordPrefix
* @return
* @throws IOException
*/
public List<String> getKeywordsAutoComplete(String keyWordPrefix) throws IOException {
return metaDataRepository.getKeywordsAutoComplete(keyWordPrefix);
}
/**
* returns all creator autocompletes
*
* @param creatorPrefix
* @return
* @throws IOException
*/
public List<String> getCreatorAutoComplete(String creatorPrefix) throws IOException {
return metaDataRepository.getKeywordsAutoComplete(creatorPrefix);
}
/**
* returns all contributor autocompletes
*
* @param contributorPrefix
* @return
* @throws IOException
*/
public List<String> getContributorAutoComplete(String contributorPrefix) throws IOException {
return metaDataRepository.getKeywordsAutoComplete(contributorPrefix);
}
/**
* returns all programmingLanguage autocompletes for keyWord
*
* @param progammingLanguagePrefix
* @return
* @throws IOException
*/
public List<String> getProgrammingLanguageAutoComplete(String progammingLanguagePrefix) throws IOException {
return metaDataRepository.getProgrammingLanguageAutoComplete(progammingLanguagePrefix);
}
/**
* currently just a dummy implementation. TODO: first, and length is redundant
......
package at.ac.uibk.gitsearch.web.rest;
import java.io.IOException;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import at.ac.uibk.gitsearch.es.model.DocumentInfo;
import at.ac.uibk.gitsearch.repository.search.SearchRepositoryConstants;
import at.ac.uibk.gitsearch.service.SearchService;
import at.ac.uibk.gitsearch.service.dto.SearchInputDTO;
import at.ac.uibk.gitsearch.service.dto.SearchResultsDTO;
......@@ -44,5 +48,54 @@ public class SearchResource {
log.debug("REST request to search {}", query);
return searchService.searchResultPage(query,SearchInputDTO.PAGE_SIZE*query.getPage(), SearchInputDTO.PAGE_SIZE);
}
/**
* returns all keyword autocompletes for keyWord
*
* @param keyWordPrefix
* @return
* @throws IOException
*/
@GetMapping("/search/keywordsAutoComplete")
public List<String> getKeywordsAutoComplete(@RequestParam String keyWordPrefix) throws IOException {
return searchService.getKeywordsAutoComplete(keyWordPrefix);
}
/**
* returns all creator autocompletes
*
* @param creatorPrefix
* @return
* @throws IOException
*/
@GetMapping("/search/creatorAutoComplete")
public List<String> getCreatorAutoComplete(@RequestParam String creatorPrefix) throws IOException {
return searchService.getKeywordsAutoComplete(creatorPrefix);
}
/**
* returns all contributor autocompletes
*
* @param contributorPrefix
* @return
* @throws IOException
*/
@GetMapping("/search/contributorAutoComplete")
public List<String> getContributorAutoComplete(@RequestParam String contributorPrefix) throws IOException {
return searchService.getKeywordsAutoComplete(contributorPrefix);
}
/**
* returns all programmingLanguage autocompletes for keyWord
*
* @param progammingLanguagePrefix
* @return
* @throws IOException
*/
@GetMapping("/search/programmingLanguageAutoComplete")
public List<String> getProgrammingLanguageAutoComplete(@RequestParam String progammingLanguagePrefix) throws IOException {
return searchService.getProgrammingLanguageAutoComplete(progammingLanguagePrefix);
}
}
......@@ -29,7 +29,13 @@ import { QueryParamModule } from '@ngqp/core';
GitSearchV2AppRoutingModule,
QueryParamModule,
],
declarations: [MainComponent, NavbarComponent, ErrorComponent, PageRibbonComponent, ActiveMenuDirective, FooterComponent],
declarations: [
MainComponent,
NavbarComponent,
ErrorComponent,
PageRibbonComponent,
ActiveMenuDirective,
FooterComponent],
bootstrap: [MainComponent],
})
export class GitSearchV2AppModule {}
......@@ -33,6 +33,7 @@
<span jhiTranslate="global.messages.info.registration.disabled">Registration of new users is currently disabled. Please, contact the administrator if you want to test the system.</span>
</div>
</div>
<div><jhi-keywords-list></jhi-keywords-list></div>
</div>
</div>
......
import {NgModule} from '@angular/core';
import {RouterModule} from '@angular/router';
import { KeywordListComponent } from 'app/keyword-list/keyword-list.component';
import {GitSearchV2SharedModule} from 'app/shared/shared.module';
import {HOME_ROUTE} from './home.route';
......@@ -7,7 +8,7 @@ import {HomeComponent} from './home.component';
@NgModule({
imports: [GitSearchV2SharedModule, RouterModule.forChild([HOME_ROUTE])],
declarations: [HomeComponent],
declarations: [HomeComponent, KeywordListComponent],
})
export class GitSearchV2HomeModule {
}
<p>XXXX YYYY ZZZZ keyword-list works!</p>
<p *ngFor="let keyWord of keywords">{{keyWord}}</p>
import { Component, OnInit } from '@angular/core';
import { HttpResponse } from '@angular/common/http';
import { SearchService } from 'app/search/service/search-service';
@Component({
selector: 'jhi-keywords-list',
templateUrl: './keyword-list.component.html',
styleUrls: ['./keyword-list.component.scss']
})
export class KeywordListComponent implements OnInit {
private keywords: Array<String> = new Array<String>();
constructor(
private searchService: SearchService
) { }
ngOnInit(): void {
this.searchService.getKeywordsAutoComplete('')
.subscribe(
(data: Array<string>) => {
this.keywords = data;
}
,
() => {alert("Initializiation of keywords failed");}
);
}
}
import { Injectable } from '@angular/core';
import { HttpClient } from '@angular/common/http';
import { HttpClient, HttpResponse, HttpParams } from '@angular/common/http';
import { Observable } from 'rxjs';
import { SERVER_API_URL } from 'app/app.constants';
import { SearchResultsDTO } from 'app/shared/model/search/search-results-dto.model';
import { SearchInput } from 'app/shared/model/search-input.model';
type EntityResponseTypeAutoComplete = HttpResponse<Array<string>>;
@Injectable({ providedIn: 'root' })
export class SearchService {
public resourceSearchUrlPageDetails = SERVER_API_URL + 'api/search/page-details';
public resourceKeywordsAutoCompleteDetails = SERVER_API_URL + 'api/search/keywordsAutoComplete';
constructor(protected http: HttpClient) {}
searchPageDetails(searchInput: SearchInput): Observable<SearchResultsDTO> {
return this.http
.post<SearchResultsDTO>(this.resourceSearchUrlPageDetails, searchInput);
}
}
getKeywordsAutoComplete(prefix: string): Observable<Array<string>> {
const options: HttpParams = new HttpParams();
options.append("keyWordPrefix", prefix);
return this.http
.get<Array<string>>(this.resourceKeywordsAutoCompleteDetails,
{params: new HttpParams().set('keyWordPrefix', prefix) }
);
}
}
package at.ac.uibk.gitsearch.service;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.everyItem;
import static org.hamcrest.Matchers.hasItemInArray;
import static org.hamcrest.Matchers.hasProperty;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.MockitoAnnotations;
......@@ -18,28 +29,78 @@ public class SearchServiceIT {
@Autowired
private SearchService searchService;
@BeforeEach
public void setup() {
MockitoAnnotations.initMocks(this);
}
@Test
public void testTestData() throws Exception {
SearchInputDTO searchQuery = new SearchInputDTO("Wien", null, null, null, null, null, null, null, null, 0);
// first page
public void testFullTextSearch() throws Exception {
final String WIEN = "Wien";
SearchInputDTO searchQuery = new SearchInputDTO(WIEN, null, null, null, null, null, null, null, null, 0);
SearchResultsDTO searchResultPage = searchService.searchResultPage(searchQuery, 0, SearchInputDTO.PAGE_SIZE);
org.junit.Assert.assertNotNull(searchResultPage.getSearchResult());
org.junit.Assert.assertTrue("At least one test hit", searchResultPage.getHitCount() >= 1);
org.junit.Assert.assertThat(searchResultPage.getSearchResult(), everyItem(
anyOf(hasProperty("description", containsString(WIEN)),
hasProperty("title", containsString(WIEN))
// hasProperty("keywords", containsString("Wien"))
)));
}
@Test
public void testProgrammingLanguageSearch() throws Exception {
SearchInputDTO searchQuery = new SearchInputDTO(null, "JAVA", null, null, null, null, null, null, null, 0);
SearchResultsDTO searchResultPage = searchService.searchResultPage(searchQuery, 0, SearchInputDTO.PAGE_SIZE);
assertNotNull(searchResultPage.getSearchResult());
assertTrue("At least one test hit", searchResultPage.getHitCount() >= 1);
assertThat(searchResultPage.getSearchResult(), everyItem(
hasProperty("programmingLanguage",
hasItemInArray(containsString("JAVA")))
));
}
@Test
public void testCreatorSearch() throws IOException {
SearchInputDTO searchQuery = new SearchInputDTO(null, null, null, null, null, "Podlipnig", null, null, null, 0);
SearchResultsDTO searchResultPage = searchService.searchResultPage(searchQuery, 0, SearchInputDTO.PAGE_SIZE);
org.junit.Assert.assertNotNull(searchResultPage.getSearchResult());
org.junit.Assert.assertTrue("At least one test hit", searchResultPage.getHitCount() >= 1);
// org.junit.Assert.assertTrue("At least one test hit", searchResultPage.getSearchResult().size() == SearchInputDTO.PAGE_SIZE);
assertThat(searchResultPage.getSearchResult(), everyItem(
hasProperty("creator", hasItemInArray(hasProperty("name", containsString("Podlipnig"))))
));
}
@Test
public void testKeywordSearch() throws IOException {
SearchInputDTO searchQuery = new SearchInputDTO(null, null, "latex", null, null, null, null, null, null, 0);
SearchResultsDTO searchResultPage = searchService.searchResultPage(searchQuery, 0, SearchInputDTO.PAGE_SIZE);
// // last page
// searchResultPage = searchService.searchResultPage(searchQuery, (SearchService.NUM_TESTRESULTS / SearchInputDTO.PAGE_SIZE) * SearchInputDTO.PAGE_SIZE, SearchInputDTO.PAGE_SIZE);
// org.junit.Assert.assertNotNull(searchResultPage.getSearchResult());
// org.junit.Assert.assertTrue("At least one test hit", searchResultPage.getHitCount() == SearchService.NUM_TESTRESULTS);
// org.junit.Assert.assertTrue("At least one test hit", searchResultPage.getSearchResult().size() == SearchService.NUM_TESTRESULTS%10);
org.junit.Assert.assertNotNull(searchResultPage.getSearchResult());
org.junit.Assert.assertTrue("At least one test hit", searchResultPage.getHitCount() >= 1);
assertThat(searchResultPage.getSearchResult(), everyItem(
hasProperty("keyword", hasItemInArray(containsString("latex")))
));
}
@Test
public void testLicenseSearch() throws IOException {
SearchInputDTO searchQuery = new SearchInputDTO(null, null, null, null, "MIT", null, null, null, null, 0);
SearchResultsDTO searchResultPage = searchService.searchResultPage(searchQuery, 0, SearchInputDTO.PAGE_SIZE);
org.junit.Assert.assertNotNull(searchResultPage.getSearchResult());
org.junit.Assert.assertTrue("At least one test hit", searchResultPage.getHitCount() >= 1);
assertThat(searchResultPage.getSearchResult(), everyItem(
hasProperty("license", containsString("MIT"))
));
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment