diff --git a/pom.xml b/pom.xml
index 08e443d529b9736f1d25038289d6b438cff0dcd1..8c8e43180a378c8e378f43bcf2ab6fded310ecfd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -158,7 +158,7 @@
<dependency>
<groupId>org.codeability.sharing</groupId>
<artifactId>SharingPluginPlatformAPI</artifactId>
- <version>0.4.9</version>
+ <version>0.4.10-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springdoc</groupId>
diff --git a/src/main/docs/requirements-docs.txt b/src/main/docs/requirements-docs.txt
index d0d8a26e8921ec72e54667bb24c1a9bd89d7f084..bb4eeb5a9a7f3c2f7bdef38ab661c302584bc490 100644
--- a/src/main/docs/requirements-docs.txt
+++ b/src/main/docs/requirements-docs.txt
@@ -1,3 +1,3 @@
# docs
-Sphinx==3.2.1
-sphinx-rtd-theme==0.5.0
+Sphinx==7.1.2
+sphinx-rtd-theme==2.0.0
diff --git a/src/main/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepository.java b/src/main/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepository.java
index aa60a32e424e38d717ccae3cbc3ae96be1fd1789..91f8272463889c4825bd4c629bc39295fea12fcf 100644
--- a/src/main/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepository.java
+++ b/src/main/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepository.java
@@ -7,6 +7,10 @@ import at.ac.uibk.gitsearch.repository.jpa.StatisticsRepository;
import at.ac.uibk.gitsearch.service.dto.AutoCompleteEntry;
import at.ac.uibk.gitsearch.service.dto.StatisticsDTO;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.ScriptSortType;
+import co.elastic.clients.elasticsearch._types.SortOptions;
+import co.elastic.clients.elasticsearch._types.SortOptionsBuilders;
+import co.elastic.clients.elasticsearch._types.SortOrder;
import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.ExistsQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.MatchPhrasePrefixQuery;
@@ -56,10 +60,12 @@ import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import javax.annotation.PostConstruct;
import javax.ws.rs.NotFoundException;
+import liquibase.repackaged.org.apache.commons.text.StringSubstitutor;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codeability.sharing.plugins.api.search.SearchInputDTO;
+import org.codeability.sharing.plugins.api.search.SearchOrdering;
import org.codeability.sharing.plugins.api.search.SearchResultDTO;
import org.codeability.sharing.plugins.api.search.SearchResultsDTO;
import org.codeability.sharing.plugins.api.search.UserProvidedMetadataDTO.Person;
@@ -305,7 +311,7 @@ public class MetaDataRepository {
}
});
- reloadedCachedCompletions.values().forEach(c -> simplify(c));
+ reloadedCachedCompletions.values().forEach(this::simplify);
cachedCompletions = reloadedCachedCompletions;
} catch (co.elastic.clients.elasticsearch._types.ElasticsearchException | ElasticsearchException | ConnectException ex) {
// this may happen mainly during testing, when
@@ -330,7 +336,7 @@ public class MetaDataRepository {
entries.forEach((lowerCaseValue, upperCaseValues) -> {
final Optional<Entry<String, Integer>> max = upperCaseValues.entrySet().stream().max((e1, e2) -> e1.getValue() - e2.getValue());
max.ifPresent(maxEntry -> {
- int sum = upperCaseValues.entrySet().stream().mapToInt(o -> o.getValue()).sum();
+ int sum = upperCaseValues.entrySet().stream().mapToInt(Entry::getValue).sum();
entries.put(lowerCaseValue, Collections.singletonMap(maxEntry.getKey(), sum));
});
});
@@ -532,8 +538,6 @@ public class MetaDataRepository {
* parameters.
*
* @param searchInputDTO the query parameters.
- * @param pageSize the size of the page (i.e. the maximum of returned
- * hits).
* @param user the user that queries.
* @return the search hits.
* @throws IOException when search index is not available.
@@ -565,7 +569,13 @@ public class MetaDataRepository {
final BoolQuery query = queryBuilder.build();
// LOGGER.info("ElasticSearch Query using Java Client API:\n{}", query)
co.elastic.clients.elasticsearch.core.SearchResponse<SearchResultDTO> searchResponse = elasticsearchAPIClient.search(
- search -> search.index(SearchRepositoryConstants.INDEX_METADATA).query(q -> q.bool(query)).from(from).size(pageSize),
+ search ->
+ search
+ .index(SearchRepositoryConstants.INDEX_METADATA)
+ .query(q -> q.bool(query))
+ .sort(getSortStrategy(searchInputDTO.getOrdering()))
+ .from(from)
+ .size(pageSize),
SearchResultDTO.class
);
@@ -583,6 +593,48 @@ public class MetaDataRepository {
}
}
+ private List<SortOptions> getSortStrategy(SearchOrdering searchOrdering) {
+ String sortingScriptTemplate =
+ "double boost = 0;" +
+ "if (doc.containsKey('${badgeRewarded}') && doc['${badgeRewarded}'].size() > 0) {" +
+ " boost = boost + (doc['${badgeRewarded}'].value ? 1.0 : 0.0) * params.factorBadge;" +
+ "}" +
+ "if (doc.containsKey('${downloads}') && doc['${downloads}'].size() > 0) {" +
+ " boost = boost + doc['${downloads}'].value * params.factorDownloads;" +
+ "}" +
+ "if (doc.containsKey('${views}') && doc['${views}'].size() > 0) {" +
+ " boost = boost + doc['${views}'].value * params.factorViews;" +
+ "}" +
+ "return boost;";
+
+ Map<String, String> templateValueMapping = Map.of(
+ "badgeRewarded",
+ SearchRepositoryConstants.SEARCHSTATISTICS_BADGEREWARDED,
+ "downloads",
+ SearchRepositoryConstants.SEARCHSTATISTICS_DOWNLOADS,
+ "views",
+ SearchRepositoryConstants.SEARCHSTATISTICS_VIEWS
+ );
+
+ var scriptSort = SortOptionsBuilders.script(b ->
+ b
+ .type(ScriptSortType.Number)
+ .order(SortOrder.Desc)
+ .script(sb ->
+ sb.inline(isb ->
+ isb
+ .lang("painless")
+ .source(StringSubstitutor.replace(sortingScriptTemplate, templateValueMapping))
+ .params("factorBadge", JsonData.of(searchOrdering.getFactorBadge()))
+ .params("factorDownloads", JsonData.of(searchOrdering.getFactorDownloads()))
+ .params("factorViews", JsonData.of(searchOrdering.getFactorViews()))
+ )
+ )
+ );
+
+ return List.of(SortOptionsBuilders.score(s -> s.order(SortOrder.Desc)), scriptSort);
+ }
+
/**
* creates the query (builder) for the searchInput and the user authorization
*
@@ -774,12 +826,14 @@ public class MetaDataRepository {
*/
private void addAuthorizationQueryWithJavaApi(Optional<User> user, BoolQuery.Builder queryBuilder) {
// Authorization restrictions
+ // the boost value of 0 tells elastic to exclude these restrictions from the scoring process
final TermQuery.Builder simplePublicQuery = new TermQuery.Builder()
.value("public")
.field(SearchRepositoryConstants.PROJECT_VISIBILITY)
.boost(0.0f);
final ExistsQuery.Builder publicVisibilityQuery = new ExistsQuery.Builder()
- .field(SearchRepositoryConstants.METADATA_PUBLICVISIBILITY);
+ .field(SearchRepositoryConstants.METADATA_PUBLICVISIBILITY)
+ .boost(0.0f);
final BoolQuery.Builder publicQuery = new BoolQuery.Builder()
.should(q -> q.term(simplePublicQuery.build()))
.should(q -> q.exists(publicVisibilityQuery.build()));
diff --git a/src/main/java/at/ac/uibk/gitsearch/repository/search/SearchRepositoryConstants.java b/src/main/java/at/ac/uibk/gitsearch/repository/search/SearchRepositoryConstants.java
index f907f768fb5b0487507b0d376ecef38f7d53bfe0..3fe6f2cba967f71d44b2ec81b1e11aed6251560c 100644
--- a/src/main/java/at/ac/uibk/gitsearch/repository/search/SearchRepositoryConstants.java
+++ b/src/main/java/at/ac/uibk/gitsearch/repository/search/SearchRepositoryConstants.java
@@ -41,6 +41,10 @@ public final class SearchRepositoryConstants {
public static final String PROJECT_GROUPS = "project.groups";
public static final String PROJECT_LASTACTIVITYAT = "project.last_activity_at";
+ public static final String SEARCHSTATISTICS_VIEWS = "searchStatistics.views";
+ public static final String SEARCHSTATISTICS_DOWNLOADS = "searchStatistics.downloads";
+ public static final String SEARCHSTATISTICS_BADGEREWARDED = "searchStatistics.badgeRewarded";
+
private SearchRepositoryConstants() {
// just a list of constants
}
diff --git a/src/main/java/at/ac/uibk/gitsearch/service/GitlabService.java b/src/main/java/at/ac/uibk/gitsearch/service/GitlabService.java
index 0203392b5e1c6cc54fadf8817597d09f8e681a41..f558766a8b2d11b71a7ec862acaaeeb3363f6fbe 100644
--- a/src/main/java/at/ac/uibk/gitsearch/service/GitlabService.java
+++ b/src/main/java/at/ac/uibk/gitsearch/service/GitlabService.java
@@ -148,15 +148,20 @@ public class GitlabService {
RepositoryFile file;
try (GitLabApi gitLabApi = gitLabRepository.getGitLabApi(tokenProvider.getGitLabAccessInfo());) {
- file =
- gitLabApi
- .getRepositoryFileApi()
- .getFile(
- Long.valueOf(exercise.getProject().getProject_id()),
- exerciseId.extendPath(filePath),
- exercise.getFile().getCommit_id()
- );
- return new ByteArrayInputStream(file.getDecodedContentAsBytes());
+ if (exercise.getFile().getCommit_id() == null) {
+ log.warn("commit_id must not be null for {}", exerciseId);
+ return null;
+ } else {
+ file =
+ gitLabApi
+ .getRepositoryFileApi()
+ .getFile(
+ Long.valueOf(exercise.getProject().getProject_id()),
+ exerciseId.extendPath(filePath),
+ exercise.getFile().getCommit_id()
+ );
+ return new ByteArrayInputStream(file.getDecodedContentAsBytes());
+ }
} catch (GitLabApiException e) {
log.warn(
"GitlabAPI Exception when looking for {} {} {} with message {}",
diff --git a/src/main/java/at/ac/uibk/gitsearch/service/gitlab_events/GitlabEventService.java b/src/main/java/at/ac/uibk/gitsearch/service/gitlab_events/GitlabEventService.java
index ae56cd2ddf2307b0580fe8b730bd7a002f5e49ca..5e5ea5ee71e0adc27d29ce176898ccdf0943534b 100644
--- a/src/main/java/at/ac/uibk/gitsearch/service/gitlab_events/GitlabEventService.java
+++ b/src/main/java/at/ac/uibk/gitsearch/service/gitlab_events/GitlabEventService.java
@@ -47,8 +47,8 @@ import java.util.stream.Stream;
import javax.annotation.PostConstruct;
import javax.el.MethodNotFoundException;
import org.apache.commons.lang3.StringUtils;
+import org.codeability.sharing.plugins.api.search.GitProjectDTO;
import org.codeability.sharing.plugins.api.search.SearchResultDTO;
-import org.codeability.sharing.plugins.api.search.SearchResultDTO.GitProject;
import org.codeability.sharing.plugins.api.search.SearchResultDTO.MetadataFile;
import org.codeability.sharing.plugins.api.search.UserProvidedMetadataDTO;
import org.codeability.sharing.plugins.api.search.UserProvidedMetadataDTO.InteractivityTypeXXX;
@@ -63,7 +63,6 @@ import org.gitlab4j.api.models.ProjectSharedGroup;
import org.gitlab4j.api.models.RepositoryFile;
import org.gitlab4j.api.models.TreeItem;
import org.gitlab4j.api.models.User;
-import org.gitlab4j.api.models.Visibility;
import org.gitlab4j.api.systemhooks.GroupSystemHookEvent;
import org.gitlab4j.api.systemhooks.ProjectSystemHookEvent;
import org.gitlab4j.api.systemhooks.PushSystemHookEvent;
@@ -255,99 +254,6 @@ public class GitlabEventService {
}
}
- /**
- * just to add further meta data
- *
- * @author Micha
- *
- */
- @SuppressWarnings({ "PMD.MethodNamingConventions", "PMD.FormalParameterNamingConventions" })
- private static class ExtendedGitProject extends GitProject {
-
- private Visibility visibility;
- private List<String> users;
- private List<String> groups;
- private int star_count;
- private int open_issues_count;
- private int forks_count;
-
- private String description;
- private boolean archived;
-
- @SuppressWarnings({ "unused", "PMD.AvoidDuplicateLiterals" })
- public Visibility getVisibility() {
- return visibility;
- }
-
- public void setVisibility(Visibility visibility) {
- this.visibility = visibility;
- }
-
- @SuppressWarnings("unused")
- public List<String> getUsers() {
- return users;
- }
-
- public void setUsers(List<String> users) {
- this.users = users;
- }
-
- @SuppressWarnings("unused")
- public List<String> getGroups() {
- return groups;
- }
-
- public void setGroups(List<String> groups) {
- this.groups = groups;
- }
-
- @SuppressWarnings("unused")
- public int getStar_count() {
- return star_count;
- }
-
- @SuppressWarnings("unused")
- public void setStar_count(int star_count) {
- this.star_count = star_count;
- }
-
- @SuppressWarnings("unused")
- public int getOpen_issues_count() {
- return open_issues_count;
- }
-
- public void setOpen_issues_count(int open_issues_count) {
- this.open_issues_count = open_issues_count;
- }
-
- @SuppressWarnings("unused")
- public int getForks_count() {
- return forks_count;
- }
-
- public void setForks_count(int forks_count) {
- this.forks_count = forks_count;
- }
-
- @SuppressWarnings("unused")
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- }
-
- @SuppressWarnings("unused")
- public boolean isArchived() {
- return archived;
- }
-
- public void setArchived(boolean archived) {
- this.archived = archived;
- }
- }
-
/**
* Represents a link to another Item (either relative to this path, to another
* project,
@@ -743,12 +649,12 @@ public class GitlabEventService {
indexMetadataNode(projectPermissions, metadataTree);
}
- protected ExtendedGitProject mapProject(Project p, ProjectPermissions perm) {
+ protected GitProjectDTO mapProject(Project p, ProjectPermissions perm) {
final String[] split = p.getPathWithNamespace().split("/");
String mainGroup = split[0];
String subGroup = split.length > 2 ? split[1] : null;
- ExtendedGitProject gp = new ExtendedGitProject();
+ GitProjectDTO gp = new GitProjectDTO();
gp.setLast_activity_at(p.getLastActivityAt().toInstant());
gp.setMain_group(mainGroup);
gp.setNamespace(p.getPathWithNamespace());
diff --git a/src/test/java/at/ac/uibk/gitsearch/repository/search/testESService/ElasticSearchTestConfiguration.java b/src/test/java/at/ac/uibk/gitsearch/repository/search/testESService/ElasticSearchTestConfiguration.java
index 6476b8479c352a8a8ce7177e8f7dd0b1e20c67fc..8504f9d982002cb77358d621e509522c67c56c91 100644
--- a/src/test/java/at/ac/uibk/gitsearch/repository/search/testESService/ElasticSearchTestConfiguration.java
+++ b/src/test/java/at/ac/uibk/gitsearch/repository/search/testESService/ElasticSearchTestConfiguration.java
@@ -41,8 +41,8 @@ import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHost;
+import org.codeability.sharing.plugins.api.search.GitProjectDTO;
import org.codeability.sharing.plugins.api.search.SearchResultDTO;
-import org.codeability.sharing.plugins.api.search.SearchResultDTO.GitProject;
import org.codeability.sharing.plugins.api.search.SearchResultDTO.MetadataFile;
import org.codeability.sharing.plugins.api.search.UserProvidedMetadataDTO;
import org.codeability.sharing.plugins.api.search.util.ExerciseId;
@@ -51,6 +51,7 @@ import org.elasticsearch.node.InternalSettingsPreparer;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
+import org.gitlab4j.api.models.Visibility;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@@ -202,130 +203,6 @@ public class ElasticSearchTestConfiguration {
return new HttpHost(ipAddress, esPort);
}
- /**
- * just a helper class to store also data that is not exported to the user
- *
- * @author Michael Breu
- *
- */
- public static class ExtendedGitProject extends GitProject {
-
- private String visibility;
- private boolean archived;
- private int star_count;
- private int open_issues_count;
- private int forks_count;
- private String description;
- private String[] groups;
- private String commit_id;
-
- /**
- * @return the visibility
- */
- public String getVisibility() {
- return visibility;
- }
-
- /**
- * @param visibility the visibility to set
- */
- public void setVisibility(String visibility) {
- this.visibility = visibility;
- }
-
- /**
- * @return the archived
- */
- public boolean isArchived() {
- return archived;
- }
-
- /**
- * @param archived the archived to set
- */
- public void setArchived(boolean archived) {
- this.archived = archived;
- }
-
- /**
- * @return the star_count
- */
- @SuppressWarnings("PMD")
- public int getStar_count() {
- return star_count;
- }
-
- /**
- * @param star_count the star_count to set
- */
- @SuppressWarnings("PMD")
- public void setStar_count(int star_count) {
- this.star_count = star_count;
- }
-
- /**
- * @return the open_issues_count
- */
- @SuppressWarnings("PMD")
- public int getOpen_issues_count() {
- return open_issues_count;
- }
-
- /**
- * @param open_issues_count the open_issues_count to set
- */
- @SuppressWarnings("PMD")
- public void setOpen_issues_count(int open_issues_count) {
- this.open_issues_count = open_issues_count;
- }
-
- /**
- * @return the forks_count
- */
- @SuppressWarnings("PMD")
- public int getForks_count() {
- return forks_count;
- }
-
- /**
- * @param forks_count the forks_count to set
- */
- @SuppressWarnings("PMD")
- public void setForks_count(int forks_count) {
- this.forks_count = forks_count;
- }
-
- /**
- * @return the description
- */
- public String getDescription() {
- return description;
- }
-
- /**
- * @param description the description to set
- */
- public void setDescription(String description) {
- this.description = description;
- }
-
- public String[] getGroups() {
- return groups;
- }
-
- public void setGroups(String[] groups) {
- this.groups = groups;
- }
-
- public String getCommit_id() {
- return commit_id;
- }
-
- public void setCommit_id(String commit_id) {
- this.commit_id = commit_id;
- }
- }
-
private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(ElasticSearchTestConfiguration.class);
private TestESNode testNode;
@@ -383,7 +260,7 @@ public class ElasticSearchTestConfiguration {
contentCount = 0;
visitSubdirectories(
metaDataTestDir,
- (File metaDataFile, List<TreeNode<SearchResultDTO>> ancestors, String relativePath, ExtendedGitProject gitProject) -> {
+ (File metaDataFile, List<TreeNode<SearchResultDTO>> ancestors, String relativePath, GitProjectDTO gitProject) -> {
try {
final UserProvidedMetadataDTO userMetaData = parseMetaDataFile(metaDataFile);
@@ -393,11 +270,11 @@ public class ElasticSearchTestConfiguration {
final ExerciseId exerciseId = calculateProjectId(relativePath);
if (gitProject == null) {
- gitProject = new ExtendedGitProject();
+ gitProject = new GitProjectDTO();
gitProject.setProject_id(Integer.parseInt(exerciseId.getProjectId()));
gitProject.setProject_name("SomeGitProject" + exerciseId.getProjectId());
- gitProject.setVisibility("public");
+ gitProject.setVisibility(Visibility.PUBLIC);
}
toIndex.setProject(gitProject);
@@ -413,10 +290,7 @@ public class ElasticSearchTestConfiguration {
: (relativePath.substring(slashPos + 1) + "/" + metaDataFile.getName());
file.setPath(projectRelativePath);
file.setIndexing_date(Instant.now());
- file.setCommit_id("Unused Commit Id");
- if (gitProject.getCommit_id() != null) {
- file.setCommit_id(gitProject.getCommit_id());
- }
+ file.setCommit_id(gitProject.getCommit_id());
file.setChildren(
ancestors.stream().map(node -> node.getData().getExerciseId()).collect(Collectors.toList()).toArray(new String[] {})
);
@@ -512,34 +386,10 @@ public class ElasticSearchTestConfiguration {
}
}
- private ExtendedGitProject parseProjectFile(File projectFile) throws JsonParseException, JsonMappingException, IOException {
+ private GitProjectDTO parseProjectFile(File projectFile) throws JsonParseException, JsonMappingException, IOException {
ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
mapper.findAndRegisterModules();
- ExtendedGitProject result = mapper.readValue(projectFile, ExtendedGitProject.class);
- return result;
- }
-
- /**
- * an interface that is invoked on each node of the tree with a metadata
- *
- * @author Michael Breu
- *
- * @param <T> the harvested data type
- */
- private interface Harvester<T> {
- /**
- *
- * @param f
- * the metaData File
- * @param subResults
- * the tree of results of the next node level
- * @param relativePath
- * the relative path from test root
- * @param gitProject
- * the information about the git project.
- * @return
- */
- T harvest(File f, List<TreeNode<T>> subResults, String relativePath, ExtendedGitProject gitProject);
+ return mapper.readValue(projectFile, GitProjectDTO.class);
}
/**
@@ -558,8 +408,8 @@ public class ElasticSearchTestConfiguration {
* the gitProject that may be collected from root.
* @return the tree of descendants
*/
- private <T> TreeNode<T> visitSubdirectories(File dir, Harvester<T> harvester, String relativePath, ExtendedGitProject gitProject) {
- ExtendedGitProject currentGitProject = gitProject;
+ private <T> TreeNode<T> visitSubdirectories(File dir, Harvester<T> harvester, String relativePath, GitProjectDTO gitProject) {
+ GitProjectDTO currentGitProject = gitProject;
Assert.state(dir.exists() && dir.isDirectory(), "must be a directory");
List<TreeNode<T>> children = new ArrayList<TreeNode<T>>();
for (File containedFile : dir.listFiles()) {
@@ -607,6 +457,23 @@ public class ElasticSearchTestConfiguration {
return root;
}
+ /**
+ * an interface that is invoked on each node of the tree with a metadata
+ *
+ * @param <T> the harvested data type
+ * @author Michael Breu
+ */
+ private interface Harvester<T> {
+ /**
+ * @param f the metaData File
+ * @param subResults the tree of results of the next node level
+ * @param relativePath the relative path from test root
+ * @param gitProject the information about the git project.
+ * @return
+ */
+ T harvest(File f, List<TreeNode<T>> subResults, String relativePath, GitProjectDTO gitProject);
+ }
+
/**
* replaces existing metadata by new content
*
diff --git a/src/test/java/at/ac/uibk/gitsearch/service/EditorialPagesServiceIT.java b/src/test/java/at/ac/uibk/gitsearch/service/EditorialPagesServiceIT.java
index 0c2b89734c5d0ddb3c0b61d7465db08b7a292b8b..309884cc149e8d3a986639f108e021dd0befa705 100644
--- a/src/test/java/at/ac/uibk/gitsearch/service/EditorialPagesServiceIT.java
+++ b/src/test/java/at/ac/uibk/gitsearch/service/EditorialPagesServiceIT.java
@@ -6,9 +6,11 @@ import static org.hamcrest.Matchers.not;
import at.ac.uibk.gitsearch.GitsearchApp;
import at.ac.uibk.gitsearch.service.dto.EditorialPageDTO;
+import java.util.concurrent.TimeUnit;
import org.gitlab4j.api.GitLabApiException;
import org.hamcrest.MatcherAssert;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.transaction.annotation.Transactional;
@@ -18,13 +20,14 @@ import org.springframework.transaction.annotation.Transactional;
*/
@SpringBootTest(classes = GitsearchApp.class)
@Transactional
-public class EditorialPagesServiceIT {
+class EditorialPagesServiceIT {
@Autowired
private EditorialPagesService editorialPagesService;
@Test
- public void testSimplePage() throws GitLabApiException {
+ @Timeout(value = 60, unit = TimeUnit.SECONDS) // gitlab may respond slowly
+ void testSimplePage() throws GitLabApiException {
EditorialPageDTO page = editorialPagesService.getContent("/de/start");
MatcherAssert.assertThat(page.getContent(), not(is(emptyOrNullString())));
}
diff --git a/src/test/java/at/ac/uibk/gitsearch/service/SearchServiceIT.java b/src/test/java/at/ac/uibk/gitsearch/service/SearchServiceIT.java
index 743279bd4f909aed5c1148773417011c0fd6a94e..eab4854181f4058a9e7185be627cb9af5beada7e 100644
--- a/src/test/java/at/ac/uibk/gitsearch/service/SearchServiceIT.java
+++ b/src/test/java/at/ac/uibk/gitsearch/service/SearchServiceIT.java
@@ -17,21 +17,24 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import at.ac.uibk.gitsearch.IntegrationTest;
-import at.ac.uibk.gitsearch.domain.Statistics;
import at.ac.uibk.gitsearch.repository.gitlab.GitLabRepository;
import at.ac.uibk.gitsearch.repository.jpa.StatisticsRepository;
+import at.ac.uibk.gitsearch.repository.search.ElasticSearchRepository;
import at.ac.uibk.gitsearch.repository.search.MetaDataRepository;
import at.ac.uibk.gitsearch.repository.search.testESService.ElasticSearchTestConfiguration;
import at.ac.uibk.gitsearch.security.jwt.TokenProvider;
import at.ac.uibk.gitsearch.service.PluginManagementService.ConnectorConfigWrapper;
import at.ac.uibk.gitsearch.service.dto.AutoCompleteEntry;
+import at.ac.uibk.gitsearch.service.dto.StatisticsDTO;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import javax.ws.rs.NotFoundException;
@@ -78,6 +81,12 @@ public class SearchServiceIT {
@Autowired
StatisticsRepository statisticsRepository;
+ @Autowired
+ StatisticsService statisticsService;
+
+ @Autowired
+ ElasticSearchRepository elasticSearchRepository;
+
@MockBean
RepositoryApi mockedRepositoryApi;
@@ -91,7 +100,8 @@ public class SearchServiceIT {
static ElasticSearchTestConfiguration staticElasticSearchTestConfiguration;
- @BeforeEach // must be started as BeforeEach, in order to autowire the ElasticSearchTestConfiguration
+ @BeforeEach // must be started as BeforeEach, in order to autowire the
+ // ElasticSearchTestConfiguration
@Timeout(value = 3, unit = TimeUnit.MINUTES)
public void setUpESServer() throws IOException, NodeValidationException {
if (staticElasticSearchTestConfiguration == null) {
@@ -152,41 +162,78 @@ public class SearchServiceIT {
}
@Test
- @Timeout(value = 1, unit = TimeUnit.MINUTES)
+ @Timeout(value = 10, unit = TimeUnit.MINUTES)
@Transactional
void testAllSearchWithStatistics() throws Exception {
// prepare
final SearchInputMetadataDTO searchMetadata = new SearchInputMetadataDTO(null, null, null, null, null, null);
SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null, null, null, 0);
+ searchQuery.setPageSize(500);
SearchResultsDTO searchResultPage = searchService.searchResultPage(searchQuery);
- AtomicInteger count = new AtomicInteger(0);
+ AtomicInteger downloadsCount = new AtomicInteger(0);
+ AtomicBoolean badgeRewarded = new AtomicBoolean(false);
+ AtomicInteger viewsCount = new AtomicInteger(0);
// setup statistics data
searchResultPage
.getSearchResult()
- .forEach(sr -> {
+ .stream()
+ .map(sr -> {
assertThat(sr.getSearchStatistics()).isNull(); // this may be critical, if some other test already
- // defined
- // statistics
- Statistics stat = new Statistics();
+ return sr;
+ })
+ .skip(1) // leave one stat empty
+ .forEach(sr -> {
+ // define statistics
+ StatisticsDTO stat = new StatisticsDTO();
stat.setExerciseID(sr.getExerciseId());
- stat.setDownloads(100 + count.get());
- stat.setBadgeRewarded(count.get() % 2 == 1 ? true : false);
- stat.setViews(1000 + (count.get()));
- count.addAndGet(1);
- statisticsRepository.save(stat);
+ stat.setDownloads(5 + downloadsCount.get());
+ stat.setBadgeRewarded(badgeRewarded.get());
+ stat.setViews(95 + (viewsCount.get()));
+ if (badgeRewarded.getAndSet(!badgeRewarded.get())) {
+ downloadsCount.getAndIncrement();
+ }
+ viewsCount.getAndIncrement();
+ statisticsService.save(stat);
});
+
statisticsRepository.flush();
+ elasticSearchRepository.waitForIndexingFinished();
SearchResultsDTO searchResultPage2 = searchService.searchResultPage(searchQuery);
-
- searchResultPage2
- .getSearchResult()
- .forEach(sr -> {
+ // it occurs that upon requesting elastic search right after updating the
+ // indices, the old order is retrieved
+
+ Comparator<SearchResultDTO> orderComparator = Comparator.comparing(sr -> {
+ if (sr.getSearchStatistics() == null) {
+ return 0D;
+ }
+ return (
+ (sr.getSearchStatistics().getBadgeRewarded() ? 1 : 0) *
+ searchQuery.getOrdering().getFactorBadge() +
+ sr.getSearchStatistics().getDownloads() *
+ searchQuery.getOrdering().getFactorDownloads() +
+ sr.getSearchStatistics().getViews() *
+ searchQuery.getOrdering().getFactorViews()
+ );
+ });
+
+ SearchResultDTO previousSearchResult = null;
+ var searchResultsIterator = searchResultPage2.getSearchResult().iterator();
+ while (searchResultsIterator.hasNext()) {
+ var sr = searchResultsIterator.next();
+ if (!searchResultsIterator.hasNext()) {
+ assertThat(sr.getSearchStatistics()).isNull();
+ } else {
assertThat(sr.getSearchStatistics()).isNotNull();
- assertThat(sr.getSearchStatistics().getDownloads()).isGreaterThanOrEqualTo(100);
- assertThat(sr.getSearchStatistics().getViews()).isGreaterThanOrEqualTo(1000);
- });
+ assertThat(sr.getSearchStatistics().getDownloads()).isGreaterThanOrEqualTo(5);
+ assertThat(sr.getSearchStatistics().getViews()).isGreaterThanOrEqualTo(95);
+ }
+ if (previousSearchResult != null) {
+ assertThat(orderComparator.compare(previousSearchResult, sr)).isPositive();
+ }
+ previousSearchResult = sr;
+ }
}
@Test
@@ -283,19 +330,24 @@ public class SearchServiceIT {
}
// does not work with metadata version 0.4
- // @Test
- // void testTypesSearch() throws IOException {
- // final SearchInputMetadataDTO searchMetadata = new SearchInputMetadataDTO(null, "latex", null, null, null,null);
- // searchMetadata.setTypes(Collections.singletonList(ExerciseType.COLLECTION));
- // SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null, null, null, 0);
- // SearchResultsDTO searchResultPage = searchService.searchResultPage(searchQuery);
+ // @Test
+ // void testTypesSearch() throws IOException {
+ // final SearchInputMetadataDTO searchMetadata = new
+ // SearchInputMetadataDTO(null, "latex", null, null, null,null);
+ // searchMetadata.setTypes(Collections.singletonList(ExerciseType.COLLECTION));
+ // SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null,
+ // null, null, 0);
+ // SearchResultsDTO searchResultPage =
+ // searchService.searchResultPage(searchQuery);
//
- // org.junit.Assert.assertNotNull(searchResultPage.getSearchResult());
- // org.junit.Assert.assertTrue("At least one test hit", searchResultPage.getHitCount() >= 1);
- // assertThat(searchResultPage.getSearchResult(),
- // everyItem(hasProperty("metadata", hasProperty("keyword", hasItemInArray(containsString("latex"))))));
+ // org.junit.Assert.assertNotNull(searchResultPage.getSearchResult());
+ // org.junit.Assert.assertTrue("At least one test hit",
+ // searchResultPage.getHitCount() >= 1);
+ // assertThat(searchResultPage.getSearchResult(),
+ // everyItem(hasProperty("metadata", hasProperty("keyword",
+ // hasItemInArray(containsString("latex"))))));
//
- // }
+ // }
@Test
@Timeout(value = 1, unit = TimeUnit.MINUTES)
@@ -344,20 +396,24 @@ public class SearchServiceIT {
}
// does not work with metadata version 0.4
- // @Test
- // void testTypesSearchNegative() throws IOException {
- // final SearchInputMetadataDTO searchMetadata = new SearchInputMetadataDTO(null, "latex", null, null, null,null);
- // searchMetadata.setTypes(Collections.singletonList(ExerciseType.OTHER));
- // SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null, null, null, 0);
- // SearchResultsDTO searchResultPage = searchService.searchResultPage(searchQuery);
+ // @Test
+ // void testTypesSearchNegative() throws IOException {
+ // final SearchInputMetadataDTO searchMetadata = new
+ // SearchInputMetadataDTO(null, "latex", null, null, null,null);
+ // searchMetadata.setTypes(Collections.singletonList(ExerciseType.OTHER));
+ // SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null,
+ // null, null, 0);
+ // SearchResultsDTO searchResultPage =
+ // searchService.searchResultPage(searchQuery);
//
- // org.junit.Assert.assertNotNull(searchResultPage.getSearchResult());
- // org.junit.Assert.assertTrue("We expect no hit for \"latex\" and ExerciseType.OTHER", searchResultPage.getHitCount() == 0);
+ // org.junit.Assert.assertNotNull(searchResultPage.getSearchResult());
+ // org.junit.Assert.assertTrue("We expect no hit for \"latex\" and
+ // ExerciseType.OTHER", searchResultPage.getHitCount() == 0);
//
- // }
+ // }
- // @Test()
- // @Ignore() // Test funktioniert momentan nicht?
+ // @Test()
+ // @Ignore() // Test funktioniert momentan nicht?
void testLicenseSearch() throws IOException {
final SearchInputMetadataDTO searchMetadata = new SearchInputMetadataDTO(null, null, null, "MIT", null, null);
SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null, null, null, 0);
diff --git a/src/test/java/at/ac/uibk/gitsearch/service/dto/VariousDTOTest.java b/src/test/java/at/ac/uibk/gitsearch/service/dto/VariousDTOTest.java
index e268ae03fe70805b742014e99b6b31bb76c685c0..ba2a28a0dad1c3ffd11aa825322dd9a3d0ca5f26 100644
--- a/src/test/java/at/ac/uibk/gitsearch/service/dto/VariousDTOTest.java
+++ b/src/test/java/at/ac/uibk/gitsearch/service/dto/VariousDTOTest.java
@@ -8,8 +8,8 @@ import at.ac.uibk.gitsearch.testingUtilities.PropertiesTester;
import java.lang.reflect.InvocationTargetException;
import nl.jqno.equalsverifier.EqualsVerifier;
import nl.jqno.equalsverifier.Warning;
+import org.codeability.sharing.plugins.api.search.GitProjectDTO;
import org.codeability.sharing.plugins.api.search.SearchResultDTO;
-import org.codeability.sharing.plugins.api.search.SearchResultDTO.GitProject;
import org.codeability.sharing.plugins.api.search.SearchResultsDTO;
import org.codeability.sharing.plugins.api.search.UserProvidedMetadataDTO;
import org.codeability.sharing.plugins.api.search.UserProvidedMetadataDTO.Person;
@@ -53,7 +53,7 @@ class VariousDTOTest {
propertiesTester.testProperties(SearchResultsDTO.class);
propertiesTester.testProperties(SearchResultDTO.class);
propertiesTester.testProperties(Person.class);
- propertiesTester.testProperties(GitProject.class);
+ propertiesTester.testProperties(GitProjectDTO.class);
// just for test coverage
@SuppressWarnings("unused")
@@ -85,19 +85,36 @@ class VariousDTOTest {
@SuppressWarnings("deprecation")
@org.junit.jupiter.api.Test
void testVariousEquals() throws IllegalAccessException, InvocationTargetException {
- GitProject p1 = new GitProject();
+ GitProjectDTO p1 = new GitProjectDTO();
p1.setProjectId(10);
- GitProject p2 = new GitProject();
+ GitProjectDTO p2 = new GitProjectDTO();
p2.setProjectId(10);
- GitProject p3 = new GitProject();
+ GitProjectDTO p3 = new GitProjectDTO();
p3.setProjectId(20);
Assert.assertEquals(p1, p2);
Assert.assertNotEquals(p1, p3);
EqualsVerifier
- .forClass(GitProject.class)
+ .forClass(GitProjectDTO.class)
.suppress(Warning.NONFINAL_FIELDS)
- .withIgnoredFields("project_name", "namespace", "main_group", "last_activity_at", "url", "sub_group")
+ .withIgnoredFields(
+ "project_name",
+ "namespace",
+ "main_group",
+ "sub_group",
+ "url",
+ "notPresent",
+ "last_activity_at",
+ "visibility",
+ "users",
+ "groups",
+ "star_count",
+ "open_issues_count",
+ "forks_count",
+ "description",
+ "archived",
+ "commit_id"
+ )
.verify();
EqualsVerifier.forClass(Person.class).suppress(Warning.NONFINAL_FIELDS).verify();
}
diff --git a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/1/project.yaml b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/1/project.yaml
index ea1d730c89e72766ee42cba2b1de3d619c211320..cbba1840963ce52c7f1f61b35ec026369498aac6 100644
--- a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/1/project.yaml
+++ b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/1/project.yaml
@@ -13,3 +13,5 @@ open_issues_count: 0
forks_count: 1
last_activity_at: 2021-02-17T15:13:27.123Z
description:
+commit_id: "01e2fabbd79b913d7180453ff3e32c68cb57d660"
+
diff --git a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/143/project.yaml b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/143/project.yaml
index 8e6d3e1fe58c8b0385929eacb4f612a05f957cbf..f3e36797775ac235f5d6bb081aa0241d4ef384bc 100644
--- a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/143/project.yaml
+++ b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/143/project.yaml
@@ -13,3 +13,4 @@ open_issues_count: 0
forks_count: 1
last_activity_at: 2021-04-08T15:13:27.123Z
description: Just a test project
+commit_id: "01e2fabbd79b913d7180453ff3e32c68cb57d660"
diff --git a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/2/project.yaml b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/2/project.yaml
index c4c63330fb72acc19b936f36106653eef87909bd..0452323f1d2403f49582cfdf2c280bd079db4f31 100644
--- a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/2/project.yaml
+++ b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/2/project.yaml
@@ -1,4 +1,4 @@
-project_id: 1
+project_id: 2
project_name: latex
namespace: sharing/vienna-universityof-technology/latex
main_group: sharing
@@ -13,3 +13,5 @@ open_issues_count: 0
forks_count: 1
last_activity_at: 2021-02-17T15:13:27.123Z
description:
+commit_id: "01e2fabbd79b913d7180453ff3e32c68cb57d660"
+
diff --git a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/272/project.yaml b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/272/project.yaml
index e289b4c262a2cc06a1dd1b185ba3a816b6b52998..3ee38c7459d5ddb095012510a07cb36a518a0e5a 100644
--- a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/272/project.yaml
+++ b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/272/project.yaml
@@ -13,3 +13,4 @@ open_issues_count: 0
forks_count: 1
last_activity_at: 2021-04-08T15:13:27.123Z
description: Just a test project
+commit_id: "01e2fabbd79b913d7180453ff3e32c68cb57d660"
diff --git a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/274/project.yaml b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/274/project.yaml
index 5931ccfc69e318919d0ec3fcf29e82173302d676..0f507fe85e26b2f642bf89cff71cca53ded4054e 100644
--- a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/274/project.yaml
+++ b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/274/project.yaml
@@ -13,3 +13,5 @@ open_issues_count: 0
forks_count: 1
last_activity_at: 2021-04-08T15:13:27.123Z
description: Just a test project
+commit_id: "35573081a2232dd8e71ab6375f1adb69e65486c8"
+
diff --git a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/3/project.yaml b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/3/project.yaml
index ab56c4d27021850d03126658351afef0c38b465c..ea2ad2b482698ff1910e89092ef793c973c8085d 100644
--- a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/3/project.yaml
+++ b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/3/project.yaml
@@ -1,4 +1,4 @@
-project_id: 272
+project_id: 3
project_name: latex
namespace: sharing/vienna-universityof-technology/latex
main_group: sharing
@@ -13,3 +13,4 @@ open_issues_count: 0
forks_count: 1
last_activity_at: 2021-02-17T15:13:27.123Z
description:
+commit_id: "35573081a2232dd8e71ab6375f1adb69e65486c8"
diff --git a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/333/project.yaml b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/333/project.yaml
index 2e1508644f2f849c4a7ee4ef0d01f95f43d8d9f0..8fb1f4e2478c133f6ae7ff3fe36ef7e7c4ac7ed5 100644
--- a/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/333/project.yaml
+++ b/src/test/resources/at/ac/uibk/gitsearch/service/testData/metaData/333/project.yaml
@@ -13,3 +13,4 @@ open_issues_count: 0
forks_count: 1
last_activity_at: 2021-04-08T15:13:27.123Z
description: Just a test project
+commit_id: "35573081a2232dd8e71ab6375f1adb69e65486c8"