diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 194b4911cb95ebbac632c98c6bbadcd6bd8eba7f..31a4771ea0d84a6308648de375c9f26a71532e09 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -90,7 +90,7 @@ maven-test:
# override the policy
policy: pull
script:
- - ./mvnw -ntp verify -P-webapp -Dmaven.repo.local=$MAVEN_USER_HOME -Dapplication.gitlab.guestAccessToken=$APPLICATION_GITLAB_GENERALACCESSTOKEN -Dapplication.gitlab.adminAccessToken=$APPLICATION_GITLAB_ADMINACCESSTOKEN -Dspring.profiles.active=testcontainers -Dlogging.level.ROOT=ERROR -Dlogging.level.org.zalando=OFF -Dlogging.level.tech.jhipster=OFF -Dlogging.level.at.ac.uibk.gitsearch=WARN -Dlogging.level.org.springframework=WARN -Dlogging.level.org.springframework.web=WARN -Dlogging.level.org.springframework.security=OFF -Dlogging.level.org.hibernate.SQL=WARN
+ - ./mvnw -ntp verify -P-webapp -Dmaven.repo.local=$MAVEN_USER_HOME -Dapplication.gitlab.guestAccessToken=$APPLICATION_GITLAB_GENERALACCESSTOKEN -Dapplication.gitlab.adminAccessToken=$APPLICATION_GITLAB_ADMINACCESSTOKEN -Dspring.profiles.active=testcontainers
allow_failure: true
artifacts:
reports:
diff --git a/package-lock.json b/package-lock.json
index 81c14953190df06ffc54d364099418c10f801ba2..502a65416f6f32251843c6a1e2351d4cce746078 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -480,12 +480,6 @@
"node": ">=6.9.0"
}
},
- "node_modules/@angular-devkit/build-angular/node_modules/@types/estree": {
- "version": "0.0.50",
- "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.50.tgz",
- "integrity": "sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw==",
- "dev": true
- },
"node_modules/@angular-devkit/build-angular/node_modules/ajv": {
"version": "8.8.2",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.8.2.tgz",
@@ -789,15 +783,6 @@
"win32"
]
},
- "node_modules/@angular-devkit/build-angular/node_modules/events": {
- "version": "3.3.0",
- "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
- "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
- "dev": true,
- "engines": {
- "node": ">=0.8.x"
- }
- },
"node_modules/@angular-devkit/build-angular/node_modules/glob-parent": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
@@ -983,102 +968,6 @@
"semver": "bin/semver.js"
}
},
- "node_modules/@angular-devkit/build-angular/node_modules/webpack": {
- "version": "5.65.0",
- "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.65.0.tgz",
- "integrity": "sha512-Q5or2o6EKs7+oKmJo7LaqZaMOlDWQse9Tm5l1WAfU/ujLGN5Pb0SqGeVkN/4bpPmEqEP5RnVhiqsOtWtUVwGRw==",
- "dev": true,
- "dependencies": {
- "@types/eslint-scope": "^3.7.0",
- "@types/estree": "^0.0.50",
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/wasm-edit": "1.11.1",
- "@webassemblyjs/wasm-parser": "1.11.1",
- "acorn": "^8.4.1",
- "acorn-import-assertions": "^1.7.6",
- "browserslist": "^4.14.5",
- "chrome-trace-event": "^1.0.2",
- "enhanced-resolve": "^5.8.3",
- "es-module-lexer": "^0.9.0",
- "eslint-scope": "5.1.1",
- "events": "^3.2.0",
- "glob-to-regexp": "^0.4.1",
- "graceful-fs": "^4.2.4",
- "json-parse-better-errors": "^1.0.2",
- "loader-runner": "^4.2.0",
- "mime-types": "^2.1.27",
- "neo-async": "^2.6.2",
- "schema-utils": "^3.1.0",
- "tapable": "^2.1.1",
- "terser-webpack-plugin": "^5.1.3",
- "watchpack": "^2.3.1",
- "webpack-sources": "^3.2.2"
- },
- "bin": {
- "webpack": "bin/webpack.js"
- },
- "engines": {
- "node": ">=10.13.0"
- },
- "funding": {
- "type": "opencollective",
- "url": "https://opencollective.com/webpack"
- },
- "peerDependenciesMeta": {
- "webpack-cli": {
- "optional": true
- }
- }
- },
- "node_modules/@angular-devkit/build-angular/node_modules/webpack/node_modules/ajv": {
- "version": "6.12.6",
- "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
- "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
- "dev": true,
- "dependencies": {
- "fast-deep-equal": "^3.1.1",
- "fast-json-stable-stringify": "^2.0.0",
- "json-schema-traverse": "^0.4.1",
- "uri-js": "^4.2.2"
- },
- "funding": {
- "type": "github",
- "url": "https://github.com/sponsors/epoberezkin"
- }
- },
- "node_modules/@angular-devkit/build-angular/node_modules/webpack/node_modules/ajv-keywords": {
- "version": "3.5.2",
- "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz",
- "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==",
- "dev": true,
- "peerDependencies": {
- "ajv": "^6.9.1"
- }
- },
- "node_modules/@angular-devkit/build-angular/node_modules/webpack/node_modules/json-schema-traverse": {
- "version": "0.4.1",
- "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
- "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
- "dev": true
- },
- "node_modules/@angular-devkit/build-angular/node_modules/webpack/node_modules/schema-utils": {
- "version": "3.1.1",
- "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz",
- "integrity": "sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw==",
- "dev": true,
- "dependencies": {
- "@types/json-schema": "^7.0.8",
- "ajv": "^6.12.5",
- "ajv-keywords": "^3.5.2"
- },
- "engines": {
- "node": ">= 10.13.0"
- },
- "funding": {
- "type": "opencollective",
- "url": "https://opencollective.com/webpack"
- }
- },
"node_modules/@angular-devkit/build-webpack": {
"version": "0.1301.2",
"resolved": "https://registry.npmjs.org/@angular-devkit/build-webpack/-/build-webpack-0.1301.2.tgz",
@@ -25100,7 +24989,6 @@
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.66.0.tgz",
"integrity": "sha512-NJNtGT7IKpGzdW7Iwpn/09OXz9inIkeIQ/ibY6B+MdV1x6+uReqz/5z1L89ezWnpPDWpXF0TY5PCYKQdWVn8Vg==",
"dev": true,
- "peer": true,
"dependencies": {
"@types/eslint-scope": "^3.7.0",
"@types/estree": "^0.0.50",
@@ -25531,15 +25419,13 @@
"version": "0.0.50",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.50.tgz",
"integrity": "sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw==",
- "dev": true,
- "peer": true
+ "dev": true
},
"node_modules/webpack/node_modules/ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"dev": true,
- "peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
@@ -25556,7 +25442,6 @@
"resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz",
"integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==",
"dev": true,
- "peer": true,
"peerDependencies": {
"ajv": "^6.9.1"
}
@@ -25566,7 +25451,6 @@
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
"integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
"dev": true,
- "peer": true,
"engines": {
"node": ">=0.8.x"
}
@@ -25575,15 +25459,13 @@
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
- "dev": true,
- "peer": true
+ "dev": true
},
"node_modules/webpack/node_modules/schema-utils": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz",
"integrity": "sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw==",
"dev": true,
- "peer": true,
"dependencies": {
"@types/json-schema": "^7.0.8",
"ajv": "^6.12.5",
@@ -26712,12 +26594,6 @@
"@babel/types": "^7.16.0"
}
},
- "@types/estree": {
- "version": "0.0.50",
- "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.50.tgz",
- "integrity": "sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw==",
- "dev": true
- },
"ajv": {
"version": "8.8.2",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.8.2.tgz",
@@ -26895,12 +26771,6 @@
"dev": true,
"optional": true
},
- "events": {
- "version": "3.3.0",
- "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
- "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
- "dev": true
- },
"glob-parent": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
@@ -27039,76 +26909,6 @@
"dev": true
}
}
- },
- "webpack": {
- "version": "5.65.0",
- "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.65.0.tgz",
- "integrity": "sha512-Q5or2o6EKs7+oKmJo7LaqZaMOlDWQse9Tm5l1WAfU/ujLGN5Pb0SqGeVkN/4bpPmEqEP5RnVhiqsOtWtUVwGRw==",
- "dev": true,
- "requires": {
- "@types/eslint-scope": "^3.7.0",
- "@types/estree": "^0.0.50",
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/wasm-edit": "1.11.1",
- "@webassemblyjs/wasm-parser": "1.11.1",
- "acorn": "^8.4.1",
- "acorn-import-assertions": "^1.7.6",
- "browserslist": "^4.14.5",
- "chrome-trace-event": "^1.0.2",
- "enhanced-resolve": "^5.8.3",
- "es-module-lexer": "^0.9.0",
- "eslint-scope": "5.1.1",
- "events": "^3.2.0",
- "glob-to-regexp": "^0.4.1",
- "graceful-fs": "^4.2.4",
- "json-parse-better-errors": "^1.0.2",
- "loader-runner": "^4.2.0",
- "mime-types": "^2.1.27",
- "neo-async": "^2.6.2",
- "schema-utils": "^3.1.0",
- "tapable": "^2.1.1",
- "terser-webpack-plugin": "^5.1.3",
- "watchpack": "^2.3.1",
- "webpack-sources": "^3.2.2"
- },
- "dependencies": {
- "ajv": {
- "version": "6.12.6",
- "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
- "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
- "dev": true,
- "requires": {
- "fast-deep-equal": "^3.1.1",
- "fast-json-stable-stringify": "^2.0.0",
- "json-schema-traverse": "^0.4.1",
- "uri-js": "^4.2.2"
- }
- },
- "ajv-keywords": {
- "version": "3.5.2",
- "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz",
- "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==",
- "dev": true,
- "requires": {}
- },
- "json-schema-traverse": {
- "version": "0.4.1",
- "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
- "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
- "dev": true
- },
- "schema-utils": {
- "version": "3.1.1",
- "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz",
- "integrity": "sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw==",
- "dev": true,
- "requires": {
- "@types/json-schema": "^7.0.8",
- "ajv": "^6.12.5",
- "ajv-keywords": "^3.5.2"
- }
- }
- }
}
}
},
@@ -45238,7 +45038,6 @@
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.66.0.tgz",
"integrity": "sha512-NJNtGT7IKpGzdW7Iwpn/09OXz9inIkeIQ/ibY6B+MdV1x6+uReqz/5z1L89ezWnpPDWpXF0TY5PCYKQdWVn8Vg==",
"dev": true,
- "peer": true,
"requires": {
"@types/eslint-scope": "^3.7.0",
"@types/estree": "^0.0.50",
@@ -45270,15 +45069,13 @@
"version": "0.0.50",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.50.tgz",
"integrity": "sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw==",
- "dev": true,
- "peer": true
+ "dev": true
},
"ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"dev": true,
- "peer": true,
"requires": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
@@ -45291,29 +45088,25 @@
"resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz",
"integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==",
"dev": true,
- "peer": true,
"requires": {}
},
"events": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
"integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
- "dev": true,
- "peer": true
+ "dev": true
},
"json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
- "dev": true,
- "peer": true
+ "dev": true
},
"schema-utils": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz",
"integrity": "sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw==",
"dev": true,
- "peer": true,
"requires": {
"@types/json-schema": "^7.0.8",
"ajv": "^6.12.5",
diff --git a/pom.xml b/pom.xml
index c973632f2099aece9f84b9f37454509856403abf..a966234dd493c39776475889e37beb9cec3af22a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -56,7 +56,7 @@
<jhipster-dependencies.version>7.6.0</jhipster-dependencies.version>
<!-- The spring-boot version should match the one managed by
https://mvnrepository.com/artifact/tech.jhipster/jhipster-dependencies/${jhipster-dependencies.version} -->
- <spring-boot.version>2.6.14</spring-boot.version>
+ <spring-boot.version>2.7.9</spring-boot.version>
<!-- The hibernate version should match the one managed by
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-dependencies/${spring-boot.version} -->
<hibernate.version>5.6.4.Final</hibernate.version>
@@ -277,7 +277,6 @@
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
- <version>2.7.4</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
@@ -396,7 +395,22 @@
<!-- <scope>test</scope> -->
</dependency>
<!-- jhipster-needle-maven-add-dependency -->
- </dependencies>
+ <dependency>
+ <groupId>co.elastic.clients</groupId>
+ <artifactId>elasticsearch-java</artifactId>
+ <version>7.17.9</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>7.17.9</version>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.json</groupId>
+ <artifactId>jakarta.json-api</artifactId>
+ <version>2.0.1</version>
+ </dependency>
+ </dependencies>
<build>
<defaultGoal>spring-boot:run</defaultGoal>
diff --git a/src/main/java/at/ac/uibk/gitsearch/config/ElasticsearchConfiguration.java b/src/main/java/at/ac/uibk/gitsearch/config/ElasticsearchConfiguration.java
index 633af41692291280117b8e4fe49e9dbd06c745ea..0f92221333220045645cdf2691933603eb38359e 100644
--- a/src/main/java/at/ac/uibk/gitsearch/config/ElasticsearchConfiguration.java
+++ b/src/main/java/at/ac/uibk/gitsearch/config/ElasticsearchConfiguration.java
@@ -1,10 +1,29 @@
package at.ac.uibk.gitsearch.config;
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.json.jackson.JacksonJsonpMapper;
+import co.elastic.clients.transport.ElasticsearchTransport;
+import co.elastic.clients.transport.rest_client.RestClientTransport;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
import java.time.Instant;
import java.time.LocalDate;
+import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.util.ArrayList;
import java.util.Arrays;
+import org.apache.http.HttpHost;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.RestClient;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.convert.converter.Converter;
@@ -12,10 +31,14 @@ import org.springframework.data.convert.ReadingConverter;
import org.springframework.data.convert.WritingConverter;
import org.springframework.data.elasticsearch.config.ElasticsearchConfigurationSupport;
import org.springframework.data.elasticsearch.core.convert.ElasticsearchCustomConversions;
+import org.springframework.util.Assert;
@Configuration
public class ElasticsearchConfiguration extends ElasticsearchConfigurationSupport {
+ @Value("${spring.elasticsearch.uris}")
+ private String elasticSearchUrls;
+
@Bean
@Override
public ElasticsearchCustomConversions elasticsearchCustomConversions() {
@@ -102,4 +125,53 @@ public class ElasticsearchConfiguration extends ElasticsearchConfigurationSuppor
return LocalDate.parse(source);
}
}
+
+ public static class MoreRobustLocalDateTimeConverter extends com.fasterxml.jackson.databind.JsonDeserializer<LocalDateTime> {
+
+ @SuppressWarnings("unused")
+ private final Logger log = LogManager.getLogger(MoreRobustLocalDateTimeConverter.class);
+
+ @Override
+ @SuppressWarnings("PMD.PreserveStackTrace")
+ public LocalDateTime deserialize(com.fasterxml.jackson.core.JsonParser p, DeserializationContext ctxt) throws IOException {
+ try {
+ final String text = p.getText();
+ return LocalDateTime.parse(text, DateTimeFormatter.ISO_ZONED_DATE_TIME);
+ } catch (DateTimeParseException e) {
+ try {
+ final String text = p.getText();
+ return LocalDateTime.parse(text);
+ } catch (DateTimeParseException e2) {
+ throw new IOException(e2);
+ }
+ }
+ }
+ }
+
+ @Bean
+ public ElasticsearchClient getElasticsearchAPIClient() throws MalformedURLException {
+ // Create the transport with a Jackson mapper
+ final JacksonJsonpMapper mapper = new JacksonJsonpMapper();
+ final JavaTimeModule timeModule = new JavaTimeModule();
+ timeModule.addDeserializer(LocalDateTime.class, new ElasticsearchConfiguration.MoreRobustLocalDateTimeConverter());
+
+ mapper.objectMapper().registerModule(timeModule);
+ mapper.objectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ ElasticsearchTransport transport = new RestClientTransport(buildClient(), mapper);
+
+ // And create the API client
+ return new ElasticsearchClient(transport);
+ }
+
+ private RestClient buildClient() throws MalformedURLException {
+ Assert.hasText(elasticSearchUrls, "[Assertion Failed] At least one host must be set.");
+
+ ArrayList<HttpHost> httpHosts = new ArrayList<>();
+ for (String host : elasticSearchUrls.split(",")) {
+ URL hostUrl = new URL(host);
+ httpHosts.add(new HttpHost(hostUrl.getHost(), hostUrl.getPort(), hostUrl.getProtocol()));
+ }
+ return RestClient.builder(httpHosts.toArray(new HttpHost[0])).build();
+ }
}
diff --git a/src/main/java/at/ac/uibk/gitsearch/config/SecurityConfiguration.java b/src/main/java/at/ac/uibk/gitsearch/config/SecurityConfiguration.java
index 40fb99b094ca44bddacc0d2ae531ddc19ddd4656..02afd72eb25bb4771de3373e50792f9a6fe7bc0c 100644
--- a/src/main/java/at/ac/uibk/gitsearch/config/SecurityConfiguration.java
+++ b/src/main/java/at/ac/uibk/gitsearch/config/SecurityConfiguration.java
@@ -461,11 +461,7 @@ public class SecurityConfiguration extends WebSecurityConfigurerAdapter {
}
Map<String, Object> claims = new ConcurrentHashMap<>(oidcUser.getUserInfo().getClaims());
- userDetailsInfo
- .getUserToken()
- .ifPresent(token -> {
- claims.put(TokenProvider.GITLAB_ACCESS_TOKEN, token.getToken());
- });
+ userDetailsInfo.getUserToken().ifPresent(token -> claims.put(TokenProvider.GITLAB_ACCESS_TOKEN, token.getToken()));
OidcUserInfo userInfo = new OidcUserInfo(claims);
oidcUser = new DefaultOidcUser(mappedAuthorities, oidcUser.getIdToken(), userInfo);
diff --git a/src/main/java/at/ac/uibk/gitsearch/repository/jpa/StatisticsRepository.java b/src/main/java/at/ac/uibk/gitsearch/repository/jpa/StatisticsRepository.java
index 1a501cc89a719697140f2417c4a88acde88b927b..324124c5c2c26c31ab84242afdedef385457d6ac 100644
--- a/src/main/java/at/ac/uibk/gitsearch/repository/jpa/StatisticsRepository.java
+++ b/src/main/java/at/ac/uibk/gitsearch/repository/jpa/StatisticsRepository.java
@@ -5,6 +5,7 @@ import java.util.List;
import java.util.Optional;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
/**
@@ -16,15 +17,18 @@ public interface StatisticsRepository extends JpaRepository<Statistics, Long> {
Optional<Statistics> findById(Long id);
- @Query(value = "SELECT SUM(u.views) FROM statistics u WHERE u.exercise_id IN (?1)", nativeQuery = true)
- Optional<Integer> findNumberOfViewsByExerciseIDs(List<String> list);
+ @Query("SELECT SUM(u.views) FROM Statistics u WHERE u.exerciseID IN (:ids)")
+ Optional<Integer> findNumberOfViewsByExerciseIDs(@Param("ids") List<String> exerciseIds);
- @Query(value = "SELECT SUM(u.downloads) FROM statistics u WHERE u.exercise_id IN (?1)", nativeQuery = true)
- Optional<Integer> findNumberOfDownloadsByExerciseIDs(List<String> list);
+ @Query("SELECT SUM(u.downloads) FROM Statistics u WHERE u.exerciseID IN (:ids)")
+ Optional<Integer> findNumberOfDownloadsByExerciseIDs(@Param("ids") List<String> exerciseIds);
- @Query(value = "SELECT SUM(u.downloads), SUM(u.views) FROM statistics u WHERE u.exercise_id IN (?1)", nativeQuery = true)
- List<Integer[]> findNumberOfDownloadsAndViewsByExerciseIDs(List<String> list);
+ @Query("SELECT SUM(u.downloads), SUM(u.views) FROM Statistics u WHERE u.exerciseID IN (:ids)")
+ List<Integer[]> findNumberOfDownloadsAndViewsByExerciseIDs(@Param("ids") List<String> exerciseIds);
- @Query(value = "SELECT COUNT(u.exercise_id) FROM watch_list_entry u where exercise_id = ?1", nativeQuery = true)
- Integer findNumberOfWatchListEntriesByExerciseID(String exerciseID);
+ @Query("SELECT COUNT(u.exerciseId) FROM WatchListEntry u where exerciseId = :id")
+ Integer findNumberOfWatchListEntriesByExerciseID(@Param("id") String exerciseID);
+
+ @Query("SELECT SUM(u.downloads), SUM(u.views) FROM Statistics u")
+ List<Integer[]> findNumberOfDownloadsAndViews();
}
diff --git a/src/main/java/at/ac/uibk/gitsearch/repository/search/GitFilesRepositoryImpl.java b/src/main/java/at/ac/uibk/gitsearch/repository/search/GitFilesRepositoryImpl.java
index c0ae745382e745f9d74c921f8d66dc9bbbf76687..3a8a21ea2663fc835ff084c4e6e2b2c0059a3069 100644
--- a/src/main/java/at/ac/uibk/gitsearch/repository/search/GitFilesRepositoryImpl.java
+++ b/src/main/java/at/ac/uibk/gitsearch/repository/search/GitFilesRepositoryImpl.java
@@ -83,7 +83,6 @@ public class GitFilesRepositoryImpl implements GitFilesRepository {
.highlighter(highlightBuilder)
.size(searchInputDTO.getPageSize())
.from((searchInputDTO.getPage() - 1) * searchInputDTO.getPageSize());
- //.fetchSource(includes, excludes);
searchRequest.source(sourceBuilder);
@@ -98,7 +97,7 @@ public class GitFilesRepositoryImpl implements GitFilesRepository {
DocumentInfo documentInfo = objectMapper.readValue(searchHit.getSourceAsString(), DocumentInfo.class);
gitFilesDTOS.add(new GitFilesDTO(documentInfo, searchHit, preTags, postTags));
}
- long hitCount = searchResponse.getHits().getTotalHits().value;
+ long hitCount = searchResponse.getHits().getTotalHits() == null ? 0 : searchResponse.getHits().getTotalHits().value;
return new GitFilesPageDetailsDTO(gitFilesDTOS, hitCount);
}
diff --git a/src/main/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepository.java b/src/main/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepository.java
index 5c8403a1507bd5c72153e9e9afebef2b87862def..c8863935de48a24f6e7ee2697a0f4b8f853e5655 100644
--- a/src/main/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepository.java
+++ b/src/main/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepository.java
@@ -3,75 +3,91 @@ package at.ac.uibk.gitsearch.repository.search;
import at.ac.uibk.gitsearch.domain.Authority;
import at.ac.uibk.gitsearch.domain.util.TreeNode;
import at.ac.uibk.gitsearch.service.dto.AutoCompleteEntry;
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.ExistsQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.MatchPhrasePrefixQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.MatchQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.MultiMatchQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.PrefixQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.RangeQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.TermQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.TextQueryType;
+import co.elastic.clients.elasticsearch.core.search.Hit;
+import co.elastic.clients.json.JsonData;
+import co.elastic.clients.transport.rest_client.RestClientTransport;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.io.IOException;
+import java.net.ConnectException;
import java.text.ParseException;
import java.time.Instant;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.ConcurrentModificationException;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.NoSuchElementException;
import java.util.Optional;
+import java.util.Spliterator;
+import java.util.Spliterators;
import java.util.StringTokenizer;
-import java.util.Timer;
-import java.util.TimerTask;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
import javax.annotation.PostConstruct;
import javax.ws.rs.NotFoundException;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import org.apache.lucene.search.TotalHits;
+import org.codeability.sharing.plugins.api.search.SearchInputDTO;
import org.codeability.sharing.plugins.api.search.SearchResultDTO;
import org.codeability.sharing.plugins.api.search.SearchResultsDTO;
import org.codeability.sharing.plugins.api.search.UserProvidedMetadataDTO.Person;
import org.codeability.sharing.plugins.api.search.util.ExerciseId;
import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RequestOptions.Builder;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.ExistsQueryBuilder;
-import org.elasticsearch.index.query.MultiMatchQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.annotation.Cacheable;
-import org.springframework.context.event.ContextStoppedEvent;
-import org.springframework.context.event.EventListener;
+import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.userdetails.User;
import org.springframework.stereotype.Repository;
@Repository
-@SuppressWarnings({ "PMD.GodClass", "PMD.CyclomaticComplexity", "PMD.TooManyMethods" })
+@SuppressWarnings(
+ {
+ "PMD.GodClass",
+ "PMD.CyclomaticComplexity",
+ "PMD.TooManyMethods",
+ "PMD.CognitiveComplexity",
+ "PMD.NPathComplexity",
+ "PMD.ExcessiveClassLength",
+ "PMD.AvoidDuplicateLiterals",
+ }
+)
public class MetaDataRepository {
- private final RestHighLevelClient elasticsearchClient;
+ private final ElasticsearchClient elasticsearchAPIClient;
private static final Logger LOGGER = LogManager.getLogger(MetaDataRepository.class);
- private static final long AUTOCOMPLETION_RECHECK_TIMEOUT = 5 * 60 * 1000L; // every 5 Minutes
- private final Timer autocompletionReloadTimer = new Timer("AutocompletionReloadTimer");
+ private static final int SCROLL_SIZE = 10;
public static final String EXERCISE_BY_ID_CACHE = "at.ac.uibk.gitsearch.repository.search.MetaDataRepository.ExerciseByIdCache";
@@ -90,10 +106,17 @@ public class MetaDataRepository {
* with hit counts E.g. "metadata.creator" -> Kerstin -> {"Kerstin Limbeck" ->
* 3, "Kerstin Mair" -> 2)}
*/
- private Map<String, Map<String, Map<String, Integer>>> cachedCompletions = null;
+ private Map<String, Map<String, Map<String, Integer>>> cachedCompletions;
+
+ public MetaDataRepository(ElasticsearchClient elasticsearchAPIClient) {
+ this.elasticsearchAPIClient = elasticsearchAPIClient;
- public MetaDataRepository(RestHighLevelClient elasticsearchClient) {
- this.elasticsearchClient = elasticsearchClient;
+ cachedCompletions = new ConcurrentHashMap<>();
+ cachedCompletions.put(SearchRepositoryConstants.METADATA_KEYWORDS, new TreeMap<>());
+ cachedCompletions.put(SearchRepositoryConstants.METADATA_CREATOR, new TreeMap<>());
+ cachedCompletions.put(SearchRepositoryConstants.METADATA_CONTRIBUTOR, new TreeMap<>());
+ cachedCompletions.put(SearchRepositoryConstants.METADATA_PROGRAMMING_LANGUAGES, new TreeMap<>());
+ cachedCompletions.put(SearchRepositoryConstants.METADATA_FORMATS, new TreeMap<>());
}
/**
@@ -103,7 +126,7 @@ public class MetaDataRepository {
* @return
* @throws IOException
*/
- public List<AutoCompleteEntry> getKeywordsAutoComplete(String keyWordPrefix, int max) throws IOException {
+ public List<AutoCompleteEntry> getKeywordsAutoComplete(String keyWordPrefix, int max) {
return getFieldAutoCompletion(keyWordPrefix, SearchRepositoryConstants.METADATA_KEYWORDS, max);
}
@@ -114,7 +137,7 @@ public class MetaDataRepository {
* @return
* @throws IOException
*/
- public List<AutoCompleteEntry> getFormatsAutoComplete(String formatPrefix, int max) throws IOException {
+ public List<AutoCompleteEntry> getFormatsAutoComplete(String formatPrefix, int max) {
return getFieldAutoCompletion(formatPrefix, SearchRepositoryConstants.METADATA_FORMATS, max);
}
@@ -125,7 +148,7 @@ public class MetaDataRepository {
* @return
* @throws IOException
*/
- public List<AutoCompleteEntry> getProgrammingLanguageAutoComplete(String programmingLanguagePrefix, int max) throws IOException {
+ public List<AutoCompleteEntry> getProgrammingLanguageAutoComplete(String programmingLanguagePrefix, int max) {
return getFieldAutoCompletion(programmingLanguagePrefix, SearchRepositoryConstants.METADATA_PROGRAMMING_LANGUAGES, max);
}
@@ -136,7 +159,7 @@ public class MetaDataRepository {
* @return
* @throws IOException
*/
- public List<AutoCompleteEntry> getCreatorAutoComplete(String creatorPrefix, int max) throws IOException {
+ public List<AutoCompleteEntry> getCreatorAutoComplete(String creatorPrefix, int max) {
return getFieldAutoCompletion(creatorPrefix, SearchRepositoryConstants.METADATA_CREATOR, max);
}
@@ -147,7 +170,7 @@ public class MetaDataRepository {
* @return
* @throws IOException
*/
- public List<AutoCompleteEntry> getContributorAutoComplete(String contributorPrefix, int max) throws IOException {
+ public List<AutoCompleteEntry> getContributorAutoComplete(String contributorPrefix, int max) {
return getFieldAutoCompletion(contributorPrefix, SearchRepositoryConstants.METADATA_CONTRIBUTOR, max);
}
@@ -158,7 +181,7 @@ public class MetaDataRepository {
* @return
* @throws IOException
*/
- public List<AutoCompleteEntry> getContributorCreatorAutoComplete(String contributorPrefix, int max) throws IOException {
+ public List<AutoCompleteEntry> getContributorCreatorAutoComplete(String contributorPrefix, int max) {
final List<AutoCompleteEntry> contributors = getContributorAutoComplete(contributorPrefix, max);
final List<AutoCompleteEntry> creatorAutoComplete = getCreatorAutoComplete(contributorPrefix, max);
// merging
@@ -189,9 +212,7 @@ public class MetaDataRepository {
* @throws JsonProcessingException
* @throws JsonMappingException
*/
- private List<AutoCompleteEntry> getFieldAutoCompletion(String prefix, final String metaDataField, int maxCount) throws IOException {
- fillAutoCompletion();
-
+ private List<AutoCompleteEntry> getFieldAutoCompletion(String prefix, final String metaDataField, int maxCount) {
String lcKeyWordPrefix = prefix.toLowerCase(Locale.getDefault());
// höllisch kompliziert und höllisch unverständlich mit Streams :-)
// idea
@@ -222,112 +243,70 @@ public class MetaDataRepository {
}
+ @Scheduled(cron = "0 */10 * * * ?") // should run every 10 minutes
@PostConstruct
- @SuppressWarnings("PMD")
- private void setUpAutocompletion() {
- autocompletionReloadTimer.schedule(
- new TimerTask() {
- @Override
- public void run() {
- try {
- updateAutocompletionCache();
- } catch (IOException | RuntimeException e) {
- LOGGER.warn("Cannot refresh Autocompletion Cache", e);
- }
- }
- },
- 10000L,
- AUTOCOMPLETION_RECHECK_TIMEOUT
- );
- }
-
- /**
- * stops the cache reload timer, during shutdown or between tests.
- *
- * @param event ContextStoppedEvent
- */
- @EventListener
- protected void stopReloadTimer(ContextStoppedEvent event) {
- autocompletionReloadTimer.cancel();
- }
-
- private void fillAutoCompletion() throws IOException {
+ public void updateAutocompletionCache() throws IOException {
synchronized (this) {
- if (cachedCompletions == null) {
- updateAutocompletionCache();
- }
- }
- }
-
- @SuppressWarnings({ "PMD.CognitiveComplexity", "PMD.CyclomaticComplexity", "PMD.NPathComplexity" })
- private void updateAutocompletionCache() throws IOException {
- Map<String, Map<String, Map<String, Integer>>> reloadedCachedCompletions = new ConcurrentHashMap<>();
- reloadedCachedCompletions.put(SearchRepositoryConstants.METADATA_KEYWORDS, new TreeMap<>());
- reloadedCachedCompletions.put(SearchRepositoryConstants.METADATA_CREATOR, new TreeMap<>());
- reloadedCachedCompletions.put(SearchRepositoryConstants.METADATA_CONTRIBUTOR, new TreeMap<>());
- reloadedCachedCompletions.put(SearchRepositoryConstants.METADATA_PROGRAMMING_LANGUAGES, new TreeMap<>());
- reloadedCachedCompletions.put(SearchRepositoryConstants.METADATA_FORMATS, new TreeMap<>());
-
- SearchRequest searchRequest = new SearchRequest(SearchRepositoryConstants.INDEX_METADATA);
-
- final int pageSize = 20;
- int currentPage = 0;
- boolean tryNextPage = true;
-
- while (tryNextPage) {
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().size(pageSize).from(currentPage++ * pageSize);
-
- searchRequest.source(sourceBuilder);
-
- SearchResponse searchResponse = elasticsearchClient.search(searchRequest, RequestOptions.DEFAULT);
-
- ObjectMapper objectMapper = getSearchResultObjectMapper();
+ Map<String, Map<String, Map<String, Integer>>> reloadedCachedCompletions = new ConcurrentHashMap<>();
+ reloadedCachedCompletions.put(SearchRepositoryConstants.METADATA_KEYWORDS, new TreeMap<>());
+ reloadedCachedCompletions.put(SearchRepositoryConstants.METADATA_CREATOR, new TreeMap<>());
+ reloadedCachedCompletions.put(SearchRepositoryConstants.METADATA_CONTRIBUTOR, new TreeMap<>());
+ reloadedCachedCompletions.put(SearchRepositoryConstants.METADATA_PROGRAMMING_LANGUAGES, new TreeMap<>());
+ reloadedCachedCompletions.put(SearchRepositoryConstants.METADATA_FORMATS, new TreeMap<>());
- tryNextPage = searchResponse.getHits().getHits().length > 0;
-
- for (SearchHit searchHit : searchResponse.getHits().getHits()) {
- final String sourceAsString = filterDateString(searchHit.getSourceAsString());
- SearchResultDTO entry = objectMapper.readValue(sourceAsString, SearchResultDTO.class);
- if (entry.getMetadata().getKeyword() != null) {
- for (String keyWord : entry.getMetadata().getKeyword()) {
- addTo(reloadedCachedCompletions.get(SearchRepositoryConstants.METADATA_KEYWORDS), split(keyWord), keyWord);
- }
- }
- if (entry.getMetadata().getFormat() != null) {
- for (String format : entry.getMetadata().getFormat()) {
- addTo(reloadedCachedCompletions.get(SearchRepositoryConstants.METADATA_FORMATS), split(format), format);
- }
- }
- if (entry.getMetadata().getContributor() != null) {
- for (Person contributor : entry.getMetadata().getContributor()) {
- addTo(
- reloadedCachedCompletions.get(SearchRepositoryConstants.METADATA_CONTRIBUTOR),
- split(contributor.getName()),
- contributor.getName()
- );
- }
- }
- if (entry.getMetadata().getCreator() != null) {
- for (Person creator : entry.getMetadata().getCreator()) {
- addTo(
- reloadedCachedCompletions.get(SearchRepositoryConstants.METADATA_CREATOR),
- split(creator.getName()),
- creator.getName()
- );
- }
- }
- if (entry.getMetadata().getProgrammingLanguage() != null) {
- for (String language : entry.getMetadata().getProgrammingLanguage()) {
- addTo(
- reloadedCachedCompletions.get(SearchRepositoryConstants.METADATA_PROGRAMMING_LANGUAGES),
- Collections.singletonList(language),
- language
- );
- }
- }
+ try {
+ this.getAllResultsAsStream()
+ .forEach(entry -> {
+ if (entry.getMetadata().getKeyword() != null) {
+ for (String keyWord : entry.getMetadata().getKeyword()) {
+ addTo(reloadedCachedCompletions.get(SearchRepositoryConstants.METADATA_KEYWORDS), split(keyWord), keyWord);
+ }
+ }
+ if (entry.getMetadata().getFormat() != null) {
+ for (String format : entry.getMetadata().getFormat()) {
+ addTo(reloadedCachedCompletions.get(SearchRepositoryConstants.METADATA_FORMATS), split(format), format);
+ }
+ }
+ if (entry.getMetadata().getContributor() != null) {
+ for (Person contributor : entry.getMetadata().getContributor()) {
+ addTo(
+ reloadedCachedCompletions.get(SearchRepositoryConstants.METADATA_CONTRIBUTOR),
+ split(contributor.getName()),
+ contributor.getName()
+ );
+ }
+ }
+ if (entry.getMetadata().getCreator() != null) {
+ for (Person creator : entry.getMetadata().getCreator()) {
+ addTo(
+ reloadedCachedCompletions.get(SearchRepositoryConstants.METADATA_CREATOR),
+ split(creator.getName()),
+ creator.getName()
+ );
+ }
+ }
+ if (entry.getMetadata().getProgrammingLanguage() != null) {
+ for (String language : entry.getMetadata().getProgrammingLanguage()) {
+ addTo(
+ reloadedCachedCompletions.get(SearchRepositoryConstants.METADATA_PROGRAMMING_LANGUAGES),
+ Collections.singletonList(language),
+ language
+ );
+ }
+ }
+ });
+
+ cachedCompletions = reloadedCachedCompletions;
+ } catch (ElasticsearchException | ConnectException ex) {
+ // this may happen mainly during testing, when
+ // test elasticsearch server is not setup correctly
+ LOGGER.error("Cannot update completion Cache", ex);
+ } catch (IOException ex) {
+ // this may happen mainly during testing, when
+ // test elasticsearch server is not setup correctly
+ LOGGER.error("IOException: Cannot update completion Cache");
}
}
- cachedCompletions = reloadedCachedCompletions;
}
public static String filterDateString(String sourceAsJSON) {
@@ -341,7 +320,8 @@ public class MetaDataRepository {
*/
public static ObjectMapper getSearchResultObjectMapper() {
ObjectMapper objectMapper = new ObjectMapper();
- objectMapper.registerModule(new JavaTimeModule());
+ final JavaTimeModule javaTimeModule = new JavaTimeModule();
+ objectMapper.registerModule(javaTimeModule);
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
return objectMapper;
}
@@ -374,72 +354,149 @@ public class MetaDataRepository {
return result;
}
- public SearchResultsDTO getAllResources() throws IOException {
- SearchRequest searchRequest = new SearchRequest(SearchRepositoryConstants.INDEX_METADATA);
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
- sourceBuilder.size(10000);
- sourceBuilder.query(QueryBuilders.matchAllQuery());
- searchRequest.source(sourceBuilder);
- SearchResponse searchResponse = null;
- try {
- searchResponse = elasticsearchClient.search(searchRequest, RequestOptions.DEFAULT);
- } catch (IOException e) {
- LOGGER.warn("Error while searching for all resources", e);
- }
- return parseSearchResponse(searchResponse);
+ public List<SearchResultDTO> getAllResources() throws IOException {
+ return getAllResultsAsStream().collect(Collectors.toList());
}
public SearchResultsDTO searchByEmail(String email, int pageSize, Optional<User> user) throws IOException {
- SearchRequest searchRequest = new SearchRequest(SearchRepositoryConstants.INDEX_METADATA);
-
- BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+ BoolQuery.Builder queryBuilder = new BoolQuery.Builder();
- BoolQueryBuilder authorBuilder = QueryBuilders.boolQuery();
+ BoolQuery.Builder authorBuilder = new BoolQuery.Builder();
if (email != null) {
- authorBuilder.should(
- QueryBuilders
- .multiMatchQuery(
- email,
- SearchRepositoryConstants.METADATA_CREATOR_EMAIL,
- SearchRepositoryConstants.METADATA_CONTRIBUTOR_EMAIL,
- SearchRepositoryConstants.METADATA_PUBLISHER_EMAIL
- )
- .type(MultiMatchQueryBuilder.Type.PHRASE_PREFIX)
+ authorBuilder.should(q ->
+ q.multiMatch(q2 ->
+ new MultiMatchQuery.Builder()
+ .fields(
+ SearchRepositoryConstants.METADATA_CREATOR_EMAIL,
+ SearchRepositoryConstants.METADATA_CONTRIBUTOR_EMAIL,
+ SearchRepositoryConstants.METADATA_PUBLISHER_EMAIL
+ )
+ .type(TextQueryType.PhrasePrefix)
+ .query(email)
+ )
);
}
- if (authorBuilder.hasClauses()) {
- queryBuilder.must(authorBuilder);
+
+ queryBuilder.must(q -> q.bool(authorBuilder.build()));
+
+ addAuthorizationQueryWithJavaApi(user, queryBuilder);
+
+ try {
+ final BoolQuery query = queryBuilder.build();
+ // LOGGER.info("ElasticSearch Query using Java Client API:\n{}", query)
+ co.elastic.clients.elasticsearch.core.SearchResponse<SearchResultDTO> searchResponse = elasticsearchAPIClient.search(
+ search -> search.index(SearchRepositoryConstants.INDEX_METADATA).query(q -> q.bool(query)).from(0).size(pageSize),
+ SearchResultDTO.class
+ );
+
+ return parseSearchResponseWithJavaAPI(searchResponse);
+ } catch (ElasticsearchException ese) {
+ LOGGER.warn("Exception in elastic search", ese);
+ throw ese;
+ } catch (ConnectException ce) {
+ final RestClientTransport esTransport = (RestClientTransport) elasticsearchAPIClient._transport();
+ final String nodes = esTransport.restClient().getNodes().stream().map(Object::toString).collect(Collectors.joining(", "));
+ LOGGER.warn("ConnectionException in elastic search: {} ", nodes, ce);
+ throw ce;
}
+ }
- // Authorization restrictions
- final BoolQueryBuilder publicQuery = createPublicQuery();
+ public static class StreamedSearchResults {
- if (user.isEmpty()) {
- queryBuilder.must(publicQuery);
- } else {
- final Collection<GrantedAuthority> authorities = user.get().getAuthorities();
- final BoolQueryBuilder authQuery = QueryBuilders.boolQuery().boost(0.0f);
- final Stream<QueryBuilder> prefixAuthQueries = authorities
- .stream()
- .filter(auth -> !auth.getAuthority().startsWith("ROLE"))
- .filter(auth -> !auth.getAuthority().startsWith(SCOPE))
- .map(auth -> QueryBuilders.prefixQuery(SearchRepositoryConstants.PROJECT_NAMESPACE, auth.getAuthority()));
- prefixAuthQueries.forEach(authQuery::should);
- if (authQuery.hasClauses()) {
- queryBuilder.must(QueryBuilders.boolQuery().should(authQuery).should(publicQuery));
- } else {
- queryBuilder.must(publicQuery);
- }
+ private final long estimatedSize;
+ private final Stream<SearchResultDTO> searchStream;
+
+ protected StreamedSearchResults(long estimatedSize, Stream<SearchResultDTO> searchStream) {
+ super();
+ this.estimatedSize = estimatedSize;
+ this.searchStream = searchStream;
+ }
+
+ public long getEstimatedSize() {
+ return estimatedSize;
}
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
- sourceBuilder.query(queryBuilder).size(pageSize).from(0);
+ public Stream<SearchResultDTO> getSearchStream() {
+ return searchStream;
+ }
+ }
- searchRequest.source(sourceBuilder);
+ /**
+ * Method to stream all exercises.
+ *
+ * @return the search result page DTO
+ * @throws IOException
+ */
+ public Stream<SearchResultDTO> getAllResultsAsStream() throws IOException {
+ final org.codeability.sharing.plugins.api.search.SearchInputMetadataDTO emptySearchMetadata = new org.codeability.sharing.plugins.api.search.SearchInputMetadataDTO(
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+ SearchInputDTO emptySearchQuery = new SearchInputDTO(null, emptySearchMetadata, null, null, null, 0);
+ return streamResults(emptySearchQuery, Optional.empty(), true).getSearchStream();
+ }
- SearchResponse searchResponse = elasticsearchClient.search(searchRequest, RequestOptions.DEFAULT);
+ /**
+ * returns the hits from the meta data search index for the input query
+ * parameters.
+ *
+ * @param searchInputDTO the query parameters.
+ * @param user the user that queries.
+ * @param ignoreUserAuthorization if true, user authorization is ignored
+ * @return the search hits.
+ * @throws IOException when search index is not available.
+ */
+ @SuppressWarnings({ "PMD.ExcessiveMethodLength", "PMD.NcssCount" })
+ public StreamedSearchResults streamResults(
+ final SearchInputDTO searchInputDTO,
+ final Optional<User> user,
+ boolean ignoreUserAuthorization
+ ) throws IOException {
+ searchInputDTO.setPageSize(SCROLL_SIZE); // for streaming we set the page size to SCROLL_SIZE
+ searchInputDTO.setPage(0); // we force start page to 0!
+
+ final SearchResultsDTO pageDetails = pageDetailsWithJavaApi(searchInputDTO, user, ignoreUserAuthorization);
+
+ final long size = pageDetails.getHitCount();
+
+ Iterator<SearchResultDTO> resultsIterator = new Iterator<>() {
+ int count = 0;
+ int pagePointer = 0;
+ SearchResultsDTO currentPageDetails = pageDetails;
+
+ @Override
+ public boolean hasNext() {
+ return count < currentPageDetails.getHitCount();
+ }
- return parseSearchResponse(searchResponse);
+ @Override
+ public SearchResultDTO next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ if (pagePointer < SCROLL_SIZE) {
+ count++;
+ return currentPageDetails.getSearchResult().get(pagePointer++);
+ } else {
+ // load next page
+ searchInputDTO.setPage(searchInputDTO.getPage() + 1);
+ try {
+ currentPageDetails = pageDetailsWithJavaApi(searchInputDTO, user, ignoreUserAuthorization);
+ } catch (IOException e) {
+ throw new ConcurrentModificationException("It seems that scroll result changed during update", e);
+ }
+ pagePointer = 0;
+ return next();
+ }
+ }
+ };
+
+ var resultStream = StreamSupport.stream(Spliterators.spliterator(resultsIterator, (int) size, Spliterator.SIZED), false);
+ return new StreamedSearchResults(size, resultStream);
}
/**
@@ -453,60 +510,122 @@ public class MetaDataRepository {
* @return the search hits.
* @throws IOException when search index is not available.
*/
- @SuppressWarnings(
- { "PMD.CognitiveComplexity", "PMD.CyclomaticComplexity", "PMD.NPathComplexity", "PMD.ExcessiveMethodLength", "PMD.NcssCount" }
- )
- public SearchResultsDTO pageDetails(org.codeability.sharing.plugins.api.search.SearchInputDTO searchInputDTO, Optional<User> user)
+ @SuppressWarnings({ "PMD.ExcessiveMethodLength", "PMD.NcssCount" })
+ public SearchResultsDTO pageDetailsWithJavaApi(SearchInputDTO searchInputDTO, Optional<User> user) throws IOException {
+ return pageDetailsWithJavaApi(searchInputDTO, user, false);
+ }
+
+ /**
+ * returns the hits from the meta data search index for the input query
+ * parameters.
+ *
+ * @param searchInputDTO the query parameters.
+ * @param user the user that queries.
+ * @param ignoreUserAuthorization ignores the authorization of the user
+ * @return the search hits.
+ * @throws IOException when search index is not available.
+ */
+ @SuppressWarnings({ "PMD.ExcessiveMethodLength", "PMD.NcssCount" })
+ public SearchResultsDTO pageDetailsWithJavaApi(SearchInputDTO searchInputDTO, Optional<User> user, boolean ignoreUserAuthorization)
throws IOException {
- fillAutoCompletion();
+ BoolQuery.Builder queryBuilder = createQueryBuilder(searchInputDTO, user, ignoreUserAuthorization);
+
+ final int pageSize = searchInputDTO.getPageSize();
+ final int from = (searchInputDTO.getPage()) * pageSize;
- BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+ try {
+ final BoolQuery query = queryBuilder.build();
+ // LOGGER.info("ElasticSearch Query using Java Client API:\n{}", query)
+ co.elastic.clients.elasticsearch.core.SearchResponse<SearchResultDTO> searchResponse = elasticsearchAPIClient.search(
+ search -> search.index(SearchRepositoryConstants.INDEX_METADATA).query(q -> q.bool(query)).from(from).size(pageSize),
+ SearchResultDTO.class
+ );
+
+ return parseSearchResponseWithJavaAPI(searchResponse);
+ } catch (ElasticsearchException ese) {
+ LOGGER.warn("Exception in elastic search", ese);
+ throw ese;
+ } catch (ConnectException ce) {
+ final RestClientTransport esTransport = (RestClientTransport) elasticsearchAPIClient._transport();
+ final String nodes = esTransport.restClient().getNodes().stream().map(Object::toString).collect(Collectors.joining(", "));
+ LOGGER.warn("ConnectionException in elastic search: {} ", nodes, ce);
+ throw ce;
+ }
+ }
+
+ /**
+ * creates the query (builder) for the searchInput and the user authorization
+ *
+ * @param searchInputDTO the search input
+ * @param user the user (optional)
+ * @param ignoreUserAuthorization if true, the user authorization is ignored
+ * @return
+ */
+ @SuppressWarnings("PMD.ExcessiveMethodLength")
+ private BoolQuery.Builder createQueryBuilder(SearchInputDTO searchInputDTO, Optional<User> user, boolean ignoreUserAuthorization) {
+ BoolQuery.Builder queryBuilder = new BoolQuery.Builder();
- BoolQueryBuilder fullTextBuilder = QueryBuilders.boolQuery();
forEachTerm(
searchInputDTO.getFulltextQuery(),
term ->
- fullTextBuilder.should(
- QueryBuilders
- .multiMatchQuery(
- term,
- SearchRepositoryConstants.METADATA_DESCRIPTION,
- SearchRepositoryConstants.METADATA_KEYWORDS,
- SearchRepositoryConstants.METADATA_TITLE
- )
- .type(MultiMatchQueryBuilder.Type.PHRASE_PREFIX)
+ queryBuilder.must(q ->
+ q.multiMatch(
+ new MultiMatchQuery.Builder()
+ .query(term)
+ .fields(
+ SearchRepositoryConstants.METADATA_DESCRIPTION,
+ SearchRepositoryConstants.METADATA_KEYWORDS,
+ SearchRepositoryConstants.METADATA_TITLE
+ )
+ .type(TextQueryType.PhrasePrefix)
+ .build()
+ )
)
);
- if (fullTextBuilder.hasClauses()) {
- queryBuilder.must(fullTextBuilder);
- }
- BoolQueryBuilder plBuilder = QueryBuilders.boolQuery();
+ BoolQuery.Builder plBuilder = new BoolQuery.Builder();
forEachTerm(
searchInputDTO.getMetadata().getProgrammingLanguage(),
- term -> plBuilder.should(QueryBuilders.prefixQuery(SearchRepositoryConstants.METADATA_PROGRAMMING_LANGUAGES, term))
+ term ->
+ plBuilder.should(q ->
+ q.prefix(
+ new PrefixQuery.Builder()
+ .value(term)
+ .caseInsensitive(true)
+ .field(SearchRepositoryConstants.METADATA_PROGRAMMING_LANGUAGES)
+ .build()
+ )
+ ),
+ () -> queryBuilder.must(q -> q.bool(plBuilder.build()))
);
- if (plBuilder.hasClauses()) {
- queryBuilder.must(plBuilder);
- }
- BoolQueryBuilder keywordBuilder = QueryBuilders.boolQuery();
+ BoolQuery.Builder keywordBuilder = new BoolQuery.Builder();
forEachTerm(
searchInputDTO.getMetadata().getKeyword(),
- term -> keywordBuilder.should(QueryBuilders.prefixQuery(SearchRepositoryConstants.METADATA_KEYWORDS, term))
+ term ->
+ keywordBuilder.should(q ->
+ q.matchPhrasePrefix(
+ new MatchPhrasePrefixQuery.Builder().query(term).field(SearchRepositoryConstants.METADATA_KEYWORDS).build()
+ )
+ ),
+ () -> queryBuilder.must(q -> q.bool(keywordBuilder.build()))
);
- if (keywordBuilder.hasClauses()) {
- queryBuilder.must(keywordBuilder);
- }
- BoolQueryBuilder formatBuilder = QueryBuilders.boolQuery();
+ BoolQuery.Builder formatBuilder = new BoolQuery.Builder();
forEachTerm(
searchInputDTO.getMetadata().getFormat(),
- term -> formatBuilder.should(QueryBuilders.prefixQuery(SearchRepositoryConstants.METADATA_FORMATS, term))
+ term ->
+ formatBuilder.should(q ->
+ q.prefix(
+ new PrefixQuery.Builder()
+ .value(term)
+ .field(SearchRepositoryConstants.METADATA_FORMATS)
+ .caseInsensitive(true)
+ .build()
+ )
+ ),
+ () -> queryBuilder.must(q -> q.bool(formatBuilder.build()))
);
- if (formatBuilder.hasClauses()) {
- queryBuilder.must(formatBuilder);
- }
String parentId = searchInputDTO.getMetadata().getParentId();
SearchResultDTO parentExercise = null;
@@ -514,7 +633,7 @@ public class MetaDataRepository {
try {
parentExercise = getExerciseById(parentId, user);
} catch (NotFoundException e) {
- String stripped = convertToPreviousId(parentId);
+ String stripped = convertToOldFormat(parentId);
if (stripped.equals(parentId)) {
try {
parentExercise = getExerciseById(stripped, user);
@@ -528,73 +647,69 @@ public class MetaDataRepository {
return null;
} else {
String[] childrenIds = parentExercise.getFile().getChildren();
- BoolQueryBuilder collectionBuilder = QueryBuilders.boolQuery();
+ BoolQuery.Builder collectionBuilder = new BoolQuery.Builder();
for (String childId : childrenIds) {
- collectionBuilder.should(QueryBuilders.termQuery(SearchRepositoryConstants.EXERCISE_ID, childId));
- String previousId = convertToPreviousId(childId);
+ collectionBuilder.should(q ->
+ q.term((new TermQuery.Builder()).value(childId).field(SearchRepositoryConstants.EXERCISE_ID).build())
+ );
+ String previousId = convertToOldFormat(childId);
if (!previousId.equals(childId)) {
- collectionBuilder.should(QueryBuilders.termQuery(SearchRepositoryConstants.EXERCISE_ID, previousId));
+ collectionBuilder.should(q ->
+ q.term((new TermQuery.Builder()).value(previousId).field(SearchRepositoryConstants.EXERCISE_ID).build())
+ );
}
}
- queryBuilder.must(collectionBuilder);
+ queryBuilder.must(q -> q.bool(collectionBuilder.build()));
}
}
- BoolQueryBuilder authorBuilder = QueryBuilders.boolQuery();
if (searchInputDTO.getMetadata().getAuthor() != null) {
- authorBuilder.should(
- QueryBuilders
- .multiMatchQuery(
- searchInputDTO.getMetadata().getAuthor(),
- SearchRepositoryConstants.METADATA_CREATOR,
- SearchRepositoryConstants.METADATA_CONTRIBUTOR,
- SearchRepositoryConstants.METADATA_PUBLISHER
+ queryBuilder.must(q1 ->
+ q1.bool(q2 ->
+ q2.should(q ->
+ q.multiMatch(
+ new MultiMatchQuery.Builder()
+ .query(searchInputDTO.getMetadata().getAuthor())
+ .type(TextQueryType.PhrasePrefix)
+ .fields(
+ SearchRepositoryConstants.METADATA_CREATOR,
+ SearchRepositoryConstants.METADATA_CONTRIBUTOR,
+ SearchRepositoryConstants.METADATA_PUBLISHER
+ )
+ .build()
+ )
)
- .type(MultiMatchQueryBuilder.Type.PHRASE_PREFIX)
+ )
);
}
- if (authorBuilder.hasClauses()) {
- queryBuilder.must(authorBuilder);
- }
if (searchInputDTO.getMetadata().getNaturalLanguage() != null && !searchInputDTO.getMetadata().getNaturalLanguage().isEmpty()) {
- BoolQueryBuilder nlBuilder = QueryBuilders.boolQuery();
+ BoolQuery.Builder nlBuilder = new BoolQuery.Builder();
+
searchInputDTO
.getMetadata()
.getNaturalLanguage()
- .forEach(language -> nlBuilder.should(QueryBuilders.matchQuery(SearchRepositoryConstants.METADATA_LANGUAGE, language)));
- if (nlBuilder.hasClauses()) {
- queryBuilder.must(nlBuilder);
- }
+ .forEach(language ->
+ nlBuilder.should(q ->
+ q.match(new MatchQuery.Builder().query(language).field(SearchRepositoryConstants.METADATA_LANGUAGE).build())
+ )
+ );
+ queryBuilder.must(q -> q.bool((nlBuilder.build())));
}
forEachTerm(
searchInputDTO.getMetadata().getLicense(),
- term -> queryBuilder.must(QueryBuilders.prefixQuery(SearchRepositoryConstants.METADATA_LICENSE, term))
+ term ->
+ queryBuilder.must(q ->
+ q.prefix(new PrefixQuery.Builder().value(term).field(SearchRepositoryConstants.METADATA_LICENSE).build())
+ )
);
- addAuthorizationQuery(user, queryBuilder);
-
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
- sourceBuilder
- .query(queryBuilder)
- .size(searchInputDTO.getPageSize())
- .from((searchInputDTO.getPage()) * searchInputDTO.getPageSize());
-
- SearchRequest searchRequest = new SearchRequest(SearchRepositoryConstants.INDEX_METADATA);
- searchRequest.source(sourceBuilder);
-
- SearchResponse searchResponse;
-
- final Builder optionsBuilder = RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> false);
- try {
- searchResponse = elasticsearchClient.search(searchRequest, optionsBuilder.build());
- } catch (ElasticsearchException ese) {
- LOGGER.warn("Exception in elastic search", ese);
- throw ese;
+ if (!ignoreUserAuthorization) {
+ addAuthorizationQueryWithJavaApi(user, queryBuilder);
}
- return parseSearchResponse(searchResponse);
+ return queryBuilder;
}
/**
@@ -604,7 +719,7 @@ public class MetaDataRepository {
* @param exerciseId
* @return
*/
- public String convertToPreviousId(String exerciseId) {
+ public String convertToOldFormat(String exerciseId) {
Pattern newParser = Pattern.compile("^\\[(\\d+)\\](.*)$");
Matcher matcher = newParser.matcher(exerciseId.trim());
if (matcher.matches()) {
@@ -627,61 +742,79 @@ public class MetaDataRepository {
* @param user
* @param queryBuilder
*/
- private void addAuthorizationQuery(Optional<User> user, BoolQueryBuilder queryBuilder) {
+ private void addAuthorizationQueryWithJavaApi(Optional<User> user, BoolQuery.Builder queryBuilder) {
// Authorization restrictions
- final BoolQueryBuilder publicQuery = createPublicQuery();
+ final TermQuery.Builder simplePublicQuery = new TermQuery.Builder()
+ .value("public")
+ .field(SearchRepositoryConstants.PROJECT_VISIBILITY)
+ .boost(0.0f);
+ final ExistsQuery.Builder publicVisibilityQuery = new ExistsQuery.Builder()
+ .field(SearchRepositoryConstants.METADATA_PUBLICVISIBILITY);
+ final BoolQuery.Builder publicQuery = new BoolQuery.Builder()
+ .should(q -> q.term(simplePublicQuery.build()))
+ .should(q -> q.exists(publicVisibilityQuery.build()));
if (user.isEmpty()) {
- queryBuilder.must(publicQuery);
+ queryBuilder.must(q -> q.bool(publicQuery.build()));
} else {
+ final AtomicBoolean hasAuthQueries = new AtomicBoolean(false);
final Collection<GrantedAuthority> authorities = user.get().getAuthorities();
- final BoolQueryBuilder authQuery = QueryBuilders.boolQuery().boost(0.0f);
+ final BoolQuery.Builder authQuery = new BoolQuery.Builder().boost(0.0f);
// old membership management (can be eliminated some day)
- final Stream<QueryBuilder> prefixAuthQueries = authorities
+ final Stream<PrefixQuery.Builder> prefixAuthQueries = authorities
.stream()
.filter(auth -> !Authority.isStandardRole(auth.getAuthority()))
.filter(auth -> !auth.getAuthority().startsWith(SCOPE))
- .map(auth -> QueryBuilders.termQuery(SearchRepositoryConstants.PROJECT_NAMESPACE, auth.getAuthority()));
- prefixAuthQueries.forEach(pq -> authQuery.should(pq));
+ .map(auth -> new PrefixQuery.Builder().field(SearchRepositoryConstants.PROJECT_NAMESPACE).value(auth.getAuthority()));
+ prefixAuthQueries.forEach(pq -> {
+ hasAuthQueries.set(true);
+ authQuery.should(q -> q.prefix(pq.build()));
+ });
// new membership management
- final Stream<QueryBuilder> prefixAuthQueries2 = authorities
+ final Stream<TermQuery.Builder> prefixAuthQueries2 = authorities
.stream()
.filter(auth -> !Authority.isStandardRole(auth.getAuthority()))
.filter(auth -> !auth.getAuthority().startsWith(SCOPE))
- .map(auth -> QueryBuilders.termQuery(SearchRepositoryConstants.PROJECT_GROUPS, auth.getAuthority()));
- prefixAuthQueries2.forEach(authQuery::should);
+ .map(auth -> new TermQuery.Builder().field(SearchRepositoryConstants.PROJECT_GROUPS).value(auth.getAuthority()));
+ prefixAuthQueries2.forEach(paq -> {
+ hasAuthQueries.set(true);
+ authQuery.should(q -> q.term(paq.build()));
+ });
+
final String email = user.get().getUsername().toLowerCase(Locale.getDefault());
if (StringUtils.isNotEmpty(email)) {
- authQuery.should(QueryBuilders.termQuery(SearchRepositoryConstants.PROJECT_USERS, email));
+ hasAuthQueries.set(true);
+ authQuery.should(q -> q.term(new TermQuery.Builder().field(SearchRepositoryConstants.PROJECT_USERS).value(email).build()));
}
- if (authQuery.hasClauses()) {
- queryBuilder.must(QueryBuilders.boolQuery().should(authQuery).should(publicQuery));
+
+ if (hasAuthQueries.get()) {
+ queryBuilder.must(q1 ->
+ q1.bool(new BoolQuery.Builder().should(q -> q.bool(authQuery.build())).should(q -> q.bool(publicQuery.build())).build())
+ );
} else {
- queryBuilder.must(publicQuery);
+ queryBuilder.must(q -> q.bool(publicQuery.build()));
}
}
}
- private BoolQueryBuilder createPublicQuery() {
- final TermQueryBuilder simplePublicQuery = QueryBuilders
- .termQuery(SearchRepositoryConstants.PROJECT_VISIBILITY, "public")
- .boost(0.0f);
- final ExistsQueryBuilder publicVisibilityQuery = QueryBuilders.existsQuery(SearchRepositoryConstants.METADATA_PUBLICVISIBILITY);
-
- return QueryBuilders.boolQuery().should(simplePublicQuery).should(publicVisibilityQuery);
- }
-
- private static SearchResultsDTO parseSearchResponse(final SearchResponse searchResponse) throws JsonProcessingException {
- float maxScore = searchResponse.getHits().getMaxScore();
- ObjectMapper objectMapper = new ObjectMapper();
- objectMapper.registerModule(new JavaTimeModule());
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ private static SearchResultsDTO parseSearchResponseWithJavaAPI(
+ final co.elastic.clients.elasticsearch.core.SearchResponse<SearchResultDTO> searchResponse
+ ) {
+ final Double maxScoreD = searchResponse.hits().maxScore();
+ final double maxScore = maxScoreD == null ? 0.0d : maxScoreD;
final List<SearchResultDTO> searchResults = new ArrayList<>();
- for (SearchHit searchHit : searchResponse.getHits().getHits()) {
- float hitScore = searchHit.getScore();
- final SearchResultDTO parsedSearchHit = parseSearchHit(searchHit, objectMapper);
+ for (Hit<SearchResultDTO> searchHit : searchResponse.hits().hits()) {
+ final Double scoreD = searchHit.score();
+ double hitScore = scoreD == null ? 0.0 : scoreD;
+ final SearchResultDTO parsedSearchHit = searchHit.source();
+ if (parsedSearchHit == null) {
+ LOGGER.warn("received null search hit?");
+ continue;
+ }
+
+ parsedSearchHit.setExerciseId(searchHit.id());
int ranking5 = 1;
if (maxScore > 0.0f) {
ranking5 = (int) ((hitScore * 5.0) / maxScore + 0.5f);
@@ -690,10 +823,10 @@ public class MetaDataRepository {
searchResults.add(parsedSearchHit);
}
- TotalHits totalHits = searchResponse.getHits().getTotalHits();
+ co.elastic.clients.elasticsearch.core.search.TotalHits totalHits = searchResponse.hits().total();
long hitCount = 0;
if (totalHits != null) {
- hitCount = totalHits.value;
+ hitCount = totalHits.value();
}
final SearchResultsDTO results = new SearchResultsDTO();
results.setHitCount(hitCount);
@@ -701,22 +834,26 @@ public class MetaDataRepository {
return results;
}
- private static SearchResultDTO parseSearchHit(SearchHit searchHit, ObjectMapper objectMapper) throws JsonProcessingException {
- String sourceAsString = filterDateString(searchHit.getSourceAsString());
- final SearchResultDTO mappedValue = objectMapper.readValue(sourceAsString, SearchResultDTO.class);
- mappedValue.setExerciseId(searchHit.getId());
- return mappedValue;
+ /** Helper **/
+ private static void forEachTerm(String searchTerm, Consumer<String> exec) {
+ forEachTerm(searchTerm, exec, () -> {});
}
/** Helper **/
- private static void forEachTerm(String searchTerm, Consumer<String> exec) {
+ private static void forEachTerm(String searchTerm, Consumer<String> exec, Runnable whenAtLeastOne) {
if (searchTerm == null) {
return;
}
StringTokenizer st = new StringTokenizer(searchTerm);
+
+ boolean atLeastOne = false;
while (st.hasMoreTokens()) {
+ atLeastOne = true;
exec.accept(st.nextToken());
}
+ if (atLeastOne) {
+ whenAtLeastOne.run();
+ }
}
public SearchResultsDTO getExercisesById(Stream<String> exerciseIds, Optional<User> user, int page, int pageSize) {
@@ -736,42 +873,60 @@ public class MetaDataRepository {
* @return
*/
public SearchResultsDTO getExercisesById(Stream<String> exerciseIds, Optional<User> user, int page, int pageSize, Instant since) {
- BoolQueryBuilder idQueryBuilder = QueryBuilders.boolQuery();
- exerciseIds.forEach(exerciseId -> idQueryBuilder.should(QueryBuilders.matchQuery(SearchRepositoryConstants.EXERCISE_ID, exerciseId))
- );
+ BoolQuery.Builder idQueryBuilder = new BoolQuery.Builder();
+
+ AtomicInteger idCounter = new AtomicInteger(0);
+ exerciseIds.forEach(exerciseId -> {
+ idCounter.incrementAndGet();
+ idQueryBuilder.should(q ->
+ q.match(q2 -> new MatchQuery.Builder().field(SearchRepositoryConstants.EXERCISE_ID).query(exerciseId))
+ );
+ });
+
// Böse Falle: Wenn exerciseIds empty, dann wird alles zurückgegeben!
- if (!idQueryBuilder.hasClauses()) {
+ if (idCounter.get() == 0) {
final SearchResultsDTO results = new SearchResultsDTO();
results.setHitCount(0L);
results.setSearchResult(new ArrayList<>());
return results;
}
- BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+ BoolQuery.Builder queryBuilder = new BoolQuery.Builder();
+
+ queryBuilder.must(q -> q.bool(idQueryBuilder.build()));
- queryBuilder.must(idQueryBuilder);
if (since != null) {
- queryBuilder.must(
- QueryBuilders.rangeQuery(SearchRepositoryConstants.PROJECT_LASTACTIVITYAT).gte(since)
- /* .lte(Instant.now()) */
+ // Noch ein böse Falle! JsonData.of(since) konvertiert nicht in Zeitstring
+ // sondern in
+ // ein Array of Strings
+ String sinceString = DateTimeFormatter.ISO_INSTANT.format(since);
+ // "2023-03-15T08:32:52
+ queryBuilder.must(q ->
+ q.range(
+ new RangeQuery.Builder().field(SearchRepositoryConstants.PROJECT_LASTACTIVITYAT).gte(JsonData.of(sinceString)).build()
+ )
);
}
- addAuthorizationQuery(user, queryBuilder);
-
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
- sourceBuilder.query(queryBuilder).size(pageSize).from(page * pageSize);
-
- SearchRequest searchRequest = new SearchRequest(SearchRepositoryConstants.INDEX_METADATA);
- searchRequest.source(sourceBuilder);
+ addAuthorizationQueryWithJavaApi(user, queryBuilder);
try {
- SearchResponse searchResponse = elasticsearchClient.search(searchRequest, RequestOptions.DEFAULT);
+ final BoolQuery query = queryBuilder.build();
+ LOGGER.info("ElasticSearch Query using Java Client API:\n{}", query);
+ co.elastic.clients.elasticsearch.core.SearchResponse<SearchResultDTO> searchResponse = elasticsearchAPIClient.search(
+ search ->
+ search.index(SearchRepositoryConstants.INDEX_METADATA).query(q -> q.bool(query)).from(page * pageSize).size(pageSize),
+ SearchResultDTO.class
+ );
- return parseSearchResponse(searchResponse);
- } catch (JsonProcessingException e) {
- throw new NotFoundException("Unable to process JSON, Exercise with id " + idQueryBuilder.toString() + " not found", e);
- } catch (IOException e) {
- throw new NotFoundException("IOException, Query for exercise with id " + idQueryBuilder.toString() + " went wrong", e);
+ return parseSearchResponseWithJavaAPI(searchResponse);
+ } catch (ElasticsearchException ese) {
+ LOGGER.warn("Exception in elastic search", ese);
+ throw ese;
+ } catch (IOException ce) {
+ final RestClientTransport esTransport = (RestClientTransport) elasticsearchAPIClient._transport();
+ final String nodes = esTransport.restClient().getNodes().stream().map(Object::toString).collect(Collectors.joining(", "));
+ LOGGER.warn("ConnectionException in elastic search: {} ", nodes, ce);
+ throw new ElasticsearchException("IOException", ce);
}
}
@@ -800,8 +955,10 @@ public class MetaDataRepository {
public TreeNode<SearchResultDTO> getExerciseCollectionTree(ExerciseId exerciseId, Optional<User> user) {
SearchResultDTO root = getExerciseById(exerciseId, user);
if (root == null) {
+ LOGGER.debug("Exercise with id {} not found", exerciseId);
return null;
}
+ LOGGER.debug("Loaded Exercise with id {}: ({})", exerciseId, root);
TreeNode<SearchResultDTO> result = new TreeNode<>(root);
final String[] children = root.getFile().getChildren();
if (children != null) {
@@ -838,42 +995,45 @@ public class MetaDataRepository {
if (!bracketedExerciseId.startsWith("[")) { // just for backward compatibility
bracketedExerciseId = "[" + exerciseId + "]";
}
- SearchRequest searchRequest = new SearchRequest(SearchRepositoryConstants.INDEX_METADATA);
+ final String finalBracketedExerciseId = bracketedExerciseId;
- BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+ BoolQuery.Builder queryBuilder = new BoolQuery.Builder();
- queryBuilder.must(QueryBuilders.matchQuery(SearchRepositoryConstants.EXERCISE_ID, bracketedExerciseId));
-
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
- sourceBuilder.query(queryBuilder).size(10);
- addAuthorizationQuery(user, queryBuilder);
+ queryBuilder.must(q ->
+ q.match(q2 -> new MatchQuery.Builder().field(SearchRepositoryConstants.EXERCISE_ID).query(finalBracketedExerciseId))
+ );
- searchRequest.source(sourceBuilder);
+ addAuthorizationQueryWithJavaApi(user, queryBuilder);
try {
- SearchResponse searchResponse = elasticsearchClient.search(searchRequest, RequestOptions.DEFAULT);
-
- ObjectMapper objectMapper = new ObjectMapper();
- objectMapper.registerModule(new JavaTimeModule());
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ final BoolQuery query = queryBuilder.build();
+ // LOGGER.info("ElasticSearch Query using Java Client API:\n{}", query)
+ co.elastic.clients.elasticsearch.core.SearchResponse<SearchResultDTO> searchResponse = elasticsearchAPIClient.search(
+ search -> search.index(SearchRepositoryConstants.INDEX_METADATA).query(q -> q.bool(query)).from(0).size(1),
+ SearchResultDTO.class
+ );
- TotalHits totalHits = searchResponse.getHits().getTotalHits();
+ co.elastic.clients.elasticsearch.core.search.TotalHits totalHits = searchResponse.hits().total();
long hitCount = 0;
if (totalHits != null) {
- hitCount = totalHits.value;
+ hitCount = totalHits.value();
}
if (hitCount > 1) {
LOGGER.warn("Found {} hits for exercise id {}, expected only one!", hitCount, exerciseId);
}
- if (searchResponse.getHits().getHits().length > 0) {
- final SearchResultDTO parseSearchHit = parseSearchHit(searchResponse.getHits().getHits()[0], objectMapper);
- parseSearchHit.setRanking5(5);
- return parseSearchHit;
+
+ if (hitCount > 0) {
+ final SearchResultsDTO parsedHits = parseSearchResponseWithJavaAPI(searchResponse);
+ return parsedHits.getSearchResult().get(0);
}
- } catch (JsonProcessingException e) {
- throw new NotFoundException("Failed to process Json when searching for exercise with id " + exerciseId, e);
- } catch (IOException e) {
- throw new NotFoundException("Encountered IOException when searching for exercise with id " + exerciseId, e);
+ } catch (ElasticsearchException ese) {
+ LOGGER.warn("Exception in elastic search", ese);
+ throw ese;
+ } catch (IOException ce) {
+ final RestClientTransport esTransport = (RestClientTransport) elasticsearchAPIClient._transport();
+ final String nodes = esTransport.restClient().getNodes().stream().map(Object::toString).collect(Collectors.joining(", "));
+ LOGGER.warn("ConnectionException in elastic search: {} ", nodes, ce);
+ throw new NotFoundException("Exercise with id " + exerciseId + " not found", ce);
}
throw new NotFoundException("Exercise with id " + exerciseId + " not found");
}
diff --git a/src/main/java/at/ac/uibk/gitsearch/security/oauth2/UserDetailsFetcher.java b/src/main/java/at/ac/uibk/gitsearch/security/oauth2/UserDetailsFetcher.java
index 90e0ec7f026f99a207e94a16c9e2a21be4fdde24..d6e3013014f949d6081a834209b2d0cd2ccc4255 100644
--- a/src/main/java/at/ac/uibk/gitsearch/security/oauth2/UserDetailsFetcher.java
+++ b/src/main/java/at/ac/uibk/gitsearch/security/oauth2/UserDetailsFetcher.java
@@ -11,6 +11,7 @@ import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Consumer;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.gitlab4j.api.Constants.TokenType;
@@ -94,13 +95,23 @@ public class UserDetailsFetcher {
String email = (String) oidcUser.getAttribute("email");
User gitUser = gitLabApi.getUserApi().getUserByEmail(email);
- userToken = requestGitLabUserToken(gitUser, gitLabApi);
if (gitUser == null) {
- u.setLastModifiedDate(Instant.now());
- modified = true;
- return new UserDetailsInfo(userToken, modified);
+ // try to create the user
+ gitUser = createNewGitLabUserFor(oidcUser);
+ String generatedPassword = generatePassword();
+ gitLabApi.getUserApi().createUser(gitUser, generatedPassword, true);
+ gitUser = gitLabApi.getUserApi().getUserByEmail(email);
+
+ if (gitUser == null) {
+ // gitUser still null. Something went wrong?
+ u.setLastModifiedDate(Instant.now());
+ modified = true;
+ return new UserDetailsInfo(userToken, modified);
+ }
}
+ userToken = requestGitLabUserToken(gitUser, gitLabApi);
+
List<Membership> memberships = gitLabApi.getUserApi().getMemberships(gitUser.getId());
for (Membership membership : memberships) {
authorities.add(membership.getSourceName());
@@ -125,6 +136,24 @@ public class UserDetailsFetcher {
return new UserDetailsInfo(userToken, modified);
}
+ private String generatePassword() {
+ return RandomStringUtils.random(8, "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz=%$-.");
+ }
+
+ private User createNewGitLabUserFor(final OidcUser oidcUser) {
+ User gitUser = new User();
+ String email = oidcUser.getEmail();
+ gitUser.setEmail(email);
+ final Object nameAttribute = oidcUser.getAttribute("name");
+ String name = nameAttribute == null ? "unknown" : nameAttribute.toString();
+ gitUser.setName(name);
+
+ String userName = email.replace('@', '_');
+
+ gitUser.setUsername(userName);
+ return gitUser;
+ }
+
Optional<ImpersonationToken> requestGitLabUserToken(final User gitUser, final GitLabApi gitLabApi) throws GitLabApiException {
Date expiresAt = new Date(System.currentTimeMillis() + tokenValidityInSeconds * 1000L);
diff --git a/src/main/java/at/ac/uibk/gitsearch/service/LikesService.java b/src/main/java/at/ac/uibk/gitsearch/service/LikesService.java
index 3ecad0503c849f88d68369fd9db15cb83cf65508..bc6eb8fb99a3fa16f15432bef234940445745679 100644
--- a/src/main/java/at/ac/uibk/gitsearch/service/LikesService.java
+++ b/src/main/java/at/ac/uibk/gitsearch/service/LikesService.java
@@ -42,10 +42,13 @@ public class LikesService {
*/
@Transactional(readOnly = true)
public List<Likes> findAll() {
- log.debug("Request to get all Likes");
return likesRepository.findAll();
}
+ public long countAll() {
+ return likesRepository.count();
+ }
+
/**
* Get one likes by id.
*
diff --git a/src/main/java/at/ac/uibk/gitsearch/service/MonitoringService.java b/src/main/java/at/ac/uibk/gitsearch/service/MonitoringService.java
index 14178ecf987d0a775156e2117b266f4c860f2d05..9f32aa99f9e05580e800854d8ce3788f7895a5a8 100644
--- a/src/main/java/at/ac/uibk/gitsearch/service/MonitoringService.java
+++ b/src/main/java/at/ac/uibk/gitsearch/service/MonitoringService.java
@@ -1,29 +1,24 @@
package at.ac.uibk.gitsearch.service;
import at.ac.uibk.gitsearch.service.dto.StatisticsDTO;
+import co.elastic.clients.elasticsearch._types.ElasticsearchException;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.Tags;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Stream;
+import javax.annotation.PostConstruct;
import org.codeability.sharing.plugins.api.search.SearchResultDTO;
-import org.codeability.sharing.plugins.api.search.SearchResultsDTO;
-import org.codeability.sharing.plugins.api.search.UserProvidedMetadataDTO;
-import org.elasticsearch.ElasticsearchException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
/**
* Service class for monitoring
@@ -31,44 +26,42 @@ import org.springframework.transaction.annotation.Transactional;
* @author Johannes Kainz
*/
@Service
-@Transactional
+// @Transactional // enabling this may cause NullPointerException, because of wrongly applied EnhancerBySpringCGLIB
public class MonitoringService {
+ private static final String UNCLASSIFIED = "Unclassified";
private static final String TAG_INTERVAL = "interval";
private static final String TAG_HOURLY_INTERVAL = "hourly";
- private static final String USER_TAG = "classification";
- private static final String LANGUAGE_TAG = "language";
+ public static final String USER_TAG = "classification";
+ public static final String LANGUAGE_TAG = "language";
- private static final String NUMBER_USER_NAME = "number.users";
- private static final String NUMBER_EXERCISE_NAME = "number.exercises";
- private static final String NUMBER_LIKE_NAME = "number.likes";
- private static final String NUMBER_VIEW_NAME = "number.views";
- private static final String NUMBER_DOWNLOAD_NAME = "number.downloads";
+ public static final String NUMBER_USER_NAME = "number.users";
+ public static final String NUMBER_EXERCISE_NAME = "number.exercises";
+ public static final String NUMBER_LIKE_NAME = "number.likes";
+ public static final String NUMBER_VIEW_NAME = "number.views";
+ public static final String NUMBER_DOWNLOAD_NAME = "number.downloads";
- private static final Tag HOURLY_TAG = Tag.of(TAG_INTERVAL, TAG_HOURLY_INTERVAL);
+ public static final Tag HOURLY_TAG = Tag.of(TAG_INTERVAL, TAG_HOURLY_INTERVAL);
private final Logger log = LoggerFactory.getLogger(MonitoringService.class);
private final UserService userService;
private final SearchService searchService;
private final LikesService likesService;
-
- private final AtomicLong numberUsers;
- private final AtomicLong numberActiveUsers;
-
- private final AtomicLong numberLikes;
- private final AtomicLong numberViews;
- private final AtomicLong numberDownloads;
- private final Map<String, AtomicInteger> numberExercises;
-
- private final Map<String, AtomicLong> numberLikesMap;
- private final Map<String, AtomicLong> numberViewsMap;
- private final Map<String, AtomicLong> numberDownloadsMap;
+ private final StatisticsService statisticsService;
private final MeterRegistry meterRegistry;
- private final StatisticsService statisticsService;
+ // the following elements must be stored explizit, to make sure
+ // that they are not garbage collected
+ private final AtomicInteger allUsers = new AtomicInteger(0);
+ private final AtomicInteger activeUsers = new AtomicInteger(0);
+ private final AtomicInteger allLikes = new AtomicInteger(0);
+ private final AtomicInteger allViews = new AtomicInteger(0);
+ private final AtomicInteger allDownloads = new AtomicInteger(0);
+ private final AtomicInteger allExercises = new AtomicInteger(0);
+ private final Map<String, AtomicInteger> allLanguageCount = new ConcurrentHashMap<>();
@SuppressWarnings("PMD.AvoidDuplicateLiterals")
public MonitoringService(
@@ -79,252 +72,115 @@ public class MonitoringService {
StatisticsService statisticsService
) {
this.meterRegistry = meterRegistry;
-
this.userService = userService;
this.searchService = searchService;
this.likesService = likesService;
this.statisticsService = statisticsService;
+ }
- this.numberExercises = new HashMap<>();
- this.numberLikesMap = new HashMap<>();
- this.numberViewsMap = new HashMap<>();
- this.numberDownloadsMap = new HashMap<>();
-
- this.numberUsers =
- meterRegistry.gauge(
- NUMBER_USER_NAME,
- Tags.of(HOURLY_TAG, Tag.of(USER_TAG, "All")),
- new AtomicLong(this.userService.countAllUsers())
- );
- this.numberActiveUsers =
- meterRegistry.gauge(
- NUMBER_USER_NAME,
- Tags.of(HOURLY_TAG, Tag.of(USER_TAG, "Active")),
- new AtomicLong(this.userService.countAllActiveUsers())
- );
+ @PostConstruct
+ public void initMonitoring() {
+ initializeGauges();
- try {
- final SearchResultsDTO searchResults = this.getAllSearchResults();
- final List<String> exerciseIDs = this.getExerciseIDs(searchResults.getSearchResult());
+ monitorHourlyMetrics();
+ }
- this.setNumberExercises(exerciseIDs.size());
- this.setExercisesLanguageCount(searchResults);
- this.setExerciseStatistics(exerciseIDs);
- } catch (IOException | ElasticsearchException exception) {
- log.warn("An IOException occurred when searching for all exercises\n" + Arrays.toString(exception.getStackTrace()));
- }
- this.numberLikes =
- meterRegistry.gauge(
- NUMBER_LIKE_NAME,
- Tags.of(HOURLY_TAG),
- new AtomicLong(this.numberLikesMap.values().stream().mapToLong(AtomicLong::get).sum())
- );
- this.numberViews =
- meterRegistry.gauge(
- NUMBER_VIEW_NAME,
- Tags.of(HOURLY_TAG),
- new AtomicLong(this.numberViewsMap.values().stream().mapToLong(AtomicLong::get).sum())
- );
- this.numberDownloads =
- meterRegistry.gauge(
- NUMBER_DOWNLOAD_NAME,
- Tags.of(HOURLY_TAG),
- new AtomicLong(this.numberDownloadsMap.values().stream().mapToLong(AtomicLong::get).sum())
- );
+ private void initializeGauges() {
+ meterRegistry.gauge(NUMBER_USER_NAME, Tags.of(HOURLY_TAG, Tag.of(USER_TAG, "All")), allUsers);
+ meterRegistry.gauge(NUMBER_USER_NAME, Tags.of(HOURLY_TAG, Tag.of(USER_TAG, "Active")), activeUsers);
+ meterRegistry.gauge(NUMBER_LIKE_NAME, Tags.of(HOURLY_TAG), allLikes);
+ meterRegistry.gauge(NUMBER_VIEW_NAME, Tags.of(HOURLY_TAG), allViews);
+ meterRegistry.gauge(NUMBER_DOWNLOAD_NAME, Tags.of(HOURLY_TAG), allDownloads);
+ meterRegistry.gauge(NUMBER_EXERCISE_NAME, Tags.of(HOURLY_TAG, Tag.of(LANGUAGE_TAG, "All")), allExercises);
}
/**
* Monitors metrics with an hourly cron job
*/
@Scheduled(cron = "0 30 * * * ?")
- public void monitorHourlyMetrics() {
+ public final void monitorHourlyMetrics() {
log.debug("Started monitoring cron job");
- this.setNumberUsers();
-
- try {
- SearchResultsDTO searchResults = this.getAllSearchResults();
- List<String> exerciseIDs = this.getExerciseIDs(searchResults.getSearchResult());
-
- this.setNumberExercises(exerciseIDs.size());
- this.setExercisesLanguageCount(searchResults);
- this.setExerciseStatistics(exerciseIDs);
- } catch (IOException exception) {
- log.warn("An IOException occurred\n" + Arrays.toString(exception.getStackTrace()));
- }
- this.setExerciseStatistics();
-
- log.debug("Finished monitoring cron job");
- }
-
- private void setNumberUsers() {
- log.debug("set the number of users");
-
- this.numberUsers.set(this.userService.countAllUsers());
- this.numberActiveUsers.set(this.userService.countAllActiveUsers());
- }
-
- /**
- * Sets likes, views and downloads for each given exerciseID
- *
- * @param exerciseIDs given exercise IDs
- */
- private void setExerciseStatistics(List<String> exerciseIDs) {
- log.debug("set the exercise statistics for {}", exerciseIDs);
-
- exerciseIDs.forEach(exerciseID -> {
- this.setNumberLikes(exerciseID);
-
- final Optional<StatisticsDTO> statisticOptional = this.statisticsService.findOneByExerciseID(exerciseID);
- if (statisticOptional.isPresent()) {
- StatisticsDTO statistic = statisticOptional.get();
- this.setNumberViews(exerciseID, statistic.getViews());
- this.setNumberDownloads(exerciseID, statistic.getDownloads());
+ allUsers.set((int) this.userService.countAllUsers());
+ activeUsers.set((int) this.userService.countAllActiveUsers());
+ allLikes.set((int) likesService.countAll());
+ final List<Integer[]> numberOfDownloadsAndViews = statisticsService.findNumberOfDownloadsAndViews();
+ if (numberOfDownloadsAndViews != null && numberOfDownloadsAndViews.size() == 1 && numberOfDownloadsAndViews.get(0).length == 2) {
+ if (numberOfDownloadsAndViews.get(0)[1] != null) {
+ allViews.set(numberOfDownloadsAndViews.get(0)[1]);
}
- });
- }
- /**
- * Sets likes, views and downloads
- *
- */
- private void setExerciseStatistics() {
- log.debug("set the exercise statistics");
- this.numberLikes.set(this.numberLikesMap.values().stream().mapToLong(AtomicLong::get).sum());
- this.numberViews.set(this.numberViewsMap.values().stream().mapToLong(AtomicLong::get).sum());
- this.numberDownloads.set(this.numberDownloadsMap.values().stream().mapToLong(AtomicLong::get).sum());
- }
-
- /**
- * Sets the given value as number of all exercises
- *
- * @param value given value
- */
- private void setNumberExercises(int value) {
- if (this.numberExercises.containsKey("All")) {
- this.numberExercises.get("All").set(value);
- } else {
- this.numberExercises.put(
- "All",
- meterRegistry.gauge(NUMBER_EXERCISE_NAME, Tags.of(HOURLY_TAG, Tag.of(LANGUAGE_TAG, "All")), new AtomicInteger(value))
- );
- }
- }
-
- /**
- * Sets the per programming language exercise counters for given search results
- *
- * @param searchResults the search results
- */
- private void setExercisesLanguageCount(SearchResultsDTO searchResults) {
- log.debug("set exercise language count");
-
- Map<String, Integer> languageCount = this.getLanguageCount(searchResults);
- for (String language : languageCount.keySet()) {
- if (this.numberExercises.containsKey(language)) {
- this.numberExercises.get(language).set(languageCount.get(language));
- } else {
- this.numberExercises.put(
- language,
- Metrics.gauge(
- NUMBER_EXERCISE_NAME,
- Tags.of(HOURLY_TAG, Tag.of(LANGUAGE_TAG, language)),
- new AtomicInteger(languageCount.get(language))
- )
- );
+ if (numberOfDownloadsAndViews.get(0)[0] != null) {
+ allDownloads.set(numberOfDownloadsAndViews.get(0)[0]);
}
}
- }
- /**
- * Gets a map containing the programming language counts for given search results
- *
- * @param searchResults the search results
- * @return a map containing the language counts
- */
- private Map<String, Integer> getLanguageCount(SearchResultsDTO searchResults) {
- Map<String, Integer> languageCount = new ConcurrentHashMap<>();
- List<UserProvidedMetadataDTO> metaDataDTO = new ArrayList<>();
-
- List<SearchResultDTO> searchResultList = searchResults.getSearchResult();
+ AtomicInteger exerciseCounter = new AtomicInteger(0);
+ try {
+ Map<String, Integer> languageCount = new ConcurrentHashMap<>();
+ final Stream<SearchResultDTO> searchResults = this.getAllSearchResults();
- searchResultList.forEach(searchResult -> metaDataDTO.add(searchResult.getMetadata()));
+ searchResults.forEach(exercise -> {
+ exerciseCounter.addAndGet(1);
+ updateLanguageCount(exercise, languageCount);
+ });
- for (UserProvidedMetadataDTO metaData : metaDataDTO) {
- String[] languageMetaData = metaData.getProgrammingLanguage();
- if (languageMetaData == null) {
- languageCount.put("Unclassified", languageCount.getOrDefault("Unclassified", 0) + 1);
- } else {
- Arrays
- .stream(metaData.getProgrammingLanguage())
- .forEach(language -> languageCount.put(language, languageCount.getOrDefault(language, 0) + 1));
- }
+ this.setExercisesLanguageCount(languageCount);
+ } catch (IOException | ElasticsearchException exception) {
+ log.warn("An IOException occurred\n{}", Arrays.toString(exception.getStackTrace()));
+ return;
+ } catch (org.elasticsearch.ElasticsearchException exception) {
+ // May become deprecated soon
+ log.warn("An org.elasticsearch.ElasticsearchException occurred\n{}", Arrays.toString(exception.getStackTrace()));
+ return;
}
- return languageCount;
- }
-
- /**
- * Sets the number of likes for a given exercise ID
- *
- * @param exerciseID given exercise ID
- */
- private void setNumberLikes(String exerciseID) {
- long numberLikes = this.likesService.findNumberOfLikesByExerciseID(exerciseID);
+ allExercises.set(exerciseCounter.get());
- if (this.numberLikesMap.containsKey(exerciseID)) {
- this.numberLikesMap.get(exerciseID).set(numberLikes);
- } else {
- this.numberLikesMap.put(
- exerciseID,
- //meterRegistry.gauge(
- // NUMBER_LIKE_NAME,
- // Tags.of(HOURLY_TAG, Tag.of(EXERCISE_TAG, exerciseID)),
- new AtomicLong(numberLikes)
- //)
- );
- }
+ log.debug("Finished monitoring cron job");
}
- /**
- * Sets the given number of views for a given exercise ID
- *
- * @param exerciseID given exercise ID
- * @param views given number of views
- */
- private void setNumberViews(String exerciseID, long views) {
- if (this.numberViewsMap.containsKey(exerciseID)) {
- this.numberViewsMap.get(exerciseID).set(views);
+ private void updateLanguageCount(SearchResultDTO exercise, Map<String, Integer> languageCount) {
+ var metaData = exercise.getMetadata();
+ String[] languageMetaData = metaData.getProgrammingLanguage();
+ if (languageMetaData == null) {
+ languageCount.put(UNCLASSIFIED, languageCount.getOrDefault(UNCLASSIFIED, 0) + 1);
} else {
- this.numberViewsMap.put(
- exerciseID,
- //meterRegistry.gauge(
- // NUMBER_VIEW_NAME,
- // Tags.of(HOURLY_TAG, Tag.of(EXERCISE_TAG, exerciseID)),
- new AtomicLong(views)
- //)
- );
+ Arrays
+ .stream(metaData.getProgrammingLanguage())
+ .forEach(language -> languageCount.put(language, languageCount.getOrDefault(language, 0) + 1));
}
}
/**
- * Sets the given number of downloads for a given exercise ID
+ * Sets the per programming language exercise counters for given search results
*
- * @param exerciseID given exercise ID
- * @param downloads given number of downloads
+ * @param searchResults the search results
*/
- private void setNumberDownloads(String exerciseID, long downloads) {
- if (this.numberDownloadsMap.containsKey(exerciseID)) {
- this.numberDownloadsMap.get(exerciseID).set(downloads);
- } else {
- this.numberDownloadsMap.put(
- exerciseID,
- //meterRegistry.gauge(
- // NUMBER_DOWNLOAD_NAME,
- // Tags.of(HOURLY_TAG, Tag.of(EXERCISE_TAG, exerciseID)),
- new AtomicLong(downloads)
- //)
- );
+ private void setExercisesLanguageCount(Map<String, Integer> languageCount) {
+ synchronized (this) {
+ languageCount
+ .entrySet()
+ .stream()
+ .forEach(entry -> {
+ final AtomicInteger existingEntry = allLanguageCount.get(entry.getKey());
+ if (existingEntry == null) {
+ final AtomicInteger newEntry = new AtomicInteger(entry.getValue());
+ Metrics.gauge(NUMBER_EXERCISE_NAME, Tags.of(HOURLY_TAG, Tag.of(LANGUAGE_TAG, entry.getKey())), newEntry);
+ allLanguageCount.put(entry.getKey(), newEntry);
+ } else {
+ existingEntry.set(entry.getValue());
+ }
+ });
+ // reset outdated languages to zero
+ allLanguageCount
+ .entrySet()
+ .stream()
+ .forEach(entry -> {
+ if (!languageCount.containsKey(entry.getKey())) {
+ entry.getValue().set(0);
+ }
+ });
}
}
@@ -334,22 +190,7 @@ public class MonitoringService {
* @return the search results DTO
* @throws IOException exception for timeout etc.
*/
- private SearchResultsDTO getAllSearchResults() throws IOException {
- return this.searchService.searchAllPublicResults();
- }
-
- /**
- * Gets the exercise ids from a search result list
- *
- * @param searchResultList the search result list
- * @return a list of exerciseIDs
- */
- private List<String> getExerciseIDs(List<SearchResultDTO> searchResultList) {
- log.debug("Get the exercise IDs from a list of search results");
-
- List<String> exerciseIDs = new ArrayList<>();
-
- searchResultList.forEach(searchResult -> exerciseIDs.add(searchResult.getExerciseId()));
- return exerciseIDs;
+ private Stream<SearchResultDTO> getAllSearchResults() throws IOException {
+ return this.searchService.getAllResultsAsStream();
}
}
diff --git a/src/main/java/at/ac/uibk/gitsearch/service/PluginManagementService.java b/src/main/java/at/ac/uibk/gitsearch/service/PluginManagementService.java
index 7075b76db6c7331bc14f5d5faf7a81bc7f7a3241..d247e081c630038502e6e0ab8d1b351e58ba0669 100644
--- a/src/main/java/at/ac/uibk/gitsearch/service/PluginManagementService.java
+++ b/src/main/java/at/ac/uibk/gitsearch/service/PluginManagementService.java
@@ -97,7 +97,7 @@ public class PluginManagementService {
log.warn("Cannot connect to connector at {}: {}", registeredConnector.getUrl(), ce.getMessage());
configFailures.put(registeredConnector, ce);
} catch (Exception e) {
- log.warn("Cannot (re-)load connector at {}", registeredConnector, e.getMessage());
+ log.warn("Cannot (re-)load connector at {}. Caused by {}.", registeredConnector, e.getMessage());
configFailures.put(registeredConnector, e);
final Iterator<Entry<String, ConnectorConfigWrapper>> pluginIterator = registeredPluginConfigs.entrySet().iterator();
while (pluginIterator.hasNext()) {
diff --git a/src/main/java/at/ac/uibk/gitsearch/service/ReviewService.java b/src/main/java/at/ac/uibk/gitsearch/service/ReviewService.java
index 0e5f124646b33cede77fd83866bf3cde3e7188c6..8388cb06d33180ee5539cc848a553c21bed0e537 100644
--- a/src/main/java/at/ac/uibk/gitsearch/service/ReviewService.java
+++ b/src/main/java/at/ac/uibk/gitsearch/service/ReviewService.java
@@ -235,7 +235,7 @@ public class ReviewService {
mailService.sendEmail(
userService.getUserById(review.getRequestedBy()).get().getEmail(),
"New feedback for resource",
- "A reviewer has just updated you review request for resource: " + review.getResource(),
+ "A reviewer has just updated your review request for resource: " + review.getResource(),
false,
false
);
@@ -250,6 +250,8 @@ public class ReviewService {
statisticsService.save(statistics);
}
notifyReviewers(review.getId(), "The review for " + review.getResource() + " has been deleted.");
+ notifyRequester(review.getId(), "The review for " + review.getResource() + " has been deleted.");
+
reviewRatingRepository.deleteByReviewId(id);
reviewRepository.deleteById(id);
}
@@ -279,6 +281,10 @@ public class ReviewService {
review.getId(),
"The review for " + review.getResource() + " has been completed and a badge is now being displayed when people search for it."
);
+ notifyRequester(
+ review.getId(),
+ "The review for " + review.getResource() + " has been completed and a badge is now being displayed when people search for it."
+ );
}
public void requestReviewForExercise(ReviewRequest request) {
@@ -323,4 +329,11 @@ public class ReviewService {
.collect(Collectors.toList());
emails.stream().forEach(email -> mailService.sendEmail(email, "Review Updated", message, false, false));
}
+
+ // notify requester of review
+ public void notifyRequester(Long id, String message) {
+ Review review = reviewRepository.findById(id).get();
+ User user = userService.getUserById(review.getRequestedBy()).get();
+ mailService.sendEmail(user.getEmail(), "Review Updated", message, false, false);
+ }
}
diff --git a/src/main/java/at/ac/uibk/gitsearch/service/SearchService.java b/src/main/java/at/ac/uibk/gitsearch/service/SearchService.java
index 4a66a7157e7c21f7e8a5e4c052c56e2685d8d1f6..115a2adddf6be1ad5ff40c427aa28e05f328c3f0 100644
--- a/src/main/java/at/ac/uibk/gitsearch/service/SearchService.java
+++ b/src/main/java/at/ac/uibk/gitsearch/service/SearchService.java
@@ -12,6 +12,7 @@ import java.net.URISyntaxException;
import java.text.ParseException;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -75,7 +76,7 @@ public class SearchService {
final Optional<User> principal = tokenProvider.getCurrentPrincipal();
log.debug("Searchrequest for {} ", searchInput);
- final SearchResultsDTO pageDetails = metaDataRepository.pageDetails(searchInput, principal);
+ final SearchResultsDTO pageDetails = metaDataRepository.pageDetailsWithJavaApi(searchInput, principal);
pageDetails
.getSearchResult()
@@ -134,7 +135,7 @@ public class SearchService {
}
// Returns all resources indexed by elastic search
- public SearchResultsDTO getAllResources() throws IOException {
+ public List<SearchResultDTO> getAllResources() throws IOException {
return metaDataRepository.getAllResources();
}
@@ -254,8 +255,9 @@ public class SearchService {
if (tokenProvider.getCurrentPrincipal().isEmpty()) {
log.warn("Cannot find a principal for for exercise {}", exerciseId);
return false;
- }
- if (tokenProvider.getCurrentPrincipal().get().getAuthorities().contains(AuthoritiesConstantEnum.ADMIN.getGrantedAuthority())) {
+ } else if (
+ tokenProvider.getCurrentPrincipal().get().getAuthorities().contains(AuthoritiesConstantEnum.ADMIN.getGrantedAuthority())
+ ) {
log.info("Admin can access everything : {}", exerciseId);
return true; // ADMIN is always allowed
}
@@ -320,19 +322,50 @@ public class SearchService {
return result;
}
+ /**
+ * Method to stream all exercises.
+ *
+ * @return the search result page DTO
+ * @throws IOException
+ */
+ public Stream<SearchResultDTO> getAllResultsAsStream() throws IOException {
+ return metaDataRepository.getAllResultsAsStream();
+ }
+
+ /**
+ * Method to search for all public exercises.
+ * Searches all public exercises with an empty search query and without a
+ * specified user.
+ *
+ * @return the search result page DTO
+ * @throws IOException
+ */
+ public Stream<SearchResultDTO> getAllPublicResultsAsStream() throws IOException {
+ log.debug("Search request to find all public exercises");
+
+ final SearchInputMetadataDTO emptySearchMetadata = new SearchInputMetadataDTO(null, null, null, null, null, null);
+ SearchInputDTO emptySearchQuery = new SearchInputDTO(null, emptySearchMetadata, null, null, null, 0);
+
+ return metaDataRepository.streamResults(emptySearchQuery, Optional.empty(), false).getSearchStream();
+ }
+
/**
* Method to search for all public exercises.
- * Searches all public exercises with an empty search query and without a specified user.
+ * Searches all public exercises with an empty search query and without a
+ * specified user.
*
* @return the search result page DTO
* @throws IOException
+ * @deprecated not really reliable, because it does only return the first page
+ * of results.
*/
+ @Deprecated
public SearchResultsDTO searchAllPublicResults() throws IOException {
log.debug("Search request to find all public exercises");
final SearchInputMetadataDTO emptySearchMetadata = new SearchInputMetadataDTO(null, null, null, null, null, null);
SearchInputDTO emptySearchQuery = new SearchInputDTO(null, emptySearchMetadata, null, null, null, 0);
- return metaDataRepository.pageDetails(emptySearchQuery, Optional.empty());
+ return metaDataRepository.pageDetailsWithJavaApi(emptySearchQuery, Optional.empty());
}
}
diff --git a/src/main/java/at/ac/uibk/gitsearch/service/StatisticsService.java b/src/main/java/at/ac/uibk/gitsearch/service/StatisticsService.java
index 0e77c3cab7a9c8dc79fd025d5d0a8812f4e68659..000322ceb8e1c0e3d52bee80fbe28706bd6c0dbc 100644
--- a/src/main/java/at/ac/uibk/gitsearch/service/StatisticsService.java
+++ b/src/main/java/at/ac/uibk/gitsearch/service/StatisticsService.java
@@ -4,6 +4,7 @@ import at.ac.uibk.gitsearch.domain.Statistics;
import at.ac.uibk.gitsearch.repository.jpa.StatisticsRepository;
import at.ac.uibk.gitsearch.repository.search.StatisticsSearchRepository;
import at.ac.uibk.gitsearch.service.dto.StatisticsDTO;
+import java.util.List;
import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,4 +92,13 @@ public class StatisticsService {
statisticsRepository.deleteById(id);
statisticsSearchRepository.deleteById(id);
}
+
+ /**
+ * returns the sum of all downloads and views of all entries.
+ *
+ * @return the two values of downlodas and views (in this sequence)
+ */
+ List<Integer[]> findNumberOfDownloadsAndViews() {
+ return statisticsRepository.findNumberOfDownloadsAndViews();
+ }
}
diff --git a/src/main/java/at/ac/uibk/gitsearch/service/UserService.java b/src/main/java/at/ac/uibk/gitsearch/service/UserService.java
index 8fdc12a076e03d661c9ac74607b97fcd8d828eb9..ad833aa1f2d3d57823407c42c5834c63d992ce49 100644
--- a/src/main/java/at/ac/uibk/gitsearch/service/UserService.java
+++ b/src/main/java/at/ac/uibk/gitsearch/service/UserService.java
@@ -232,18 +232,10 @@ public class UserService {
user.setReviewingEnabled(userDTO.isReviewingEnabled());
user.setLangKey(userDTO.getLangKey());
final List<Authority> allAuthorities = authorityRepository.findAll();
- final Set<Authority> userAuthorities = userDTO
- .getAuthorities()
- .stream()
- .map(authString -> new Authority(authString))
- .collect(Collectors.toSet());
+
+ final Set<Authority> userAuthorities = userDTO.getAuthorities().stream().map(Authority::new).collect(Collectors.toSet());
// insert new authorities into managedAuthoritiesTable
- userAuthorities
- .stream()
- .filter(auth -> !allAuthorities.contains(auth))
- .forEach(auth -> {
- authorityRepository.save(auth);
- });
+ userAuthorities.stream().filter(auth -> !allAuthorities.contains(auth)).forEach(authorityRepository::save);
user.setAuthorities(userAuthorities);
userRepository.save(user);
// userSearchRepository.save(user);
diff --git a/src/main/java/at/ac/uibk/gitsearch/service/ZipRepackagingService.java b/src/main/java/at/ac/uibk/gitsearch/service/ZipRepackagingService.java
index c89fadaac2c50f35b9993049dc1975592154b2ce..991418ea24035ba86597ff4593f1dc6c6fa37bcc 100644
--- a/src/main/java/at/ac/uibk/gitsearch/service/ZipRepackagingService.java
+++ b/src/main/java/at/ac/uibk/gitsearch/service/ZipRepackagingService.java
@@ -9,7 +9,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
-import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
@@ -164,9 +163,9 @@ public class ZipRepackagingService {
getDirPath(exerciseRoot.get().getData().getFile()),
exerciseRoot.get()
);
- if (log.isWarnEnabled()) {
- log.warn("Useing Filters: {}", applicableFilters.stream().collect(Collectors.joining(", ")));
- }
+ // if (log.isWarnEnabled()) {
+ // log.warn("Useing Filters: {}", applicableFilters.stream().collect(Collectors.joining(", ")));
+ // }
for (String filter : applicableFilters) {
if (filter == null) {
log.warn("filter should never be null!");
diff --git a/src/main/java/at/ac/uibk/gitsearch/service/dto/UserDTO.java b/src/main/java/at/ac/uibk/gitsearch/service/dto/UserDTO.java
index 8ecf48a03c9df83af11fdb9c1cb73c4ed8ee3a90..4ccc302629db23b34a8ac082ddc46ce1297b0e39 100644
--- a/src/main/java/at/ac/uibk/gitsearch/service/dto/UserDTO.java
+++ b/src/main/java/at/ac/uibk/gitsearch/service/dto/UserDTO.java
@@ -3,16 +3,22 @@ package at.ac.uibk.gitsearch.service.dto;
import at.ac.uibk.gitsearch.config.Constants;
import at.ac.uibk.gitsearch.domain.Authority;
import at.ac.uibk.gitsearch.domain.User;
+import java.io.Serializable;
import java.time.Instant;
import java.util.Set;
import java.util.stream.Collectors;
-import javax.validation.constraints.*;
+import javax.validation.constraints.Email;
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.Pattern;
+import javax.validation.constraints.Size;
/**
* A DTO representing a user, with his authorities.
*/
@SuppressWarnings("PMD")
-public class UserDTO {
+public class UserDTO implements Serializable {
+
+ private static final long serialVersionUID = 653337653769367078L;
private Long id;
diff --git a/src/main/java/at/ac/uibk/gitsearch/web/rest/GitFilesResource.java b/src/main/java/at/ac/uibk/gitsearch/web/rest/GitFilesResource.java
deleted file mode 100644
index 6d6815a91bd5aaf4aa049949aad49309ab28a477..0000000000000000000000000000000000000000
--- a/src/main/java/at/ac/uibk/gitsearch/web/rest/GitFilesResource.java
+++ /dev/null
@@ -1,76 +0,0 @@
-package at.ac.uibk.gitsearch.web.rest;
-
-import at.ac.uibk.gitsearch.es.model.DocumentInfo;
-import at.ac.uibk.gitsearch.repository.search.GitFilesRepository;
-import at.ac.uibk.gitsearch.service.dto.GitFilesAggregationDTO;
-import java.io.IOException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.transaction.annotation.Transactional;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
-/**
- * REST controller for managing {@link DocumentInfo}.
- */
-@RestController
-@RequestMapping("/api")
-@Transactional
-public class GitFilesResource {
-
- private final Logger log = LoggerFactory.getLogger(GitFilesResource.class);
-
- @Value("${jhipster.clientApp.name}")
- @SuppressWarnings({ "unused", "PMD.ImmutableField" })
- private String applicationName;
-
- private final GitFilesRepository gitFilesRepository;
-
- public GitFilesResource(GitFilesRepository gitFilesRepository) {
- this.gitFilesRepository = gitFilesRepository;
- }
-
- /**
- * // * {@code SEARCH /_search/git-files/page-details?query=:query} : search for the searchResult corresponding
- * // * to the query.
- * // *
- * // * @param query the query of the searchResult search.
- * // * @return the result of the search.
- * //
- */
- /*
- @GetMapping("/_search/git-files/page-details")
- public GitFilesPageDetailsDTO
- gitFilesPageDetails(@RequestParam String fulltextQuery, @RequestParam String metadataProgrammingLanguage,
- @RequestParam String metadataKeywords, @RequestParam String metadataNaturalLanguage,
- @RequestParam String metadataLicense, @RequestParam String metadataAuthor,
- @RequestParam List<String> selectedRepository,
- @RequestParam List<String> selectedUniversity, @RequestParam List<String> selectedFileFormat,
- @RequestParam int page, @RequestParam int pageSize) throws IOException {
- SearchInputDTO searchInput = new SearchInputDTO(fulltextQuery, metadataProgrammingLanguage, metadataKeywords,
- metadataNaturalLanguage, metadataLicense, metadataAuthor, selectedRepository, selectedUniversity,
- selectedFileFormat, page);
- log.debug("REST request to search gitFilesPageDetails for searchInput {}, pageSize {}", searchInput, pageSize);
- // return searchResultSearchRepository.findByContent(query);
- return gitFilesRepository.pageDetails(searchInput, pageSize);
- }
- */
-
- /**
- * // * {@code SEARCH /_search/git-files/aggregation?query=:query} : search for the searchResult corresponding
- * // * to the query.
- * // *
- * // * @param query the query of the searchResult search.
- * // * @return the result of the search.
- * //
- */
- @GetMapping("/_search/git-files/aggregation")
- public GitFilesAggregationDTO gitFilesAggregation(@RequestParam String query) throws IOException {
- log.debug("REST request to search gitFilesAggregation for query {}", query);
- // return searchResultSearchRepository.findByContent(query);
- return gitFilesRepository.aggregation(query);
- }
-}
diff --git a/src/main/java/at/ac/uibk/gitsearch/web/rest/PluginInterfaceResource.java b/src/main/java/at/ac/uibk/gitsearch/web/rest/PluginInterfaceResource.java
index afb89b92c2db2364e8ed633bd65f7e46503fa221..329661f3fdbbe11403111c3f800f46ddee9bf924 100644
--- a/src/main/java/at/ac/uibk/gitsearch/web/rest/PluginInterfaceResource.java
+++ b/src/main/java/at/ac/uibk/gitsearch/web/rest/PluginInterfaceResource.java
@@ -101,7 +101,7 @@ public class PluginInterfaceResource {
* @return the result of the search.
*/
@GetMapping("/pluginIF/v0.1/basket/{basketToken}")
- public ShoppingBasket getBasket(@PathVariable("basketToken") String basketToken) throws IOException {
+ public ShoppingBasket getBasket(@PathVariable("basketToken") String basketToken) {
log.debug("REST request for basket {}", basketToken);
return basketService.getBasket(basketToken);
}
diff --git a/src/main/java/at/ac/uibk/gitsearch/web/rest/SearchResource.java b/src/main/java/at/ac/uibk/gitsearch/web/rest/SearchResource.java
index c01df07c86c731e219acbdf77dbf18ca2fcc4d70..1a249b40e77333d535351a272ad8718a4e7b73e4 100644
--- a/src/main/java/at/ac/uibk/gitsearch/web/rest/SearchResource.java
+++ b/src/main/java/at/ac/uibk/gitsearch/web/rest/SearchResource.java
@@ -16,6 +16,7 @@ import java.nio.file.Files;
import java.text.ParseException;
import java.util.List;
import java.util.Optional;
+import org.codeability.sharing.plugins.api.search.SearchResultDTO;
import org.codeability.sharing.plugins.api.search.SearchResultsDTO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -85,7 +86,7 @@ public class SearchResource {
}
@GetMapping("/search/all")
- public SearchResultsDTO searchAll() throws IOException {
+ public List<SearchResultDTO> searchAll() throws IOException {
log.debug("REST request to search for all resources");
return searchService.getAllResources();
}
diff --git a/src/main/resources/keycloak-theme/themes/deploy_theme.sh b/src/main/resources/keycloak-theme/themes/deploy_theme.sh
index 70baa455a5da9be85a572ed6a067e39fbb552a0f..837df84d15dffd717037fbcc06cffe08de771498 100755
--- a/src/main/resources/keycloak-theme/themes/deploy_theme.sh
+++ b/src/main/resources/keycloak-theme/themes/deploy_theme.sh
@@ -1,9 +1,11 @@
source ../../../docker/.env
export GITBRANCH=test
-# docker cp /home/contDeploy/gitsearch2/gitsearch/src/main/resources/keycloak-theme/themes/gitsearch/. docker_keycloak_1:opt/jboss/keycloak/themes/gitsearch
+# docker cp $GITSEARCH_PATH/src/main/resources/keycloak-theme/themes/gitsearch/. docker_keycloak_1:opt/jboss/keycloak/themes/gitsearch
cd $GITSEARCH_PATH/src/main/docker/
-docker-compose -f gitsearch.yml up -d --build --force-recreate gitsearch-app
-docker logs docker_keycloak_1 --follow
-# docker cp docker_keycloak_1:opt/jboss/keycloak/themes/. /home/contDeploy/gitsearch2/gitsearch/src/main/resources/keycloak-theme/themes
-# docker restart docker_keycloak_1
+#docker-compose -f gitsearch.yml up -d --build --force-recreate gitsearch-app
+# docker logs docker_keycloak_1 --follow
+
+docker cp docker_keycloak_1:opt/jboss/keycloak/themes/gitsearch/. $GITSEARCH_PATH/src/main/resources/keycloak-theme/themes/gitsearch
+docker restart docker_keycloak_1
+docker logs docker_keycloak_1 --follow
\ No newline at end of file
diff --git a/src/main/resources/keycloak-theme/themes/gitsearch/login/login.ftl b/src/main/resources/keycloak-theme/themes/gitsearch/login/login.ftl
index c6cb19335fd277c423454e47bd24a253f3b07a7d..eb2ec622bc6f1dba42db619b8426bf6915fe54d6 100644
--- a/src/main/resources/keycloak-theme/themes/gitsearch/login/login.ftl
+++ b/src/main/resources/keycloak-theme/themes/gitsearch/login/login.ftl
@@ -97,14 +97,6 @@
</#if>
</a>
</#list>
- <a id="social-admin" class="pf-c-button pf-m-control pf-m-block kc-social-item kc-social-gray " type="button" href="https://search.sharing-codeability.uibk.ac.at/login">
- <i class="kc-social-provider-logo kc-social-gray fa fa-lock" aria-hidden="true"></i>
- <span class="kc-social-provider-name kc-social-icon-text">Admin Login Sharing-Prod</span>
- </a>
- <a id="social-admin" class="pf-c-button pf-m-control pf-m-block kc-social-item kc-social-gray " type="button" href="https://dev-exchange.codeability-austria.uibk.ac.at/login">
- <i class="kc-social-provider-logo kc-social-gray fa fa-lock" aria-hidden="true"></i>
- <span class="kc-social-provider-name kc-social-icon-text">Admin Login Sharing-Dev</span>
- </a>
</ul>
</div>
</#if>
diff --git a/src/main/resources/keycloak-theme/themes/gitsearch/login/template.ftl b/src/main/resources/keycloak-theme/themes/gitsearch/login/template.ftl
index 91a5ab0a5a2c97979c2274e223470686b5a9579f..536f06d50a23be154d364b09fcc28c74921b481b 100644
--- a/src/main/resources/keycloak-theme/themes/gitsearch/login/template.ftl
+++ b/src/main/resources/keycloak-theme/themes/gitsearch/login/template.ftl
@@ -39,7 +39,7 @@
<body class="${properties.kcBodyClass!}">
<div class="${properties.kcLoginClass!}">
<div id="kc-header" class="${properties.kcHeaderClass!}">
- <div id="kc-header-wrapper"> <img _ngcontent-juv-c133="" src="/auth/resources/xs85t/login/gitsearch/img/logo-top.png" title="codeAbility Home" alt="codeAbility" width="150px"> </div>
+ <div id="kc-header-wrapper"> <img _ngcontent-juv-c133="" src="${url.resourcesPath}/img/logo-top.png" title="codeAbility Home" alt="codeAbility" width="150px"> </div>
</div>
<div class="${properties.kcFormCardClass!}">
<header class="${properties.kcFormHeaderClass!}">
diff --git a/src/main/webapp/app/admin/review-management/review-management-update/review-management-update.component.ts b/src/main/webapp/app/admin/review-management/review-management-update/review-management-update.component.ts
index cbd1cd1efc99e1c64e2b3559a6bffd0385068320..1304bc7ef7c6de17a88f2e45e81a16c2f1b0f94a 100644
--- a/src/main/webapp/app/admin/review-management/review-management-update/review-management-update.component.ts
+++ b/src/main/webapp/app/admin/review-management/review-management-update/review-management-update.component.ts
@@ -6,6 +6,7 @@ import { Account } from 'app/core/auth/account.model';
import { AccountService } from 'app/core/auth/account.service';
import { SearchService } from 'app/search/service/search-service';
import { Exercise, searchResultToExercise } from 'app/shared/model/exercise.model';
+import { SearchResultDTO } from 'app/shared/model/search/search-result-dto.model';
import { SearchResultsDTO } from 'app/shared/model/search/search-results-dto.model';
import { ReviewManagementService } from '../review-management.service';
import { Review } from '../review.model';
@@ -55,8 +56,8 @@ export class ReviewManagementUpdateComponent implements OnInit {
this.users = users;
});
this.searchService.getAllResources().subscribe(
- (data: SearchResultsDTO) => {
- const searchResults = data.searchResult.map(searchResultToExercise);
+ (data: SearchResultDTO[]) => {
+ const searchResults = data.map(searchResultToExercise);
this.results = this.results
.concat(searchResults)
.filter((value, index, self) => self.findIndex(t => t.title === value.title) === index);
diff --git a/src/main/webapp/app/admin/review-management/review-management.component.ts b/src/main/webapp/app/admin/review-management/review-management.component.ts
index 60f820a4c6067e765325fa57fceda4084b68d54b..cf2c3aa1d144f25bbd2b074e8b0b208ed253ba9c 100644
--- a/src/main/webapp/app/admin/review-management/review-management.component.ts
+++ b/src/main/webapp/app/admin/review-management/review-management.component.ts
@@ -4,6 +4,7 @@ import { Account } from 'app/core/auth/account.model';
import { AccountService } from 'app/core/auth/account.service';
import { SearchService } from 'app/search/service/search-service';
import { Exercise, searchResultToExercise } from 'app/shared/model/exercise.model';
+import { SearchResultDTO } from 'app/shared/model/search/search-result-dto.model';
import { SearchResultsDTO } from 'app/shared/model/search/search-results-dto.model';
import { UserManagementService } from '../user-management/service/user-management.service';
import { User } from '../user-management/user-management.model';
@@ -51,8 +52,8 @@ export class ReviewManagementComponent implements OnInit, OnChanges {
this.users = users;
});
this.searchService.getAllResources().subscribe(
- (data: SearchResultsDTO) => {
- const searchResults = data.searchResult.map(searchResultToExercise);
+ (data: SearchResultDTO[]) => {
+ const searchResults = data.map(searchResultToExercise);
this.results = this.results
.concat(searchResults)
.filter((value, index, self) => self.findIndex(t => t.title === value.title) === index);
diff --git a/src/main/webapp/app/search/service/search-service.ts b/src/main/webapp/app/search/service/search-service.ts
index df5ef139cbbcd8030577c7324542c44e074d5fd2..146845a5103e1dba9159a447b023d3981f23ed31 100644
--- a/src/main/webapp/app/search/service/search-service.ts
+++ b/src/main/webapp/app/search/service/search-service.ts
@@ -9,6 +9,7 @@ import { SearchInput } from 'app/shared/model/search/search-input.model';
import { Statistics } from 'app/shared/model/statistics.model';
import { encodeURIforExerciseId } from 'app/exercise/service/exercise.service';
import { ReviewRequest } from 'app/admin/review-management/reviewRequest.model';
+import { SearchResultDTO } from 'app/shared/model/search/search-result-dto.model';
@Injectable({ providedIn: 'root' })
export class SearchService {
@@ -48,8 +49,8 @@ export class SearchService {
return this.http.get<SearchResultsDTO>(this.resourseSearchUrlAuthor);
}
- getAllResources(): Observable<SearchResultsDTO> {
- return this.http.get<SearchResultsDTO>('api/search/all');
+ getAllResources(): Observable<SearchResultDTO[]> {
+ return this.http.get<SearchResultDTO[]>('api/search/all');
}
requestReview(exercise: ReviewRequest): Observable<ReviewRequest> {
diff --git a/src/test/java/at/ac/uibk/gitsearch/repository/search/GitFilesRepositoryImplIT.java b/src/test/java/at/ac/uibk/gitsearch/repository/search/GitFilesRepositoryImplIT.java
deleted file mode 100644
index bc78635d8672660f2c44cbd4726e531001f4deb7..0000000000000000000000000000000000000000
--- a/src/test/java/at/ac/uibk/gitsearch/repository/search/GitFilesRepositoryImplIT.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package at.ac.uibk.gitsearch.repository.search;
-
-import static org.junit.jupiter.api.Assertions.fail;
-
-import at.ac.uibk.gitsearch.GitsearchApp;
-import at.ac.uibk.gitsearch.service.dto.GitFilesPageDetailsDTO;
-import java.io.IOException;
-import org.codeability.sharing.plugins.api.search.SearchInputDTO;
-import org.codeability.sharing.plugins.api.search.SearchInputMetadataDTO;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-
-/**
- * is test fixture is currently empty. It seems very hard, to provide trivial tests.
- * TODO: Either remove GitFilesRepository infrastructure, or provide good tests.
- * @author Michael Breu
- *
- */
-@SpringBootTest(classes = GitsearchApp.class)
-class GitFilesRepositoryImplIT {
-
- @Autowired
- private GitFilesRepository gitFilesRepository;
-
- // @Test
- void testPageDetails() throws IOException {
- final SearchInputMetadataDTO searchMetadata = new SearchInputMetadataDTO(null, "testing", null, null, null, null);
- SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null, null, null, 0);
- @SuppressWarnings("unused")
- GitFilesPageDetailsDTO searchResultPage = gitFilesRepository.pageDetails(searchQuery);
- }
-
- // @Test
- void testAggregation() {
- fail("Not yet implemented");
- }
-}
diff --git a/src/test/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepositoryIT.java b/src/test/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepositoryIT.java
index 3bb226080aa37601c327aa7cc6ca49adb50fb557..7b618f6b05ad2b308c0bd26fee97f38457bcd33a 100644
--- a/src/test/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepositoryIT.java
+++ b/src/test/java/at/ac/uibk/gitsearch/repository/search/MetaDataRepositoryIT.java
@@ -5,6 +5,7 @@ import static org.hamcrest.Matchers.hasProperty;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import at.ac.uibk.gitsearch.GitsearchApp;
import at.ac.uibk.gitsearch.domain.util.TreeNode;
@@ -12,6 +13,8 @@ import at.ac.uibk.gitsearch.repository.search.testESService.ElasticSearchTestCon
import at.ac.uibk.gitsearch.service.dto.AutoCompleteEntry;
import java.io.IOException;
import java.text.ParseException;
+import java.time.Instant;
+import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
@@ -19,6 +22,7 @@ import javax.ws.rs.NotFoundException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codeability.sharing.plugins.api.search.SearchResultDTO;
+import org.codeability.sharing.plugins.api.search.SearchResultsDTO;
import org.codeability.sharing.plugins.api.search.util.ExerciseId;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.node.NodeValidationException;
@@ -35,7 +39,6 @@ import org.springframework.boot.test.context.SpringBootTest;
@SpringBootTest(classes = GitsearchApp.class)
public class MetaDataRepositoryIT {
- @SuppressWarnings("unused")
private static final Logger LOGGER = LogManager.getLogger(MetaDataRepositoryIT.class);
@Autowired
@@ -107,6 +110,78 @@ public class MetaDataRepositoryIT {
}
}
+ @Test
+ @Timeout(value = 5, unit = TimeUnit.MINUTES)
+ public void getByExerciseIdsSince() throws IOException {
+ // This test assumes that the test entries are public
+ for (int i : new int[] { 1 }) {
+ final String exerciseId = "[" + i + "]";
+ final Instant farAgo = Instant.ofEpochMilli(0L);
+ final SearchResultsDTO exerciseByIds = metaDataRepository.getExercisesById(
+ Collections.singleton(exerciseId).stream(),
+ Optional.empty(),
+ 0,
+ 100,
+ farAgo
+ );
+ assertNotNull("should be found:", exerciseByIds);
+ assertTrue(exerciseByIds.getHitCount() > 0); // at least one hit
+
+ exerciseByIds
+ .getSearchResult()
+ .forEach(exerciseById -> assertEquals("Id should be the same", exerciseId, exerciseById.getExerciseId()));
+ }
+ }
+
+ @Test
+ @Timeout(value = 5, unit = TimeUnit.MINUTES)
+ public void getByExerciseIds() throws IOException {
+ // This test assumes that the test entries are public
+ for (int i : new int[] { 1 }) {
+ final String exerciseId = "[" + i + "]";
+ final SearchResultsDTO exerciseByIds = metaDataRepository.getExercisesById(
+ Collections.singleton(exerciseId).stream(),
+ Optional.empty(),
+ 0,
+ 100
+ );
+ assertNotNull("should be found:", exerciseByIds);
+ assertTrue(exerciseByIds.getHitCount() > 0); // at least one hit
+
+ exerciseByIds
+ .getSearchResult()
+ .forEach(exerciseById -> assertEquals("Id should be the same", exerciseId, exerciseById.getExerciseId()));
+ }
+ }
+
+ @Test
+ @Timeout(value = 5, unit = TimeUnit.MINUTES)
+ public void getByExerciseIdsForFuture() throws IOException {
+ // This test assumes that the test entries are public
+ for (int i : new int[] { 1 }) {
+ final String exerciseId = "[" + i + "]";
+ final Instant now = Instant.now();
+ final SearchResultsDTO exerciseByIds = metaDataRepository.getExercisesById(
+ Collections.singleton(exerciseId).stream(),
+ Optional.empty(),
+ 0,
+ 100,
+ now
+ );
+ assertNotNull("should be found:", exerciseByIds);
+ assertEquals(0, exerciseByIds.getHitCount()); // no hit newer than now
+ }
+ }
+
+ @Test
+ public void getByEmail() throws IOException {
+ // This test assumes that the test entries are public
+
+ final SearchResultsDTO exercisesByEmail = metaDataRepository.searchByEmail("c703257@uibk.ac.at", 100, Optional.empty());
+
+ assertEquals("This may change if testdata changes", 3, exercisesByEmail.getHitCount());
+ }
+
@Test
public void getByExerciseIdNegative() throws IOException {
// This test assumes that the test entries have consecutive ids
@@ -130,14 +205,12 @@ public class MetaDataRepositoryIT {
Assert.assertNotNull(exerciseCollectionTree);
- Assert.assertEquals(2, exerciseCollectionTree.getChildren().size()); // this
- // may
- // change
- // when
- // test
- // data
- // changes
- Assert.assertEquals(2, exerciseCollectionTree.getChildren().get(0).getChildren().size()); // this may change when test data
- // changes
+ Assert.assertEquals("This test may fail if test data changes (1)", 2, exerciseCollectionTree.getChildren().size());
+ // either first or second child contains sub-children
+ Assert.assertTrue(
+ "This test may fail if test data changes (2)",
+ exerciseCollectionTree.getChildren().get(0).getChildren().size() == 2 ||
+ exerciseCollectionTree.getChildren().get(1).getChildren().size() == 2
+ );
}
}
diff --git a/src/test/java/at/ac/uibk/gitsearch/repository/search/testESService/ElasticSearchTestConfiguration.java b/src/test/java/at/ac/uibk/gitsearch/repository/search/testESService/ElasticSearchTestConfiguration.java
index e90b380659a0754149943e5ea1c84ef0a878d5de..ac84ff92d5d74c0fee821d47c24560b034f47014 100644
--- a/src/test/java/at/ac/uibk/gitsearch/repository/search/testESService/ElasticSearchTestConfiguration.java
+++ b/src/test/java/at/ac/uibk/gitsearch/repository/search/testESService/ElasticSearchTestConfiguration.java
@@ -3,6 +3,8 @@ package at.ac.uibk.gitsearch.repository.search.testESService;
import at.ac.uibk.gitsearch.domain.util.TreeNode;
import at.ac.uibk.gitsearch.repository.search.MetaDataRepository;
import at.ac.uibk.gitsearch.repository.search.SearchRepositoryConstants;
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.transport.rest_client.RestClientTransport;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -34,6 +36,7 @@ import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.function.Consumer;
@@ -91,29 +94,48 @@ import org.testcontainers.utility.TestcontainersConfiguration;
@Service
public class ElasticSearchTestConfiguration {
- static final DockerImageName ELASTICSEARCH_IMAGE = DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch:7.17.2");
+ static final DockerImageName ELASTICSEARCH_IMAGE = DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch:7.17.6");
private ElasticsearchContainer esContainer;
@Autowired
- private RestHighLevelClient elasticsearchClient;
+ private RestHighLevelClient elasticsearchHLClient;
+
+ @Autowired
+ private ElasticsearchClient elasticsearchAPIClient;
+
+ @Autowired
+ MetaDataRepository metaDataRepository;
@Value("${spring.elasticsearch.uris}")
private String elasticSearchTestURI;
public void stopTestContainer() {
if (esContainer != null && esContainer.isCreated() && esContainer.isRunning()) {
+ SearchRequest searchRequest = new SearchRequest(SearchRepositoryConstants.INDEX_METADATA);
+ BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+ SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+ sourceBuilder.query(queryBuilder);
+
+ try {
+ SearchResponse searchResponse = elasticsearchHLClient.search(searchRequest, RequestOptions.DEFAULT);
+ LOGGER.info("Test ES still containing {} entries before shut down", searchResponse.getHits().getTotalHits());
+ } catch (IOException e1) {
+ LOGGER.info("Last Search not working", e1);
+ }
+
esContainer.stop();
esContainer.close();
while (esContainer.isRunning()) {
try {
Thread.sleep(1000);
- LOGGER.info("Shutting down test container...");
+ LOGGER.info("Shutting down esContainer ...");
} catch (InterruptedException e) {
e.printStackTrace();
}
}
esContainer = null;
+ LOGGER.warn("Shutted down ES Test Node");
}
}
@@ -121,7 +143,7 @@ public class ElasticSearchTestConfiguration {
private int getElasticSearchPort() {
try {
- URI es = URI.create(elasticSearchTestURI);
+ URI es = URI.create(elasticSearchTestURI.split(",")[0]);
return es.getPort();
} catch (IllegalArgumentException | NullPointerException e) {
LOGGER.warn("Cannot parse " + elasticSearchTestURI + " to get the port");
@@ -330,6 +352,7 @@ public class ElasticSearchTestConfiguration {
setUpMetaDataIndex(hostURL);
setUpContent(hostURL);
+ metaDataRepository.updateAutocompletionCache();
LOGGER.info("Started ES Test Node");
} catch (Throwable t) {
LOGGER.info("ES Test Node crashed: ", t);
@@ -337,8 +360,20 @@ public class ElasticSearchTestConfiguration {
}
}
+ /**
+ * redirects the clients to the docker instance.
+ * Is there are more elegant way to mock es node urls?
+ *
+ * @param hostURL
+ * @throws IOException
+ */
private void setUpESClientForTestESServer(HttpHost hostURL) throws IOException {
- elasticsearchClient.getLowLevelClient().setNodes(RestClient.builder(hostURL).build().getNodes());
+ elasticsearchHLClient.getLowLevelClient().setNodes(RestClient.builder(hostURL).build().getNodes());
+
+ org.elasticsearch.client.Node n = new org.elasticsearch.client.Node(new HttpHost(hostURL));
+
+ final RestClientTransport esTransport = (RestClientTransport) elasticsearchAPIClient._transport();
+ esTransport.restClient().setNodes(Collections.singleton(n));
}
private void setUpMetaDataIndex(HttpHost hostURL) throws IOException {
@@ -393,7 +428,7 @@ public class ElasticSearchTestConfiguration {
// parse the JSON response
List<HashMap<String, String>> list = null;
if (response2 != null) {
- ObjectMapper mapper = new ObjectMapper();
+ ObjectMapper mapper = MetaDataRepository.getSearchResultObjectMapper();
String rawBody = EntityUtils.toString(response2.getEntity());
TypeReference<List<HashMap<String, String>>> typeRef = new TypeReference<List<HashMap<String, String>>>() {};
list = mapper.readValue(rawBody, typeRef);
@@ -647,7 +682,7 @@ public class ElasticSearchTestConfiguration {
* @param id
*/
public void updateMetadata(String content, String id) {
- String elURL = elasticsearchClient.getLowLevelClient().getNodes().get(0).getHost().toURI();
+ String elURL = elasticsearchHLClient.getLowLevelClient().getNodes().get(0).getHost().toURI();
RestTemplate restTemplate = new RestTemplate();
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
@@ -657,12 +692,12 @@ public class ElasticSearchTestConfiguration {
public String updateMetadata(SearchResultDTO testExercise, String id)
throws JsonGenerationException, JsonMappingException, IOException {
- String elURL = elasticsearchClient.getLowLevelClient().getNodes().get(0).getHost().toURI();
+ String elURL = elasticsearchHLClient.getLowLevelClient().getNodes().get(0).getHost().toURI();
// first fetch previous content
RestTemplate restTemplate = new RestTemplate();
String previousContent = restTemplate.getForObject(elURL + "/metadata/_doc/" + id, String.class);
- ObjectMapper objectMapper = new ObjectMapper();
+ ObjectMapper objectMapper = MetaDataRepository.getSearchResultObjectMapper();
final JavaTimeModule javaModule = new JavaTimeModule();
objectMapper.registerModule(javaModule);
@@ -706,7 +741,7 @@ public class ElasticSearchTestConfiguration {
* @param minExpectedHits
*/
private void waitForCleanStartUp(int minExpectedHits) {
- RestHighLevelClient client = elasticsearchClient;
+ RestHighLevelClient client = elasticsearchHLClient;
int tryCount = 0;
while (tryCount < 10) {
tryCount++;
diff --git a/src/test/java/at/ac/uibk/gitsearch/service/MonitoringServiceIT.java b/src/test/java/at/ac/uibk/gitsearch/service/MonitoringServiceIT.java
new file mode 100644
index 0000000000000000000000000000000000000000..6eac1f273ea4fe672ea306b3e1b4f790713ac44b
--- /dev/null
+++ b/src/test/java/at/ac/uibk/gitsearch/service/MonitoringServiceIT.java
@@ -0,0 +1,212 @@
+package at.ac.uibk.gitsearch.service;
+
+import static at.ac.uibk.gitsearch.service.MonitoringService.HOURLY_TAG;
+import static at.ac.uibk.gitsearch.service.MonitoringService.NUMBER_DOWNLOAD_NAME;
+import static at.ac.uibk.gitsearch.service.MonitoringService.NUMBER_EXERCISE_NAME;
+import static at.ac.uibk.gitsearch.service.MonitoringService.NUMBER_LIKE_NAME;
+import static at.ac.uibk.gitsearch.service.MonitoringService.NUMBER_USER_NAME;
+import static at.ac.uibk.gitsearch.service.MonitoringService.NUMBER_VIEW_NAME;
+import static at.ac.uibk.gitsearch.service.MonitoringService.USER_TAG;
+
+import at.ac.uibk.gitsearch.IntegrationTest;
+import at.ac.uibk.gitsearch.domain.Likes;
+import at.ac.uibk.gitsearch.domain.User;
+import at.ac.uibk.gitsearch.repository.jpa.StatisticsRepository;
+import at.ac.uibk.gitsearch.repository.search.testESService.ElasticSearchTestConfiguration;
+import at.ac.uibk.gitsearch.service.dto.AdminUserDTO;
+import at.ac.uibk.gitsearch.service.dto.StatisticsDTO;
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.Measurement;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tag;
+import io.micrometer.core.instrument.Tags;
+import io.micrometer.core.instrument.search.RequiredSearch;
+import io.micrometer.prometheus.PrometheusMeterRegistry;
+import java.io.IOException;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.elasticsearch.node.NodeValidationException;
+import org.junit.Assert;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.test.context.support.WithAnonymousUser;
+import org.springframework.security.test.context.support.WithMockUser;
+import org.springframework.transaction.annotation.Transactional;
+
+@IntegrationTest
+@WithMockUser(value = "test", authorities = "sharing")
+public class MonitoringServiceIT {
+
+ @SuppressWarnings("unused")
+ private static final Logger LOGGER = LoggerFactory.getLogger(MonitoringServiceIT.class);
+
+ @Autowired
+ private MonitoringService monitoringService;
+
+ @Autowired
+ private StatisticsService statisticsService;
+
+ @Autowired
+ private StatisticsRepository statisticsRepository;
+
+ @Autowired
+ PrometheusMeterRegistry hourlyMeterRegistry;
+
+ @Autowired
+ private LikesService likesService;
+
+ @Autowired
+ private UserService userService;
+
+ @Autowired
+ MeterRegistry meterRegistry;
+
+ @Autowired
+ private SearchService searchService;
+
+ @Autowired
+ ElasticSearchTestConfiguration elasticSearchTestConfiguration;
+
+ static ElasticSearchTestConfiguration staticElasticSearchTestConfiguration;
+
+ @BeforeEach // cleanup all likes
+ @AfterEach // before and after
+ @Transactional
+ public void cleanUpLikesEtAll() {
+ likesService.findAll().stream().forEach(l -> likesService.delete(l.getId()));
+ statisticsRepository.findAll().stream().forEach(s -> statisticsRepository.delete(s));
+ }
+
+ @BeforeEach // must be started as BeforeEach, in order to autowire the ElasticSearchTestConfiguration
+ @Timeout(value = 3, unit = TimeUnit.MINUTES)
+ public void setUpESServer() throws IOException, NodeValidationException {
+ if (staticElasticSearchTestConfiguration == null) {
+ elasticSearchTestConfiguration.startTestNode();
+ staticElasticSearchTestConfiguration = elasticSearchTestConfiguration;
+ }
+ }
+
+ @AfterAll
+ @Timeout(value = 2, unit = TimeUnit.MINUTES)
+ public static void shutDownESServer() {
+ if (staticElasticSearchTestConfiguration != null) {
+ staticElasticSearchTestConfiguration.stopTestContainer();
+ staticElasticSearchTestConfiguration = null;
+ }
+ }
+
+ @Test
+ @Timeout(value = 5, unit = TimeUnit.MINUTES)
+ @WithAnonymousUser
+ public void testMonitorHourlyMetrics() throws IOException {
+ Random random = new Random(123L);
+
+ AdminUserDTO user = new AdminUserDTO();
+ user.setFirstName("first");
+ user.setLastName("last");
+ user.setEmail("abc@de.de");
+ user.setLogin("someUser");
+ user.setLastLogin(Instant.now());
+ user.setActivated(true);
+ User u = userService.createUser(user);
+
+ AtomicInteger likes = new AtomicInteger(0);
+ AtomicInteger downloads = new AtomicInteger(0);
+ AtomicInteger views = new AtomicInteger(0);
+ AtomicInteger exerciseCount = new AtomicInteger(0);
+
+ searchService
+ .getAllResultsAsStream()
+ .forEach(ex -> {
+ exerciseCount.addAndGet(1);
+ StatisticsDTO stats = new StatisticsDTO();
+ stats.setExerciseID(ex.getExerciseId());
+ stats.setViews(random.nextInt(100));
+ stats.setDownloads(random.nextInt(100));
+ statisticsService.save(stats);
+ downloads.addAndGet(stats.getDownloads());
+ views.addAndGet(stats.getViews());
+
+ if (random.nextBoolean()) {
+ Likes like = new Likes();
+ like.date(LocalDate.now()).exerciseID(ex.getExerciseId()).userID(u.getId().intValue());
+ likesService.save(like);
+ likes.incrementAndGet();
+ }
+ });
+
+ monitoringService.monitorHourlyMetrics();
+
+ // Countercheck results against original list
+ Assert.assertEquals(
+ "User counter may fail, if test data changes",
+ 5.0d,
+ getLastMeterValue(NUMBER_USER_NAME, Tags.of(HOURLY_TAG, Tag.of(USER_TAG, "All"))),
+ 0.0001d
+ );
+ Assert.assertEquals(
+ "Activae User counter may fail, if test data changes",
+ 0.0d,
+ getLastMeterValue(NUMBER_USER_NAME, Tags.of(HOURLY_TAG, Tag.of(USER_TAG, "Active"))),
+ 0.0001d
+ );
+ Assert.assertEquals(
+ "likes counter may fail, if test data changes",
+ likes.doubleValue(),
+ getLastMeterValue(NUMBER_LIKE_NAME),
+ 0.0001d
+ );
+ Assert.assertEquals(
+ "View counter may fail, if test data changes",
+ views.doubleValue(),
+ getLastMeterValue(NUMBER_VIEW_NAME),
+ 1.0001d
+ );
+ Assert.assertEquals(
+ "Download counter may fail, if test data changes",
+ downloads.doubleValue(),
+ getLastMeterValue(NUMBER_DOWNLOAD_NAME),
+ 1.0001d
+ );
+ Assert.assertEquals(
+ "Exercise counter may fail, if test data changes",
+ exerciseCount.doubleValue(),
+ getLastMeterValue(NUMBER_EXERCISE_NAME),
+ 0.0001d
+ );
+ Assert.assertEquals(
+ "Language counter may fail, if test data changes",
+ 11.0d,
+ getLastMeterValue(NUMBER_EXERCISE_NAME, Tags.of(HOURLY_TAG, Tag.of(MonitoringService.LANGUAGE_TAG, "JAVA"))),
+ 0.0001d
+ );
+ Assert.assertEquals(
+ "Language counter may fail, if test data changes",
+ 5.0d,
+ getLastMeterValue(NUMBER_EXERCISE_NAME, Tags.of(HOURLY_TAG, Tag.of(MonitoringService.LANGUAGE_TAG, "Unclassified"))),
+ 0.0001d
+ );
+ }
+
+ public double getLastMeterValue(String name) {
+ final Gauge gauge = RequiredSearch.in(hourlyMeterRegistry).name(name).gauge();
+ final Measurement next = gauge.measure().iterator().next();
+ // System.out.println(next);
+ return next.getValue();
+ }
+
+ public double getLastMeterValue(String name, Tags tags) {
+ final Gauge gauge = RequiredSearch.in(hourlyMeterRegistry).name(name).tags(tags).gauge();
+
+ return gauge.value();
+ }
+}
diff --git a/src/test/java/at/ac/uibk/gitsearch/service/SearchServiceIT.java b/src/test/java/at/ac/uibk/gitsearch/service/SearchServiceIT.java
index b2bdf4b4d9f5bfba411cc2e5ea772d3e3546794a..65c8c3ddbe6cba96709e209fedf63fcf8cbcf696 100644
--- a/src/test/java/at/ac/uibk/gitsearch/service/SearchServiceIT.java
+++ b/src/test/java/at/ac/uibk/gitsearch/service/SearchServiceIT.java
@@ -1,15 +1,13 @@
package at.ac.uibk.gitsearch.service;
-import static org.hamcrest.CoreMatchers.containsStringIgnoringCase;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.containsStringIgnoringCase;
import static org.hamcrest.Matchers.everyItem;
import static org.hamcrest.Matchers.hasItemInArray;
import static org.hamcrest.Matchers.hasProperty;
import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
@@ -31,6 +29,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import javax.ws.rs.NotFoundException;
import org.codeability.sharing.plugins.api.SharingPluginConfig;
import org.codeability.sharing.plugins.api.search.SearchInputDTO;
@@ -43,11 +42,11 @@ import org.gitlab4j.api.GitLabApi;
import org.gitlab4j.api.GitLabApiException;
import org.gitlab4j.api.RepositoryApi;
import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
import org.junit.Assert;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.slf4j.Logger;
@@ -190,22 +189,6 @@ public class SearchServiceIT {
);
}
- @Test
- @Timeout(value = 1, unit = TimeUnit.MINUTES)
- @Disabled("Ich brings nicht zum laufen :-(")
- public void testProgrammingLanguageSearch() throws Exception {
- final SearchInputMetadataDTO searchMetadata = new SearchInputMetadataDTO("Java", null, null, null, null, null);
- SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null, null, null, 0);
- SearchResultsDTO searchResultPage = searchService.searchResultPage(searchQuery);
-
- assertNotNull(searchResultPage.getSearchResult());
- assertTrue("At least one test hit", searchResultPage.getHitCount() >= 1);
- MatcherAssert.assertThat(
- searchResultPage.getSearchResult(),
- everyItem(hasProperty("metadata", hasProperty("programmingLanguage", hasItemInArray(containsStringIgnoringCase("Java")))))
- );
- }
-
@Test
@Timeout(value = 1, unit = TimeUnit.MINUTES)
public void testCreatorSearch() throws IOException {
@@ -275,7 +258,7 @@ public class SearchServiceIT {
@Test
@Timeout(value = 1, unit = TimeUnit.MINUTES)
- public void testLanguagesSearch() throws IOException {
+ public void testNaturalLanguagesSearch() throws IOException {
final SearchInputMetadataDTO searchMetadata = new SearchInputMetadataDTO(null, "latex", null, null, null, null);
searchMetadata.setNaturalLanguage(List.of("de"));
SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null, null, null, 0);
@@ -289,6 +272,36 @@ public class SearchServiceIT {
);
}
+ @Test
+ @Timeout(value = 1, unit = TimeUnit.MINUTES)
+ public void testProgrammingLanguagesSearch() throws IOException {
+ final SearchInputMetadataDTO searchMetadata = new SearchInputMetadataDTO("Java", null, null, null, null, null);
+ SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null, null, null, 0);
+ SearchResultsDTO searchResultPage = searchService.searchResultPage(searchQuery);
+ org.junit.Assert.assertNotNull(searchResultPage.getSearchResult());
+
+ org.junit.Assert.assertTrue("At least one test hit", searchResultPage.getHitCount() >= 1);
+ MatcherAssert.assertThat(
+ searchResultPage.getSearchResult(),
+ everyItem(hasProperty("metadata", hasProperty("programmingLanguage", hasItemInArray(containsStringIgnoringCase("Java")))))
+ );
+ }
+
+ @Test
+ @Timeout(value = 1, unit = TimeUnit.MINUTES)
+ public void testProgrammingLanguagesPrefixSearch() throws IOException {
+ final SearchInputMetadataDTO searchMetadata = new SearchInputMetadataDTO("Jav", null, null, null, null, null);
+ SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null, null, null, 0);
+ SearchResultsDTO searchResultPage = searchService.searchResultPage(searchQuery);
+ org.junit.Assert.assertNotNull(searchResultPage.getSearchResult());
+
+ org.junit.Assert.assertTrue("At least one test hit", searchResultPage.getHitCount() >= 1);
+ MatcherAssert.assertThat(
+ searchResultPage.getSearchResult(),
+ everyItem(hasProperty("metadata", hasProperty("programmingLanguage", hasItemInArray(containsStringIgnoringCase("Java")))))
+ );
+ }
+
// does not work with metadata version 0.4
// @Test
// public void testTypesSearchNegative() throws IOException {
@@ -361,6 +374,7 @@ public class SearchServiceIT {
final SearchInputMetadataDTO searchMetadata = new SearchInputMetadataDTO(null, null, null, null, null, null);
SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null, null, null, 0);
+ searchQuery.setPageSize(100);
SearchResultsDTO results = searchService.searchResultPage(searchQuery);
Optional<SearchResultDTO> result = results
@@ -384,6 +398,7 @@ public class SearchServiceIT {
public void testArtemisExportActionFilteringForUnauthorizedUser() throws IOException {
final SearchInputMetadataDTO searchMetadata = new SearchInputMetadataDTO(null, null, null, null, null, null);
SearchInputDTO searchQuery = new SearchInputDTO(null, searchMetadata, null, null, null, 0);
+ searchQuery.setPageSize(100);
SearchResultsDTO results = searchService.searchResultPage(searchQuery);
Optional<SearchResultDTO> result = results
@@ -411,4 +426,35 @@ public class SearchServiceIT {
Assert.assertEquals("We start at 0", 0, searchResults.getPageStartIndex());
LOGGER.info("found {} hits for search all", searchResults.getHitCount());
}
+
+ @Test
+ @Timeout(value = 1, unit = TimeUnit.MINUTES)
+ @WithAnonymousUser
+ public void testSearchStreamAllPublic() throws IOException {
+ final List<SearchResultDTO> allPublicSearchResults = searchService.getAllPublicResultsAsStream().collect(Collectors.toList());
+
+ Assert.assertTrue("At least one hit?", allPublicSearchResults.size() >= 1);
+ Assert.assertTrue("At least more than one page full?", allPublicSearchResults.size() > 10);
+
+ SearchResultsDTO searchResults = searchService.searchAllPublicResults();
+
+ // only the first 10 results are contained in the other list
+ MatcherAssert.assertThat(
+ allPublicSearchResults.subList(0, 10),
+ Matchers.containsInAnyOrder(searchResults.getSearchResult().toArray(new SearchResultDTO[0]))
+ );
+ }
+
+ @Test
+ @Timeout(value = 1, unit = TimeUnit.MINUTES)
+ @WithAnonymousUser
+ public void testSearchStreamAll() throws IOException {
+ final List<SearchResultDTO> allSearchResults = searchService.getAllResultsAsStream().collect(Collectors.toList());
+ final List<SearchResultDTO> allPublicSearchResults = searchService.getAllPublicResultsAsStream().collect(Collectors.toList());
+
+ Assert.assertTrue("At least one hit?", allSearchResults.size() >= 1);
+ Assert.assertTrue("At least more than one page full?", allSearchResults.size() > 10);
+
+ Assert.assertTrue("At at least one non public result?", allSearchResults.size() > allPublicSearchResults.size());
+ }
}
diff --git a/src/test/java/at/ac/uibk/gitsearch/web/rest/LikesResourceIT.java b/src/test/java/at/ac/uibk/gitsearch/web/rest/LikesResourceIT.java
index fa00740df5bc2932a3d04cd6f8e0760ffc39031a..fc5bba0189aee2484ee51b65f69580f9baa1285d 100644
--- a/src/test/java/at/ac/uibk/gitsearch/web/rest/LikesResourceIT.java
+++ b/src/test/java/at/ac/uibk/gitsearch/web/rest/LikesResourceIT.java
@@ -115,6 +115,12 @@ public class LikesResourceIT {
likes = createEntity(em);
}
+ @BeforeEach // cleanup all likes
+ @Transactional
+ public void cleanUpLikes() {
+ likesService.findAll().stream().forEach(l -> likesService.delete(l.getId()));
+ }
+
@Test
@Transactional
public void createLikes() throws Exception {
diff --git a/src/test/java/at/ac/uibk/gitsearch/web/rest/SearchResourceIT.java b/src/test/java/at/ac/uibk/gitsearch/web/rest/SearchResourceIT.java
index 6d5aff504892c743b3549311341cf744e1fd2a38..7e8b185ac5c67481b380ddbe33cc1d28ff8451b2 100644
--- a/src/test/java/at/ac/uibk/gitsearch/web/rest/SearchResourceIT.java
+++ b/src/test/java/at/ac/uibk/gitsearch/web/rest/SearchResourceIT.java
@@ -126,6 +126,7 @@ public class SearchResourceIT {
}
@Test
+ @Timeout(value = 5, unit = TimeUnit.MINUTES)
public void getREADME() throws Exception {
// String exerciseId = "272"; // warning: this depends on the sequence in the current search index :-(
// restSearchMockMvc.perform(post("/api/exerciseFiles/"+exerciseId)
diff --git a/src/test/java/at/ac/uibk/gitsearch/web/rest/UserResourceIT.java b/src/test/java/at/ac/uibk/gitsearch/web/rest/UserResourceIT.java
index 1c34aae25f67992f6bee270bf4fa8b4c26806c97..5381752a6031985d6daf70db5fec3adf6403a80a 100644
--- a/src/test/java/at/ac/uibk/gitsearch/web/rest/UserResourceIT.java
+++ b/src/test/java/at/ac/uibk/gitsearch/web/rest/UserResourceIT.java
@@ -609,7 +609,7 @@ class UserResourceIT {
void testAuthorityEquals() {
Authority authorityA = new Authority();
assertThat(authorityA).isNotEqualTo(null).isNotEqualTo(new Object());
- assertThat(authorityA.hashCode()).isZero();
+ assertThat(authorityA.hashCode()).isNotZero(); // useless test?
assertThat(authorityA.toString()).isNotNull();
Authority authorityB = new Authority();
diff --git a/src/test/resources/at/ac/uibk/gitsearch/repository/search/testData/es_metadata.schema.json b/src/test/resources/at/ac/uibk/gitsearch/repository/search/testData/es_metadata.schema.json
index a12979e61a915fb4b9c0cf58b11b4c167d282bc3..9ca1a145b283bfe4cb0dbc9c094661cc31faafbd 100644
--- a/src/test/resources/at/ac/uibk/gitsearch/repository/search/testData/es_metadata.schema.json
+++ b/src/test/resources/at/ac/uibk/gitsearch/repository/search/testData/es_metadata.schema.json
@@ -1,29 +1,79 @@
{
"mappings": {
"properties": {
- "file": {
+ "project": {
"properties": {
- "children": {
+ "project_id": {
+ "type": "long"
+ },
+ "project_name": {
"type": "keyword"
},
- "commit_id": {
+ "url": {
+ "type": "keyword"
+ },
+ "namespace": {
+ "type": "keyword"
+ },
+ "main_group": {
+ "type": "keyword"
+ },
+ "sub_group": {
"type": "keyword"
},
+ "visibility": {
+ "type": "keyword"
+ },
+ "users": {
+ "type": "keyword"
+ },
+ "groups": {
+ "type": "keyword"
+ },
+ "archived": {
+ "type": "boolean"
+ },
+ "star_count": {
+ "type": "integer"
+ },
+ "open_issues_count": {
+ "type": "integer"
+ },
+ "forks_count": {
+ "type": "integer"
+ },
+ "last_activity_at": {
+ "type": "date",
+ "format": "date_optional_time"
+ },
+ "description": {
+ "type": "text"
+ }
+ }
+ },
+ "file": {
+ "properties": {
"filename": {
"type": "keyword"
},
+ "path": {
+ "type": "keyword"
+ },
+ "commit_id": {
+ "type": "keyword"
+ },
"indexing_date": {
"type": "date",
- "format": "dateOptionalTime"
+ "format": "date_optional_time"
},
"last_activity_at": {
"type": "date",
- "format": "dateOptionalTime"
+ "format": "date_optional_time"
},
"parentId": {
"type": "keyword"
},
- "path": {
+ "children": {
"type": "keyword"
}
}
@@ -36,31 +86,28 @@
"audience": {
"type": "text"
},
- "collectionContent": {
- "type": "keyword"
- },
"contributor": {
"properties": {
- "affiliation": {
+ "name": {
"type": "text"
},
- "email": {
+ "affiliation": {
"type": "text"
},
- "name": {
+ "email": {
"type": "text"
}
}
},
"creator": {
"properties": {
- "affiliation": {
+ "name": {
"type": "text"
},
- "email": {
+ "affiliation": {
"type": "text"
},
- "name": {
+ "email": {
"type": "text"
}
}
@@ -84,7 +131,7 @@
"type": "text"
},
"format": {
- "type": "text"
+ "type": "keyword"
},
"identifier": {
"type": "keyword"
@@ -110,12 +157,6 @@
"license": {
"type": "keyword"
},
- "metadataVersion": {
- "type": "keyword"
- },
- "programmingLanguage": {
- "type": "text"
- },
"publicVisibility": {
"properties": {
"except": {
@@ -123,15 +164,21 @@
}
}
},
+ "metadataVersion": {
+ "type": "keyword"
+ },
+ "programmingLanguage": {
+ "type": "keyword"
+ },
"publisher": {
"properties": {
- "affiliation": {
+ "name": {
"type": "text"
},
- "email": {
+ "affiliation": {
"type": "text"
},
- "name": {
+ "email": {
"type": "text"
}
}
@@ -174,56 +221,6 @@
"type": "keyword"
}
}
- },
- "project": {
- "properties": {
- "archived": {
- "type": "boolean"
- },
- "description": {
- "type": "text"
- },
- "forks_count": {
- "type": "integer"
- },
- "groups": {
- "type": "keyword"
- },
- "last_activity_at": {
- "type": "date",
- "format": "dateOptionalTime"
- },
- "main_group": {
- "type": "keyword"
- },
- "namespace": {
- "type": "keyword"
- },
- "open_issues_count": {
- "type": "integer"
- },
- "project_id": {
- "type": "long"
- },
- "project_name": {
- "type": "keyword"
- },
- "star_count": {
- "type": "integer"
- },
- "sub_group": {
- "type": "keyword"
- },
- "url": {
- "type": "keyword"
- },
- "users": {
- "type": "keyword"
- },
- "visibility": {
- "type": "keyword"
- }
- }
}
}
}
diff --git a/src/test/resources/config/application.yml b/src/test/resources/config/application.yml
index 4f229503b7d94bdf8ad737f90d6f1263edbff9dd..5ab560fc3173aa345742fda2841f307e66b22857 100644
--- a/src/test/resources/config/application.yml
+++ b/src/test/resources/config/application.yml
@@ -14,9 +14,14 @@
# ===================================================================
logging:
level:
- ROOT: DEBUG
- tech.jhipster: DEBUG
- org.hibernate.SQL: DEBUG
+ ROOT: WARN
+
+ tech.jhipster: WARN
+ org.hibernate.SQL: WARN
+ org.springframework: WARN
+ org.springframework.web: WARN
+ org.springframework.security: WARN
+ org.zalando: ERROR
at.ac.uibk.gitsearch: DEBUG
org.glassfish.jersey.client.ClientExecutorProvidersConfigurator: INFO
javax.management.mbeanserver: INFO
diff --git a/src/test/resources/other/soapUI/localhost-29200-soapui-project.xml b/src/test/resources/other/soapUI/localhost-29200-soapui-project.xml
new file mode 100644
index 0000000000000000000000000000000000000000..a396932bab17af170af4b573ed39549ae526eda6
--- /dev/null
+++ b/src/test/resources/other/soapUI/localhost-29200-soapui-project.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<con:soapui-project id="e5e6ec50-ebc6-40b2-b869-94fd32b98793" activeEnvironment="Default" name="localhost:29200" resourceRoot="" soapui-version="5.7.0" xmlns:con="http://eviware.com/soapui/config"><con:settings/><con:interface xsi:type="con:RestService" id="5e1934cd-82a4-4bba-9ac0-cca93457c9fb" wadlVersion="http://wadl.dev.java.net/2009/02" name="http://localhost:29200" type="rest" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><con:settings/><con:definitionCache/><con:endpoints><con:endpoint>http://localhost:29200</con:endpoint></con:endpoints><con:resource name="" path="/metadata/_search?pretty" id="b3276207-be96-4f21-8069-7490845e0d2e"><con:settings/><con:parameters/><con:method name="1" id="af5694fe-1ad1-4f25-989b-bd826e1a04b2" method="POST"><con:settings/><con:parameters/><con:representation type="REQUEST"><con:mediaType>application/json</con:mediaType><con:params/></con:representation><con:representation type="FAULT"><con:mediaType>application/json; charset=UTF-8</con:mediaType><con:status>400</con:status><con:params/><con:element xmlns:_se="http://localhost/metadata/_search/">_se:Fault</con:element></con:representation><con:representation type="RESPONSE"><con:mediaType>application/json; charset=UTF-8</con:mediaType><con:status>200</con:status><con:params/><con:element xmlns:_se="http://localhost/metadata/_search/">_se:Response</con:element></con:representation><con:request name="Request 1" id="ee6d6b05-b9a9-4d5d-99e2-d4b2cfb1c92c" mediaType="application/json" postQueryString="false"><con:settings><con:setting id="com.eviware.soapui.impl.wsdl.WsdlRequest@request-headers"><xml-fragment/></con:setting></con:settings><con:endpoint>http://localhost:29200</con:endpoint><con:request>{
+ "query": {
+ "term": {
+ "metadata.language": "de"
+ }
+ }
+}
+</con:request><con:originalUri>http://localhost/metadata/_search/</con:originalUri><con:credentials><con:authType>No Authorization</con:authType></con:credentials><con:jmsConfig JMSDeliveryMode="PERSISTENT"/><con:jmsPropertyConfig/><con:parameters/><con:parameterOrder/></con:request></con:method></con:resource><con:resource name="search Test2" path="/metadata/_search?pretty" id="b3276207-be96-4f21-8069-7490845e0d2e"><con:settings/><con:parameters/><con:method name="1" id="af5694fe-1ad1-4f25-989b-bd826e1a04b2" method="POST"><con:settings/><con:parameters/><con:representation type="REQUEST"><con:mediaType>application/json</con:mediaType><con:params/></con:representation><con:representation type="FAULT"><con:mediaType>application/json; charset=UTF-8</con:mediaType><con:status>400</con:status><con:params/><con:element xmlns:_se="http://localhost/metadata/_search/">_se:Fault</con:element></con:representation><con:representation type="RESPONSE"><con:mediaType>application/json; charset=UTF-8</con:mediaType><con:status>200</con:status><con:params/><con:element xmlns:_se="http://localhost/metadata/_search/">_se:Response</con:element></con:representation><con:representation type="RESPONSE"><con:mediaType xsi:nil="true"/><con:status>0</con:status><con:params/><con:element>data</con:element></con:representation><con:representation type="RESPONSE"><con:mediaType xsi:nil="true"/><con:status>0</con:status><con:params/><con:element>data</con:element></con:representation><con:request name="Request 1" id="ee6d6b05-b9a9-4d5d-99e2-d4b2cfb1c92c" mediaType="application/json" postQueryString="false"><con:settings><con:setting id="com.eviware.soapui.impl.wsdl.WsdlRequest@request-headers"><xml-fragment/></con:setting></con:settings><con:endpoint>http://localhost:29200</con:endpoint><con:request>{
+ "query": {
+ "bool": {
+ "must": [
+ {
+ "bool": {
+ "should": [
+ {
+ "match_phrase_prefix": {
+ "metadata.keyword": {
+ "query": "latex"
+ }
+ }
+ }
+ ]
+ }
+ },
+ {
+ "bool": {
+ "should": [
+ {
+ "match": {
+ "metadata.language": {
+ "query": "de"
+ }
+ }
+ }
+ ]
+ }
+ },
+ {
+ "bool": {
+ "should": [
+ {
+ "term": {
+ "project.visibility": {
+ "boost": 0.0,
+ "value": "public"
+ }
+ }
+ }
+ ]
+ }
+ }
+ ]
+ }
+ }
+}</con:request><con:originalUri>http://localhost/metadata/_search/</con:originalUri><con:credentials><con:authType>No Authorization</con:authType></con:credentials><con:jmsConfig JMSDeliveryMode="PERSISTENT"/><con:jmsPropertyConfig/><con:parameters/><con:parameterOrder/></con:request></con:method></con:resource></con:interface><con:properties/><con:wssContainer/><con:oAuth2ProfileContainer/><con:oAuth1ProfileContainer/></con:soapui-project>
\ No newline at end of file