This is the codeAbility Sharing Platform! Learn more about the codeAbility Sharing Platform.

Skip to content
Snippets Groups Projects
Commit 3e41b63f authored by Johannes Kainz's avatar Johannes Kainz
Browse files

added a custom prometheus endpoint

parent 5aa6b642
Branches
2 merge requests!188Merging Peer Reviewing et. al to Master,!164211 peer reviewing functionality
package at.ac.uibk.gitsearch.config.prometheus;
import io.prometheus.client.CollectorRegistry;
import java.util.Set;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.actuate.endpoint.annotation.ReadOperation;
import org.springframework.boot.actuate.endpoint.web.WebEndpointResponse;
import org.springframework.boot.actuate.metrics.export.prometheus.PrometheusScrapeEndpoint;
import org.springframework.boot.actuate.metrics.export.prometheus.TextOutputFormat;
/**
* Defines the "Skeleton" for a custom prometheus scrape endpoint.
* To define a new scrape Endpoint extend this class and add @(Web)Endpoint(id = 'id')-Annotation.
*
*
* @author Johannes Kainz
*/
public abstract class CustomPrometheusScrapeEndpoint {
private final Logger log = LoggerFactory.getLogger(CustomPrometheusScrapeEndpoint.class);
private final PrometheusScrapeEndpoint prometheusScrapeEndpoint;
public CustomPrometheusScrapeEndpoint(CollectorRegistry collectorRegistry) {
this.prometheusScrapeEndpoint = new PrometheusScrapeEndpoint(collectorRegistry);
}
@ReadOperation(producesFrom = TextOutputFormat.class)
public WebEndpointResponse<String> prometheusScrap(TextOutputFormat format, @Nullable Set<String> includedNames) {
return this.prometheusScrapeEndpoint.scrape(format, includedNames);
}
}
package at.ac.uibk.gitsearch.config.prometheus;
import io.prometheus.client.CollectorRegistry;
import org.springframework.boot.actuate.endpoint.web.annotation.WebEndpoint;
/**
* Defines a custom prometheus scrape endpoint.
*
* @author Johannes Kainz
*/
@WebEndpoint(id = "prometheusHourly")
public class HourlyPrometheusScrapeEndpoint extends CustomPrometheusScrapeEndpoint {
public HourlyPrometheusScrapeEndpoint(CollectorRegistry collectorRegistry) {
super(collectorRegistry);
}
}
package at.ac.uibk.gitsearch.config.prometheus;
import io.micrometer.core.instrument.Clock;
import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.config.MeterFilter;
import io.micrometer.prometheus.PrometheusConfig;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import io.prometheus.client.CollectorRegistry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.actuate.metrics.export.prometheus.PrometheusScrapeEndpoint;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* Configures Prometheus to use more than the default endpoint
* to scrape metrics with different time intervals.
*
* @author Johannes Kainz
*/
@Configuration
public class PrometheusConfiguration {
private static final String TAG_INTERVAL = "interval";
private static final String TAG_HOURLY_INTERVAL = "hourly";
private final Logger log = LoggerFactory.getLogger(PrometheusConfiguration.class);
/**
* represents all metrics which should be scraped from the default prometheus endpoint
*/
private final PrometheusMeterRegistry defaultMeterRegistry;
/**
* represents all metrics which should be scraped form the custom hourly scrape endpoint
*/
private final PrometheusMeterRegistry hourlyMeterRegistry;
public PrometheusConfiguration(PrometheusConfig config, Clock clock) {
this.defaultMeterRegistry = new PrometheusMeterRegistry(config, new CollectorRegistry(true), clock);
this.defaultMeterRegistry.config()
.meterFilter(MeterFilter.denyUnless(id -> id.getTags().stream().noneMatch(tag -> tag.getKey().equals(TAG_INTERVAL))));
this.hourlyMeterRegistry = new PrometheusMeterRegistry(config, new CollectorRegistry(true), clock);
this.hourlyMeterRegistry.config()
.meterFilter(MeterFilter.denyUnless(id -> id.getTags().contains(Tag.of(TAG_INTERVAL, TAG_HOURLY_INTERVAL))));
}
@Bean
PrometheusMeterRegistry defaultMeterRegistry() {
return this.defaultMeterRegistry;
}
@Bean
PrometheusMeterRegistry hourlyMeterRegistry() {
return this.hourlyMeterRegistry;
}
@Bean
PrometheusScrapeEndpoint defaultPrometheusScrapeEndpoint() {
return new PrometheusScrapeEndpoint(this.defaultMeterRegistry.getPrometheusRegistry());
}
@Bean
CustomPrometheusScrapeEndpoint hourlyPrometheusScrapeEndpoint() {
return new HourlyPrometheusScrapeEndpoint(this.hourlyMeterRegistry.getPrometheusRegistry());
}
}
/**
* Prometheus configuration files.
*/
package at.ac.uibk.gitsearch.config.prometheus;
package at.ac.uibk.gitsearch.service;
import at.ac.uibk.gitsearch.service.dto.StatisticsDTO;
import at.ac.uibk.gitsearch.service.impl.StatisticsServiceImpl;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.Tags;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.codeability.sharing.plugins.api.search.SearchResultDTO;
import org.codeability.sharing.plugins.api.search.SearchResultsDTO;
import org.codeability.sharing.plugins.api.search.UserProvidedMetadataDTO;
......@@ -15,11 +19,6 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
/**
* Service class for monitoring
*
......@@ -28,15 +27,21 @@ import java.util.concurrent.atomic.AtomicLong;
@Service
@Transactional
public class MonitoringService {
private static final String TAG_INTERVAL = "interval";
private static final String TAG_HOURLY_INTERVAL = "hourly";
private static final String USER_TAG = "classification";
private static final String LANGUAGE_TAG = "language";
private static final String EXERCISE_TAG = "exerciseID";
private static final String USER_NAME = "number.users";
private static final String EXERCISE_NAME = "number.exercises";
private static final String LIKE_NAME = "number.likes";
private static final String VIEW_NAME = "number.views";
private static final String DOWNLOAD_NAME = "number.downloads";
private static final String NUMBER_USER_NAME = "number.users";
private static final String NUMBER_EXERCISE_NAME = "number.exercises";
private static final String NUMBER_LIKE_NAME = "number.likes";
private static final String NUMBER_VIEW_NAME = "number.views";
private static final String NUMBER_DOWNLOAD_NAME = "number.downloads";
private static final Tag HOURLY_TAG = Tag.of(TAG_INTERVAL, TAG_HOURLY_INTERVAL);
private final Logger log = LoggerFactory.getLogger(MonitoringService.class);
......@@ -54,8 +59,13 @@ public class MonitoringService {
private final MeterRegistry meterRegistry;
public MonitoringService(MeterRegistry meterRegistry, UserService userService, SearchService searchService, LikesService likesService, StatisticsServiceImpl statisticsService) {
public MonitoringService(
MeterRegistry meterRegistry,
UserService userService,
SearchService searchService,
LikesService likesService,
StatisticsServiceImpl statisticsService
) {
this.meterRegistry = meterRegistry;
this.userService = userService;
......@@ -68,8 +78,18 @@ public class MonitoringService {
this.numberViews = new HashMap<>();
this.numberDownloads = new HashMap<>();
this.numberUsers = meterRegistry.gauge(USER_NAME, Tags.of(USER_TAG, "All"), new AtomicLong(this.userService.countAllUsers()));
this.numberActiveUsers = meterRegistry.gauge(USER_NAME, Tags.of(USER_TAG, "Active"), new AtomicLong(this.userService.countAllActiveUsers()));
this.numberUsers =
meterRegistry.gauge(
NUMBER_USER_NAME,
Tags.of(HOURLY_TAG, Tag.of(USER_TAG, "All")),
new AtomicLong(this.userService.countAllUsers())
);
this.numberActiveUsers =
meterRegistry.gauge(
NUMBER_USER_NAME,
Tags.of(HOURLY_TAG, Tag.of(USER_TAG, "Active")),
new AtomicLong(this.userService.countAllActiveUsers())
);
try {
final SearchResultsDTO searchResults = this.getAllSearchResults();
......@@ -78,81 +98,73 @@ public class MonitoringService {
this.setNumberExercises(exerciseIDs.size());
this.setExercisesLanguageCount(searchResults);
this.setExerciseStatistics(exerciseIDs);
} catch (IOException exception){
} catch (IOException exception) {
log.warn("An IOException occurred when searching for all exercises\n" + Arrays.toString(exception.getStackTrace()));
}
}
/**
* exports the monitoring relevant metrics regularly
* <p></p>
* This is scheduled to get fired every minute (for testing reasons)
*/
@Scheduled(cron = "30 * * * * ?")
public void exportMonitoringMetrics(){
public void exportMonitoringMetrics() {
log.debug("Export monitoring relevant metrics");
this.setNumberUsers();
try{
try {
SearchResultsDTO searchResults = this.getAllSearchResults();
List<String> exerciseIDs = this.getExerciseIDs(searchResults.getSearchResult());
this.setNumberExercises(exerciseIDs.size());
this.setExercisesLanguageCount(searchResults);
this.setExerciseStatistics(exerciseIDs);
} catch (IOException exception){
} catch (IOException exception) {
log.warn("An IOException occurred\n" + Arrays.toString(exception.getStackTrace()));
}
}
private void setNumberUsers(){
private void setNumberUsers() {
log.debug("set the number of users");
this.numberUsers.set(this.userService.countAllUsers());
this.numberActiveUsers.set(this.userService.countAllActiveUsers());
}
/**
* Sets likes, views and downloads for each given exerciseID
*
* @param exerciseIDs given exercise IDs
*/
private void setExerciseStatistics(List<String> exerciseIDs){
private void setExerciseStatistics(List<String> exerciseIDs) {
log.debug("set the exercise statistics for {}", exerciseIDs);
exerciseIDs.forEach(exerciseID -> {
this.setNumberLikesForExerciseID(exerciseID);
final Optional<StatisticsDTO> statisticOptional = this.statisticsService.findOneByExerciseID(exerciseID);
if( statisticOptional.isPresent()){
if (statisticOptional.isPresent()) {
StatisticsDTO statistic = statisticOptional.get();
this.setNumberViewsForExerciseID(exerciseID, statistic.getViews());
this.setNumberDownloadsForExerciseID(exerciseID, statistic.getDownloads());
}
});
}
/**
* Sets the given value as number of all exercises
*
* @param value given value
*/
private void setNumberExercises(int value){
if(!this.numberExercises.containsKey("All")){
this.numberExercises.put("All", meterRegistry.gauge(EXERCISE_NAME, Tags.of(LANGUAGE_TAG, "All"), new AtomicInteger(value)));
}else{
private void setNumberExercises(int value) {
if (!this.numberExercises.containsKey("All")) {
this.numberExercises.put(
"All",
meterRegistry.gauge(NUMBER_EXERCISE_NAME, Tags.of(HOURLY_TAG, Tag.of(LANGUAGE_TAG, "All")), new AtomicInteger(value))
);
} else {
this.numberExercises.get("All").set(value);
}
}
......@@ -162,19 +174,24 @@ public class MonitoringService {
*
* @param searchResults the search results
*/
private void setExercisesLanguageCount(SearchResultsDTO searchResults){
private void setExercisesLanguageCount(SearchResultsDTO searchResults) {
log.debug("set exercise language count");
Map<String, Integer> languageCount = this.getLanguageCount(searchResults);
for (String language: languageCount.keySet()) {
if(!this.numberExercises.containsKey(language)){
this.numberExercises
.put(language, Metrics.gauge(EXERCISE_NAME, Tags.of(LANGUAGE_TAG, language), new AtomicInteger(languageCount.get(language)) ));
} else{
for (String language : languageCount.keySet()) {
if (!this.numberExercises.containsKey(language)) {
this.numberExercises.put(
language,
Metrics.gauge(
NUMBER_EXERCISE_NAME,
Tags.of(HOURLY_TAG, Tag.of(LANGUAGE_TAG, language)),
new AtomicInteger(languageCount.get(language))
)
);
} else {
this.numberExercises.get(language).set(languageCount.get(language));
}
}
}
/**
......@@ -183,7 +200,7 @@ public class MonitoringService {
* @param searchResults the search results
* @return a map containing the language counts
*/
private Map<String, Integer> getLanguageCount(SearchResultsDTO searchResults){
private Map<String, Integer> getLanguageCount(SearchResultsDTO searchResults) {
Map<String, Integer> languageCount = new HashMap<>();
List<UserProvidedMetadataDTO> metaDataDTO = new ArrayList<>();
......@@ -191,13 +208,14 @@ public class MonitoringService {
searchResultList.forEach(searchResult -> metaDataDTO.add(searchResult.getMetadata()));
for (UserProvidedMetadataDTO metaData: metaDataDTO ) {
for (UserProvidedMetadataDTO metaData : metaDataDTO) {
String[] languageMetaData = metaData.getProgrammingLanguage();
if (languageMetaData != null) {
Arrays.stream(metaData.getProgrammingLanguage())
.forEach(language -> languageCount.put(language, languageCount.getOrDefault(language, 0) + 1 ));
}else{
languageCount.put("Unclassified", languageCount.getOrDefault("Unclassified",0)+1);
Arrays
.stream(metaData.getProgrammingLanguage())
.forEach(language -> languageCount.put(language, languageCount.getOrDefault(language, 0) + 1));
} else {
languageCount.put("Unclassified", languageCount.getOrDefault("Unclassified", 0) + 1);
}
}
......@@ -209,12 +227,19 @@ public class MonitoringService {
*
* @param exerciseID given exercise ID
*/
private void setNumberLikesForExerciseID(String exerciseID){
private void setNumberLikesForExerciseID(String exerciseID) {
long numberLikes = this.likesService.findNumberOfLikesByExerciseID(exerciseID);
if (!this.numberLikes.containsKey(exerciseID)){
this.numberLikes.put(exerciseID, meterRegistry.gauge(LIKE_NAME, Tags.of(EXERCISE_TAG, exerciseID), new AtomicLong(numberLikes)));
}else{
if (!this.numberLikes.containsKey(exerciseID)) {
this.numberLikes.put(
exerciseID,
meterRegistry.gauge(
NUMBER_LIKE_NAME,
Tags.of(HOURLY_TAG, Tag.of(EXERCISE_TAG, exerciseID)),
new AtomicLong(numberLikes)
)
);
} else {
this.numberLikes.get(exerciseID).set(numberLikes);
}
}
......@@ -225,10 +250,13 @@ public class MonitoringService {
* @param exerciseID given exercise ID
* @param views given number of views
*/
private void setNumberViewsForExerciseID(String exerciseID, long views){
if(!this.numberViews.containsKey(exerciseID)){
this.numberViews.put(exerciseID, meterRegistry.gauge(VIEW_NAME, Tags.of(EXERCISE_TAG, exerciseID), new AtomicLong(views)));
}else{
private void setNumberViewsForExerciseID(String exerciseID, long views) {
if (!this.numberViews.containsKey(exerciseID)) {
this.numberViews.put(
exerciseID,
meterRegistry.gauge(NUMBER_VIEW_NAME, Tags.of(HOURLY_TAG, Tag.of(EXERCISE_TAG, exerciseID)), new AtomicLong(views))
);
} else {
this.numberViews.get(exerciseID).set(views);
}
}
......@@ -239,16 +267,21 @@ public class MonitoringService {
* @param exerciseID given exercise ID
* @param downloads given number of downloads
*/
private void setNumberDownloadsForExerciseID(String exerciseID, long downloads){
if(!this.numberDownloads.containsKey(exerciseID)){
this.numberDownloads.put(exerciseID, meterRegistry.gauge(DOWNLOAD_NAME, Tags.of(EXERCISE_TAG, exerciseID), new AtomicLong(downloads)));
}else{
private void setNumberDownloadsForExerciseID(String exerciseID, long downloads) {
if (!this.numberDownloads.containsKey(exerciseID)) {
this.numberDownloads.put(
exerciseID,
meterRegistry.gauge(
NUMBER_DOWNLOAD_NAME,
Tags.of(HOURLY_TAG, Tag.of(EXERCISE_TAG, exerciseID)),
new AtomicLong(downloads)
)
);
} else {
this.numberDownloads.get(exerciseID).set(downloads);
}
}
/**
* Gets the search results of a search for all
*
......@@ -265,7 +298,7 @@ public class MonitoringService {
* @param searchResultList the search result list
* @return a list of exerciseIDs
*/
private List<String> getExerciseIDs(List<SearchResultDTO> searchResultList){
private List<String> getExerciseIDs(List<SearchResultDTO> searchResultList) {
log.debug("Get the exercise IDs from a list of search results");
List<String> exerciseIDs = new ArrayList<>();
......@@ -273,5 +306,5 @@ public class MonitoringService {
searchResultList.forEach(searchResult -> exerciseIDs.add(searchResult.getExerciseId()));
return exerciseIDs;
}
//TODO: refactor !
}
......@@ -43,6 +43,7 @@ management:
'threaddump',
'caches',
'liquibase',
'prometheusHourly',
]
endpoint:
health:
......@@ -218,6 +219,6 @@ application:
guestAccessToken: secreit
adminAccessToken: secret
deployment-info:
commit-id: "${gitCommitId}"
branch: "${gitBranch}"
deploymentDate: ${gitCommitDate}
\ No newline at end of file
commit-id: '${gitCommitId}'
branch: '${gitBranch}'
deploymentDate: ${gitCommitDate}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment