This is the codeAbility Sharing Platform! Learn more about the codeAbility Sharing Platform.

Skip to content
Snippets Groups Projects

Resolve "Die Statistiken sollten auch in ElasticSearch hinterlegt werden."

Merged Resolve "Die Statistiken sollten auch in ElasticSearch hinterlegt werden."
384-die-statistiken-sollten-auch-in-elasticsearch-hinterlegt-werden into development
Merged Michael Breu requested to merge 384-die-statistiken-sollten-auch-in-elasticsearch-hinterlegt-werden into development
Compare
and
Show latest version
2 files
+ 133
− 71
Compare changes
  • Side-by-side
  • Inline
Files
2
src/main/java/at/ac/uibk/gitsearch/service/GitlabEventService.java
+ 133
− 69
@@ -5,6 +5,7 @@ import at.ac.uibk.gitsearch.repository.search.MetaDataRepository;
import at.ac.uibk.gitsearch.service.util.SlashPath;
import at.ac.uibk.gitsearch.service.vocabulary.VocabularyService;
import at.ac.uibk.gitsearch.service.vocabulary.VocabularyService.ValidationResultDTO;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -54,6 +55,33 @@ import org.springframework.stereotype.Service;
@Service
public class GitlabEventService {
@Autowired
private GitLabRepository gitLabRepository;
private GitLabApi adminGitLabApi;
@Autowired
private VocabularyService vocabularyService;
@Autowired
private MetaDataRepository metaDataRepository;
@Autowired
private MailService mailService;
private static final Logger logger = LoggerFactory.getLogger(GitlabEventService.class);
static final String MISSING_METADATA_MSG =
"Metadata file is missing! Please, provide a metadata file! You may use \n" +
"either 'metadata.yml', 'metadata.yaml' or 'metadata.json'!";
public static final String GITLAB_MAIN_GROUP = "sharing";
protected static final String[] BRANCH_PRIORITY = { "main", "master" };
{
Arrays.sort(BRANCH_PRIORITY);
}
static class AnalysedCommit {
Project project;
@@ -70,6 +98,30 @@ public class GitlabEventService {
}
}
/**
* just a helper class that does not ignore collectionContent
*
* @author Michael
*
*/
protected static class ExtUserProvidedMetadataDTO extends UserProvidedMetadataDTO {
@JsonIgnore(false)
private String[] collectionContent;
@Override
@SuppressWarnings("PMD.MethodReturnsInternalArray")
public String[] getCollectionContent() {
return collectionContent;
}
@Override
@SuppressWarnings("PMD.ArrayIsStoredDirectly")
public void setCollectionContent(String[] collectionContent) {
this.collectionContent = collectionContent;
}
}
public static class MetadataInfo {
private String url;
@@ -156,6 +208,8 @@ public class GitlabEventService {
*/
public class ItemPath {
public static final int MAX_ID_LENGTH = 512;
// directory path either relative to some other Item path, or absolute to root
// of object
@@ -243,21 +297,19 @@ public class GitlabEventService {
return this.path.getFileName().toString();
}
public static final int MAX_ID_LENGTH = 512;
public String getDocId() {
String prefix = "[" + this.projectId + "]";
int restLength = MAX_ID_LENGTH - prefix.length();
String docId;
String pathString = this.path.getParent().toString();
if (pathString == null || pathString.equals("/")) return prefix;
if (pathString == null || pathString.equals("/")) {
return prefix;
}
int restLength = MAX_ID_LENGTH - prefix.length();
if (restLength >= pathString.length()) {
docId = prefix + pathString;
return prefix + pathString;
} else {
docId = prefix + pathString.substring(pathString.length() - restLength);
return prefix + pathString.substring(pathString.length() - restLength);
}
return docId;
}
public ItemPath createChildrenItemPath(String path, GitLabApi adminGitLabApi) throws GitLabApiException, ParseException {
@@ -324,9 +376,9 @@ public class GitlabEventService {
throw new ParseException("Cannot parse " + childPath, 0);
}
String childGitUrl = match.group(1);
String childProjectId = match.group(3);
if (StringUtils.isEmpty(childProjectId)) {
childProjectId = null;
String childProjectId = null;
if (!StringUtils.isEmpty(match.group(3))) {
childProjectId = match.group(3);
}
String childPathString = match.group(4);
if (childProjectId != null && childPathString != null && childPathString.startsWith("/")) {
@@ -346,10 +398,10 @@ public class GitlabEventService {
ItemPath item;
List<Node> children;
UserProvidedMetadataDTO metaData;
ExtUserProvidedMetadataDTO metaData;
int level;
Node(ItemPath item, UserProvidedMetadataDTO metaData, int level) {
Node(ItemPath item, ExtUserProvidedMetadataDTO metaData, int level) {
this.item = item;
this.metaData = metaData;
this.children = new ArrayList<>();
@@ -418,9 +470,9 @@ public class GitlabEventService {
private String footerMsg = "";
private String helpMsg = "";
public ErrorMessage(PushSystemHookEvent project_info, List<MetadataInfo> metadata_info) {
this.projectInfo = project_info;
this.metadataInfo = metadata_info;
public ErrorMessage(PushSystemHookEvent projectInfo, List<MetadataInfo> metadataInfo) {
this.projectInfo = projectInfo;
this.metadataInfo = metadataInfo;
}
public String createHtml() {
@@ -429,7 +481,7 @@ public class GitlabEventService {
this.helpMsg =
"More information about the expected metadata file can be found in the <a href=\"https://sharing-codeability.uibk.ac.at/sharing/codeability-sharing-platform/-/wikis/technical/MetaData-Documentation\">metadata reference</a>.";
return "TODO";
return "TODO " + footerMsg + helpMsg;
// FreeMar template_env = new TemplateEnv(FileSystemLoader(DIR));
// Template template = template_env.get_template(PATH_MAIL_TEMPLATE);
// return template.render("error_msg", this).toString();
@@ -450,10 +502,10 @@ public class GitlabEventService {
if (metaDataInfos.url != null) {
fileUrl = " (" + this.projectInfo.getRepository().getUrl() + metaDataInfos.url + ")";
}
body.append("\n");
body.append('\n');
body.append(metaDataInfos.filename);
body.append(fileUrl);
body.append(":");
body.append(':');
for (String error : metaDataInfos.errors) {
body.append("\n - ");
body.append(error);
@@ -488,7 +540,7 @@ public class GitlabEventService {
public class HealthCheck {
public void sendValidationErrorMail(PushSystemHookEvent gitLabEvent, AnalysedCommit analysed_commit) {
public void sendValidationErrorMail(PushSystemHookEvent gitLabEvent, AnalysedCommit analysedCommit) {
User user;
try {
user = adminGitLabApi.getUserApi().getUser(gitLabEvent.getUserId());
@@ -496,26 +548,37 @@ public class GitlabEventService {
logger.error("Cannot get userInfo for {}", gitLabEvent.getUserId(), e);
return;
}
EventCommit commit = analysed_commit.commitHash;
EventCommit commit = analysedCommit.commitHash;
String subject = gitLabEvent.getProject().getPathWithNamespace() + ": Health check failed!";
ErrorMessage errorMsg = new ErrorMessage(gitLabEvent, analysed_commit.errors);
ErrorMessage errorMsg = new ErrorMessage(gitLabEvent, analysedCommit.errors);
String mailHtml = errorMsg.createHtml();
String mailPlain = errorMsg.createPlain();
logger.info("TODO: {} {} {} {} {}", commit, subject, errorMsg, mailHtml, mailPlain);
mailService.sendEmail(user.getEmail(), subject, mailHtml, false, true);
}
public Pair<List<MetadataInfo>, Optional<Node>> validateProject(Project project, EventCommit commit) throws GitLabApiException {
List<String> metadataFiles = getRootRepositoryMetadataFiles(project, commit);
List<MetadataInfo> errors = checkForSingleMetadataFiles(metadataFiles);
if (!errors.isEmpty()) {
return Pair.of(errors, Optional.empty());
return validateFile(new ItemPath(SlashPath.get("/"), commit, project.getId()), 0);
}
private Pair<List<MetadataInfo>, Optional<Node>> validateFile(ItemPath path, int level) throws GitLabApiException {
String metadataFilePath;
if (path.path.isDirectory()) {
List<String> metadataFiles = getRootRepositoryMetadataFiles(path);
List<MetadataInfo> errors = checkForSingleMetadataFiles(metadataFiles);
if (!errors.isEmpty()) {
return Pair.of(errors, Optional.empty());
}
metadataFilePath = metadataFiles.get(0);
} else {
metadataFilePath = path.path.toString();
}
MetadataInfo metadataFile = new MetadataInfo(null, metadataFiles.get(0));
MetadataInfo metadataFile = new MetadataInfo(null, metadataFilePath);
UserProvidedMetadataDTO metaData;
ExtUserProvidedMetadataDTO metaData;
try {
metaData = parseMetadata(project, commit, metadataFile);
metaData = parseMetadata(path.getProjectId(), path.commit, metadataFile);
} catch (ParseException e) {
var metadataInfo = new MetadataInfo(null, metadataFile.getFilename());
metadataInfo.getErrors().add(e.getMessage());
@@ -523,10 +586,9 @@ public class GitlabEventService {
}
Node rootNode = new Node(
new ItemPath(SlashPath.get("/", metadataFile.getFilename()), commit, project.getId()),
new ItemPath(SlashPath.get(metadataFile.getFilename()), path.commit, path.getProjectId()),
metaData,
0
/* root */
level
);
return validateMetadataFile(rootNode);
}
@@ -543,17 +605,18 @@ public class GitlabEventService {
jsonMapper.findAndRegisterModules();
}
private UserProvidedMetadataDTO parseMetadata(Project project, EventCommit commit, MetadataInfo metadataFile)
private ExtUserProvidedMetadataDTO parseMetadata(long projectId, EventCommit commit, MetadataInfo metadataFile)
throws ParseException {
try {
final RepositoryFile mdFile = adminGitLabApi
.getRepositoryFileApi()
.getFile(project, metadataFile.getFilename(), commit.getId());
.getFile(projectId, metadataFile.getFilename(), commit.getId());
String content = new String(Base64.getDecoder().decode(mdFile.getContent()), StandardCharsets.UTF_8);
var mapper = mdFile.getFileName().endsWith(".json") ? jsonMapper : yamlMapper;
return mapper.readValue(content, UserProvidedMetadataDTO.class);
return mapper.readValue(content, ExtUserProvidedMetadataDTO.class);
} catch (GitLabApiException | JsonProcessingException e) {
logger.warn("Parseexception?", e);
throw new ParseException(e.getMessage(), 0);
}
}
@@ -578,27 +641,55 @@ public class GitlabEventService {
//
// }
List<MetadataInfo> metaDataInfos = new ArrayList<>();
metaDataInfos.add(metadataInfo);
final String[] collectionContent = node.metaData.getCollectionContent();
if (collectionContent != null && collectionContent.length != 0) {
List<ItemPath> thisParents = new ArrayList<GitlabEventService.ItemPath>(parents);
List<ItemPath> thisParents = new ArrayList<>(parents);
thisParents.add(thisPath);
Arrays
.stream(collectionContent)
.map(childName -> new ItemPath(SlashPath.get(thisPath.path.toString(), childName), null, null));
.map(childName -> {
try {
return node.item.createChildrenItemPath(childName, adminGitLabApi);
} catch (GitLabApiException | ParseException e1) {
logger.error("Cannot parse child {}", childName, e1);
metadataInfo.errors.add("Cannot parse child " + childName + ": " + e1.getMessage());
return null;
}
})
.forEach(childPath -> {
Pair<List<MetadataInfo>, Optional<Node>> subValidation;
try {
subValidation = validateFile(childPath, node.level + 1);
} catch (GitLabApiException e1) {
logger.error("Cannot parse child {}", e1);
metadataInfo.errors.add("Cannot parse child " + childPath.toString() + ": " + e1.getMessage());
return;
}
List<MetadataInfo> subMetadataInfos = subValidation.getLeft();
metaDataInfos.addAll(subMetadataInfos);
Optional<Node> nodeO = subValidation.getRight();
nodeO.ifPresent(n -> node.children.add(n));
});
}
return Pair.of(Collections.singletonList(metadataInfo), Optional.of(node));
return Pair.of(metaDataInfos, Optional.of(node));
}
public List<String> getRootRepositoryMetadataFiles(Project project, EventCommit commit) throws GitLabApiException {
return getRepositoryMetadataFiles(project, commit, ".");
public List<String> getRootRepositoryMetadataFiles(ItemPath item) throws GitLabApiException {
if (item.path.isDirectory()) {
return getRepositoryMetadataFiles(item.getProjectId(), item.getCommit(), item.getPath().toString());
} else {
return Collections.singletonList(item.getPath().toString());
}
}
public List<String> getRepositoryMetadataFiles(Project project, EventCommit commit, String path) throws GitLabApiException {
public List<String> getRepositoryMetadataFiles(long projectId, EventCommit commit, String path) throws GitLabApiException {
final Stream<TreeItem> files = adminGitLabApi
.getRepositoryApi()
.getTreeStream(project.getId(), path, commit.getId(), false/* only this folder */);
.getTreeStream(projectId, path, commit.getId(), false/* only this folder */);
return files.filter(f -> METADATA_FILENAMES.contains(f.getName())).map(f -> f.getPath()).collect(Collectors.toList());
}
@@ -624,13 +715,6 @@ public class GitlabEventService {
}
}
public static final String GITLAB_MAIN_GROUP = "sharing";
protected static final String[] BRANCH_PRIORITY = new String[] { "main", "master" };
{
Arrays.sort(BRANCH_PRIORITY);
}
protected static final List<String> METADATA_FILENAMES = Collections.unmodifiableList(
new ArrayList<String>() {
private static final long serialVersionUID = 1L;
@@ -655,22 +739,6 @@ public class GitlabEventService {
SCHEMA_METADATA = jsonSchema;
}
@Autowired
private GitLabRepository gitLabRepository;
private GitLabApi adminGitLabApi;
@Autowired
private VocabularyService vocabularyService;
@Autowired
private MetaDataRepository metaDataRepository;
@Autowired
private MailService mailService;
private static final Logger logger = LoggerFactory.getLogger(GitlabEventService.class);
@PostConstruct
public void setAdminApi() {
adminGitLabApi = gitLabRepository.getAdminGitLabApi();
@@ -712,10 +780,6 @@ public class GitlabEventService {
}
}
static final String MISSING_METADATA_MSG =
"Metadata file is missing! Please, provide a metadata file! You may use \n" +
"either 'metadata.yml', 'metadata.yaml' or 'metadata.json'!";
protected boolean isInMainGroup(String pathWithNamespace) {
return pathWithNamespace.startsWith(GITLAB_MAIN_GROUP + "/");
}