Intermediate commit docu and tests

src/main/docker/gitlab-setup/docker-compose.yml

+version: "3.5"
+services:
+  gitlab:
+    build: sendmail
+    image: 'gitlab_sendmail:1.0'
+    container_name: sharing_gitlab
+    restart: always
+    hostname: $GITLAB_HOSTNAME
+    networks:
+      - frontend
+      - backend
+
+    environment:
+      GITLAB_HOME: $GITLAB_HOME
+      EXTERNAL_URL: $EXTERNAL_URL
+      GITLAB_HOSTNAME: $GITLAB_HOSTNAME
+      GITLAB_API_ROOT_ACCESS_TOKEN: $GITLAB_API_ROOT_ACCESS_TOKEN
+      MAIL_USERNAME: $MAIL_USERNAME
+      MAIL_PASSWORD: $MAIL_PASSWORD
+      FILEHOOKS_CONFIG_FILE: $FILEHOOKS_CONFIG_FILE
+      GITLAB_OMNIBUS_CONFIG: |
+        external_url '$EXTERNAL_URL'
+        registry_external_url '$EXTERNAL_URL:5051'
+        gitlab_rails['gitlab_email_enabled'] = true
+        gitlab_rails['gitlab_email_from'] = 'notifications@sharing-codeability.uibk.ac.at'
+        gitlab_rails['gitlab_email_display_name'] = 'GitLab'
+        gitlab_rails['gitlab_email_reply_to'] = 'noreply@sharing-codeability.uibk.ac.at'
+        gitlab_rails['gitlab_email_subject_suffix'] = ''
+        letsencrypt['enable'] = false
+        nginx['enable'] = true
+        nginx['client_max_body_size'] = '250m'
+        nginx['redirect_http_to_https'] = false
+        nginx['ssl_ciphers'] = "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256"
+        nginx['ssl_protocols'] = "TLSv1.2 TLSv1.3"
+        nginx['ssl_session_cache'] = "builtin:1000  shared:SSL:10m"
+        nginx['referrer_policy'] = 'same-origin'
+        nginx['http2_enabled'] = true
+        nginx['custom_gitlab_server_config'] = "location /-/plantuml/ { \n    proxy_cache off; \n    proxy_pass  http://plantuml:8080/; \n}\n"
+        gitlab_rails['smtp_enable'] = true
+        gitlab_rails['smtp_address'] = "smtp.uibk.ac.at"
+        gitlab_rails['smtp_port'] = 587
+        gitlab_rails['smtp_user_name'] = '$MAIL_USERNAME'
+        gitlab_rails['smtp_password'] = '$MAIL_PASSWORD'
+        gitlab_rails['smtp_domain'] = "smtp.uibk.ac.at"
+        gitlab_rails['smtp_authentication'] = "login"
+        gitlab_rails['smtp_enable_starttls_auto'] = true
+        gitlab_rails['smtp_openssl_verify_mode'] = 'peer'
+    ports:
+      - '10082:80'
+      - '10083:443'
+      - '5051:5051'
+      - '22:22'
+    volumes:
+      - '$GITLAB_HOME/config:/etc/gitlab'
+      - '$GITLAB_HOME/logs:/var/log/gitlab'
+      - '$GITLAB_HOME/data:/var/opt/gitlab'
+      - '$GITLAB_HOME/file-hooks-src:/file-hooks-src'
+      - '$GITLAB_HOME/file_hooks:/opt/gitlab/embedded/service/gitlab-rails/file_hooks'
+      - '$GITLAB_HOME/data/backups/secrets:/secret/gitlab/backups'
+    depends_on:
+      - elasticsearch
+      - plantuml
+  elasticsearch:
+    image: 'docker.elastic.co/elasticsearch/elasticsearch:8.8.1'
+    container_name: sharing_elasticsearch
+    restart: always
+    environment:
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - node.name=es01
+      - cluster.name=sharing_elasticsearch
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+    volumes:
+      - '$ES_HOME/data:/usr/share/elasticsearch/data'
+    ports:  # add this for your local testing setup
+        - '9200:9200' # add this for your local testing setup
+        - '9300:9300' # add this for your local testing setup
+    networks:
+       - backend
+#      - frontend # add this for your local testing setup
+
+  plantuml:
+    image: 'plantuml/plantuml-server:tomcat-v1.2023.1'
+    container_name: sharing_plantuml
+    restart: always
+    networks:
+      - backend
+
+networks:
+  frontend:
+    name: sharing_frontend
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.22.1.0/24
+  backend:
+    name: sharing_backend
+    driver: bridge
+    internal: true
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.22.2.0/24

src/main/docker/gitlab-setup/sendmail/Dockerfile

+FROM gitlab/gitlab-ce:14.10.3-ce.0
+
+RUN apt-get clean \
+    && apt-get update \
+	&& apt-get install -y mailutils sendmail perl openjdk-8-jdk \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& sed -i '/echo "Starting services..."/a service sendmail start' /assets/wrapper
+RUN pip3 install --upgrade setuptools

src/main/docs/source/filehooks_infrastructure.rst

@@ -6,7 +6,6 @@ FileHooks
 The fileHooks project is a simple infrastructure for forwarding 
 events from GitLab to the GitSearch REST service at http://sharing_search:8080/api/gitlab/eventListener.
 
- comprises the backend of the CodeAbility Sharing Platform.
 The services GitLab and Elasticsearch are considered backend services.
 It is responsible for the data collection and preparation.
 This section describes the fileHooks used in GitLab and the infrastructure setup.
@@ -16,31 +15,37 @@ Finally, some tips to handle errors are provided.
 GitSearch Indexer
 =================
 
-Currently, there is one fileHook for GitLab - doing both health check and indexing -  to extend the functionality of GitLab.
-Details are provided in the following.
+.. _ref_gitsearch_indexer:
 
-FileHook - trigger_project_update
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The GitSearch Indexer listens to requests via the REST service at http://sharing_search:8080/api/gitlab/eventListener.
+It is responsible for validating and updating the Elasticsearch index.
 
-This file hook does two tasks:
+
+This GitSearch Indexer does two tasks:
 
 1. Health check and validation: It informs the user who modified the project via email if the metadata information is incomplete or invalid after a modification in a repository was conducted.
-Validation happens on the ``master``-branch of all projects in the group ``sharing``.
-It will mainly be triggered by push events, but also by moving or renaming a project.
+Validation happens on the ``master``-branch of all projects in the group ``sharing``. It also checks projects in all other groups, however if they do not contain meta data, the check is skipped.  
+
+The indexer will mainly be triggered by push events, but also by moving or renaming a project and or groups/namespaces.
 The check proceeds as follows:
 First, the root directory of the repository is checked for files named ``metadata.json``, ``metadata.yaml``, or ``metadata.yml``.
 There must be exactly one such file, otherwise the check fails.
 Subsequently, the correctness of all metadata files is validated (also dependent metadata files, if it is a collection).
 If an error occurred, an email is sent to the user who pushed the changes.
 
+Meta data checks comprise:
+ - the syntactical correctness of the metadata file as yaml or json file (results in an error)
+ - the presence of the required fields (results in an error)
+ - the presence of the required fields in the dependent metadata files (results in an error)
+ - checks against the vocabulary service at https://oeresource.logic.at/en/meta/api/v1?format=json  (results in a warning)
+
+The check fails if there is an error, but is accepted if there are only warnings. In both cases the author is informed by e-mail.
+
 2. It keeps the Elasticsearch index up-to-date by adding/updating/deleting files according to the triggered GitLab event.
 Only the ``main``-branch (or ``master`` if ``main`` does not exist) and the group ``sharing`` (including subgroups and all subprojects) are indexed in Elasticsearch.
 Metadata files (``metadata.json``, ``metadata.yaml``, or ``metadata.yml``) at the project root are indexed in the alias ``metadata``.
 
-
-..  warning::
-
-    Note that GitLab does not trigger an event if a group is transferred! Those changes remain unnoticed in Elasticsearch! To prevent those inconsistencies, users should not transfer groups!
+Finally, the GitSearch Indexer provides functionality, to recreate the index and to recheck all projects. During this task all event-processing is paused.
 
 
 Infrastructure Setup
@@ -54,42 +59,22 @@ Lastly, the manual installation procedure for file hooks is given as a reference
 Container Setup
 ~~~~~~~~~~~~~~~
 
-Subsequently, the setup for GitLab and Elasticsearch is shown.
-The setup of the Services GitLab search and MySQL are discussed in the section :ref:`ref_git_search`.
-
-To create all containers for the backend in production, a script situated in ``setup/`` is provided.
-It takes a configuration file as the only argument.
-The configuration files can be found in ``setup/config/``.
-For a local development setup, the file ``local`` can be used without further modification.
-For deployment on a server, the files ``development_template`` and ``production_template`` are provided.
-These configurations require secrets.
-Do not put secrets into these files but create a copy and put them into the copy.
-Files in the ``setup/config/`` directory are ignored by git by default, so writing secrets into a copy
-prevents accidentally committing them.
-
-The secrets which need to be added to the copy of the configuration file are:
+Subsequently, the setup for GitLab, PlantUml Elasticsearch is shown.
 
-- MAIL_USERNAME: The email user name to authenticate with at ``smtp.uibk.ac.at`` (KeePass @ artemis-support MailBox).
-- MAIL_PASSWORD The password for the authentication at ``smtp.uibk.ac.at`` (KeePass @ artemis-support MailBox).
-
-The following code block shows how to deploy the project in production.
-
-.. code-block::
+The setup of the Services GitLab search and MySQL are discussed in the section :ref:`ref_git_search`.
 
-    cd setup
-    cp config/production_template config/production
-    $EDITOR config/production  # add the secrets to the copy
-    ./setup-infrastructure.sh config/production
 
+To create all containers for the backend in production, a docker-compose script is provided in ``src/main/docker/gitlab-setup/``.
 
-Similarly, the containers for the development backend can be created with:
+It can be executed as follows:
+  - ``cd src/main/docker/gitlab-setup/``
+  - ``. .env`` the .env file can be found in the KeePass file. It contains the secrets for the containers. (Remark: the entry for the FILEHOOKS_CONFIG_FILE is missing.)
+  - ``docker-compose create``
+  - ``docker-compose start``    
 
-.. code-block::
+Files in the ``setup/config/`` directory are ignored by git by default, so writing secrets into a copy
+prevents accidentally committing them.
 
-    cd setup
-    cp config/development_template config/development
-    $EDITOR config/development  # add the secrets to the copy
-    ./setup-infrastructure.sh config/development
 
 The following environment variables are set within the config files.
 No modification should be required for those if the correct config file is used.
@@ -113,6 +98,8 @@ No modification should be required for those if the correct config file is used.
 +----------------------+----------------------------------------+------------------------------------------------+
 | GITLAB_HOSTNAME      | conf/production.ini                    | conf/staging.ini                               |
 +----------------------+----------------------------------------+------------------------------------------------+
+| FILEHOOKS_CONFIG_FILE| TODO                                   | TODO                               |
++----------------------+----------------------------------------+------------------------------------------------+
 
 .. note::
 

src/main/docs/source/infrastructure.rst

@@ -17,8 +17,7 @@ In the following, a brief description of each component is given.
 - GitSearch Application: Frontend of the CodeAbility's Sharing Platform metadata search. 
   It is implemented as a `JHipster <https://jhipster.tech>`_ application and provides a REST API for querying the metadata.
   Additionally it handles GitLab's system_hooks events and processes potential changes in meta data.
-  It validates and indexes the detected meta data. Further details can be found in the section :ref:`ref_filehooks`
- Further details can be found in the section :ref:`ref_git_search`.
+  It validates and indexes the detected meta data. Further details can be found in the sections :ref:`ref_filehooks` and :ref:`_ref_gitsearch_indexer`.	
 - MySQL: Database for the frontend to store various application data e.g., user data.
 - Elasticsearch: `Elasticsearch <https://www.elastic.co/elasticsearch/>`_ instance providing a search engine.
 
@@ -49,3 +48,4 @@ is running
 +-----------------+-----------------------+-------------------------------------------+
 | MySQL           | sharing_my_sql        |                                           |
 +-----------------+-----------------------+-------------------------------------------+
+   
\ No newline at end of file

src/test/java/at/ac/uibk/gitsearch/service/gitlab_events/GitlabEventServiceIT.java

@@ -111,7 +111,7 @@ class GitlabEventServiceIT {
         logger.info("finished handleGitLabEvent for {}", collectionCommitEvent);
 
         // the example is correct :-)
-        verify(mailService, Mockito.never())
+        verify(mailService, Mockito.times(1))
             .sendEmail(
                 Mockito.eq("Michael.Breu@uibk.ac.at"),
                 Mockito.anyString(),