branch main updated: gitlab: Add support for jobset customization in Gitlab event.

[Ludovic Courtès]

This is an automated email from the git hooks/post-receive script.

civodul pushed a commit to branch main
in repository guix-cuirass.

The following commit(s) were added to refs/heads/main by this push:
     new ca16715  gitlab: Add support for jobset customization in Gitlab event.
ca16715 is described below

commit ca167156becb6ef173a8a10656450b383f5261b9
Author: Romain GARBAGE <romain.garbage@inria.fr>
AuthorDate: Tue Jul 9 12:27:27 2024 +0200

    gitlab: Add support for jobset customization in Gitlab event.
    
    * Makefile.am: Add new test file.
    * doc/cuirass.texi: Add documentation for the "/admin/gitlab/event" 
endpoint.
    * src/cuirass/gitlab.scm (jobset-options): New record type.
    (gitlab-event)[id, options]: New fields.
    (gitlab-merge-request->specification): Add options handling.
    * tests/gitlab.scm: New file.
    * tests/http.scm: Cleanup pk call. Update tests with ID field. Prettify 
JSON strings.
    
    Signed-off-by: Ludovic Courtès <ludo@gnu.org>
---
 Makefile.am            |   1 +
 doc/cuirass.texi       |  46 +++++++++
 src/cuirass/gitlab.scm |  84 ++++++++++++++--
 tests/gitlab.scm       | 259 +++++++++++++++++++++++++++++++++++++++++++++++++
 tests/http.scm         |  30 +++++-
 5 files changed, 408 insertions(+), 12 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 4a066d3..cd8668f 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -167,6 +167,7 @@ TESTS = \
 ## tests/basic.sh # takes too long to execute
   tests/store.scm \
   tests/database.scm \
+  tests/gitlab.scm \
   tests/http.scm \
   tests/metrics.scm \
   tests/remote.scm \
diff --git a/doc/cuirass.texi b/doc/cuirass.texi
index 3f0b150..7734a2c 100644
--- a/doc/cuirass.texi
+++ b/doc/cuirass.texi
@@ -1281,6 +1281,52 @@ This request accepts a mandatory parameter.
 Limit query result to nr elements. This parameter is @emph{mandatory}.
 @end table
 
+@section Interfacing Cuirass with a GitLab Server
+
+Cuirass supports integration with GitLab through the @dfn{webhook} mechanism:
+a POST request is sent by a GitLab instance whenever a specific event is
+triggered. So far, Cuirass only support merge-request events.
+
+Sending a merge request event on the @code{/admin/gitlab/event} endpoint
+allows controlling a specific jobset related to the merge request
+content. This interface expect the JSON data to contain the following
+keys:
+@table @code
+@item "event_type"
+@item "object_attributes.action"
+@item "object_attributes.merge_status"
+@item "object_attributes.iid"
+@item "object_attributes.source_branch"
+@item "object_attributes.source.git_http_url"
+@item "object_attributes.source.name"
+@end table
+
+These keys are present in the default JSON data sent by GitLab.
+
+By default, the resulting jobset, named as
+@code{gitlab-merge-requests-@var{source}.@var{name}-@var{source_branch}-@var{iid}},
+is set to build the channel corresponding to the source branch in the
+merge request data with a priority of 1.  Tweaking of the jobset values
+can be done by creating a custom template extended with any of the
+following keys:
+@table @code
+@item "object_attributes.cuirass.name_prefix"
+A string. The resulting name will be
+@code{@var{name_prefix}-@var{source}.@var{name}-@var{source_branch}-@var{iid}}.
+@item "object_attributes.cuirass.build"
+A string or a JSON object with one key corresponding to a valid BUILD
+value and a list of strings as parameters.  See
+@code{SPECIFICATION-BUILD}.  Examples are : @code{"build": "all"} or
+@code{"build": @{ "packages": [ "hello", "emacs" ] @}}.
+@item "object_attributes.cuirass.priority"
+An integer.  See @code{SPECIFICATION-PRIORITY}.
+@item "object_attributes.cuirass.period"
+An integer.  See @code{SPECIFICATION-PERIOD}.
+@item "object_attributes.cuirass.systems"
+A JSON list of strings.  Each string must be a supported system, i.e.
+@code{"systems": [ "x86_64-linux", "aarch64-linux" ]}
+@end table
+
 @c *********************************************************************
 @node Database
 @chapter Database schema
diff --git a/src/cuirass/gitlab.scm b/src/cuirass/gitlab.scm
index 0f4a8e8..4da9e82 100644
--- a/src/cuirass/gitlab.scm
+++ b/src/cuirass/gitlab.scm
@@ -29,6 +29,7 @@
             gitlab-merge-request
             gitlab-merge-request-action
             gitlab-merge-request-project-name
+            gitlab-merge-request-cuirass-options
             json->gitlab-merge-request
             gitlab-merge-request->specification))
 
@@ -47,6 +48,42 @@
   (name     gitlab-source-name "name"
             string->symbol))
 
+;; This mapping defines a specific JSON dictionary used for tweaking Cuirass
+;; options. It is not included in the JSON data sent by default by Gitlab and
+;; must be used through the custom template mechanism (see documentation).
+(define-json-mapping <jobset-options>
+  make-jobset-options
+  jobset-options?
+  json->jobset-options
+  (name-prefix jobset-options-name-prefix "name_prefix"
+               (lambda (v)
+                 (if (unspecified? v)
+                     #f
+                     (string->symbol v))))
+  (build jobset-options-build "build"
+         (match-lambda
+           ((? unspecified?)
+            #f)
+           (((key . val) _ ...)
+            (cons (string->symbol key) (vector->list val)))
+           (str
+            (string->symbol str))))
+  (period jobset-options-period "period"
+          (lambda (v)
+            (if (unspecified? v)
+                #f
+                v)))
+  (priority jobset-options-priority "priority"
+            (lambda (v)
+              (if (unspecified? v)
+                  #f
+                  v)))
+  (systems jobset-options-systems "systems"
+           (lambda (v)
+             (if (unspecified? v)
+                 #f
+                 (vector->list v)))))
+
 (define-json-mapping <gitlab-merge-request>
   make-gitlab-merge-request
   gitlab-merge-request?
@@ -54,7 +91,13 @@
   (action          gitlab-merge-request-action "action")
   (source-branch   gitlab-merge-request-source-branch "source_branch")
   (source          gitlab-merge-request-source "source"
-                   json->gitlab-source))
+                   json->gitlab-source)
+  (id              gitlab-merge-request-id "iid")
+  (cuirass-options gitlab-merge-request-cuirass-options "cuirass"
+                   (lambda (v)
+                     (if (unspecified? v)
+                         #f
+                         (json->jobset-options v)))))
 
 (define-json-mapping <gitlab-event>
   make-gitlab-event
@@ -84,19 +127,42 @@
          (source-branch (gitlab-merge-request-source-branch merge-request))
          (source-url (gitlab-source-repo-url
                       (gitlab-merge-request-source merge-request)))
-         (spec-name (symbol-append 'gitlab-merge-requests-
-                                   source-name
-                                   '-
-                                   (string->symbol source-branch))))
+         (id (gitlab-merge-request-id merge-request))
+         (cuirass-options (gitlab-merge-request-cuirass-options merge-request))
+         (name-prefix (if (and cuirass-options
+                               (jobset-options-name-prefix cuirass-options))
+                          (jobset-options-name-prefix cuirass-options)
+                          'gitlab-merge-requests))
+         (spec-name (string->symbol
+                     (format #f "~a-~a-~a-~a" name-prefix
+                                              source-name
+                                              source-branch
+                                              id)))
+         (build (if (and cuirass-options
+                         (jobset-options-build cuirass-options))
+                    (jobset-options-build cuirass-options)
+                    `(channels ,source-name)))
+         (period (if (and cuirass-options
+                          (jobset-options-period cuirass-options))
+                     (jobset-options-period cuirass-options)
+                     3600))
+         (priority (if (and cuirass-options
+                            (jobset-options-priority cuirass-options))
+                       (jobset-options-priority cuirass-options)
+                       1))
+         (systems (if (and cuirass-options
+                           (jobset-options-systems cuirass-options))
+                      (jobset-options-systems cuirass-options)
+                      (list "x86_64-linux"))))
     (specification
      (name spec-name)
-     (build `(channels ,source-name))
+     (build build)
      (channels
       (cons* (channel
               (name source-name)
               (url source-url)
               (branch source-branch))
              %default-channels))
-     (priority 1)
-     (period 0)
-     (systems (list "x86_64-linux")))))
+     (priority priority)
+     (period period)
+     (systems systems))))
diff --git a/tests/gitlab.scm b/tests/gitlab.scm
new file mode 100644
index 0000000..b670138
--- /dev/null
+++ b/tests/gitlab.scm
@@ -0,0 +1,259 @@
+;;; gitlab.scm -- tests for (cuirass gitlab) module
+;;; Copyright © 2024 Romain GARBAGE <romain.garbage@inria.fr>
+;;;
+;;; This file is part of Cuirass.
+;;;
+;;; Cuirass is free software: you can redistribute it and/or modify
+;;; it under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation, either version 3 of the License, or
+;;; (at your option) any later version.
+;;;
+;;; Cuirass is distributed in the hope that it will be useful,
+;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with Cuirass.  If not, see <http://www.gnu.org/licenses/>.
+
+(use-modules (cuirass gitlab)
+             (cuirass specification)
+             (cuirass utils)
+             (tests common)
+             (guix channels)
+             (json)
+             (fibers)
+             (squee)
+             (web uri)
+             (web client)
+             (web response)
+             (rnrs bytevectors)
+             (srfi srfi-1)
+             (srfi srfi-64)
+             (ice-9 threads)
+             (ice-9 match))
+
+(define default-mr-json
+  "{
+    \"event_type\": \"merge_request\",
+    \"object_attributes\": {
+        \"action\": \"open\",
+        \"merge_status\": \"can_be_merged\",
+        \"iid\": \"1\",
+        \"source_branch\": \"test-branch\",
+        \"source\": {
+            \"git_http_url\": 
\"https://gitlab.instance.test/source-repo/fork-name.git\";,
+            \"name\": \"test-project\"
+        }
+    }
+  }")
+
+(define custom-mr-json
+  "{
+    \"event_type\": \"merge_request\",
+    \"object_attributes\": {
+        \"action\": \"open\",
+        \"merge_status\": \"can_be_merged\",
+        \"iid\": \"2\",
+        \"source_branch\": \"test-branch\",
+        \"source\": {
+            \"git_http_url\": 
\"https://gitlab.instance.test/source-repo/fork-name.git\";,
+            \"name\": \"test-project\"
+        },
+        \"cuirass\": {
+            \"period\": 25,
+            \"priority\": 3,
+            \"systems\": [
+                \"x86_64-linux\",
+                \"aarch64-linux\"
+            ],
+            \"build\": {
+                \"manifests\": [
+                    \"manifest\"
+                ]
+            }
+        }
+    }
+  }")
+
+(define custom-mr-json-multiple-packages
+ "{
+    \"event_type\": \"merge_request\",
+    \"object_attributes\": {
+        \"action\": \"open\",
+        \"merge_status\": \"can_be_merged\",
+        \"iid\": \"1\",
+        \"source_branch\": \"test-branch\",
+        \"source\": {
+            \"git_http_url\": 
\"https://gitlab.instance.test/source-repo/fork-name.git\";,
+            \"name\": \"test-project\"
+        },
+        \"cuirass\": {
+            \"build\": {
+                \"packages\": [
+                    \"package1\",
+                    \"package2\",
+                    \"package3\"
+                ]
+            }
+        }
+    }
+  }")
+
+(define custom-mr-json-name-prefix
+  "{
+    \"event_type\": \"merge_request\",
+    \"object_attributes\": {
+        \"action\": \"open\",
+        \"merge_status\": \"can_be_merged\",
+        \"iid\": \"1\",
+        \"source_branch\": \"test-branch\",
+        \"source\": {
+            \"git_http_url\": 
\"https://gitlab.instance.test/source-repo/fork-name.git\";,
+            \"name\": \"test-project\"
+        },
+        \"cuirass\": {
+            \"name_prefix\": \"prefix\"
+        }
+    }
+  }")
+
+(define custom-mr-json-build-all
+  "{
+    \"event_type\": \"merge_request\",
+    \"object_attributes\": {
+        \"action\": \"open\",
+        \"merge_status\": \"can_be_merged\",
+        \"iid\": \"2\",
+        \"source_branch\": \"test-branch\",
+        \"source\": {
+            \"git_http_url\": 
\"https://gitlab.instance.test/source-repo/fork-name.git\";,
+            \"name\": \"test-project\"
+        },
+        \"cuirass\": {
+            \"period\": 25,
+            \"priority\": 3,
+            \"systems\": [
+                \"x86_64-linux\",
+                \"aarch64-linux\"
+            ],
+            \"build\": \"all\"
+        }
+    }
+  }")
+
+(define (specifications-equal? spec1 spec2)
+  (and (eq? (specification-name spec1)
+            (specification-name spec2))
+       (equal? (specification-build spec1)
+               (specification-build spec2))
+       (= (specification-priority spec1)
+          (specification-priority spec2))
+       (= (specification-period spec1)
+          (specification-period spec2))
+       (equal? (specification-systems spec1)
+               (specification-systems spec2))
+       (equal? (map channel-name
+                    (specification-channels spec1))
+               (map channel-name
+                    (specification-channels spec2)))
+       (equal? (map channel-url
+                    (specification-channels spec1))
+               (map channel-url
+                    (specification-channels spec2)))
+       (equal? (map channel-branch
+                    (specification-channels spec1))
+               (map channel-branch
+                    (specification-channels spec2)))))
+
+(test-assert "default-json"
+  (specifications-equal?
+   (gitlab-merge-request->specification
+    (gitlab-event-value
+     (json->gitlab-event default-mr-json)))
+   (specification
+    (name 'gitlab-merge-requests-test-project-test-branch-1)
+    (build '(channels . (test-project)))
+    (channels
+     (cons* (channel
+             (name 'test-project)
+             (url "https://gitlab.instance.test/source-repo/fork-name.git";)
+             (branch "test-branch"))
+            %default-channels))
+    (priority 1)
+    (period 3600)
+    (systems (list "x86_64-linux")))))
+
+(test-assert "custom-json"
+  (specifications-equal?
+   (gitlab-merge-request->specification
+    (gitlab-event-value
+     (json->gitlab-event custom-mr-json)))
+   (specification
+    (name 'gitlab-merge-requests-test-project-test-branch-2)
+    (build '(manifests . ("manifest")))
+    (channels
+     (cons* (channel
+             (name 'test-project)
+             (url "https://gitlab.instance.test/source-repo/fork-name.git";)
+             (branch "test-branch"))
+            %default-channels))
+    (priority 3)
+    (period 25)
+    (systems (list "x86_64-linux"
+                   "aarch64-linux")))))
+
+(test-assert "custom-json-multiple-packages"
+  (specifications-equal?
+   (gitlab-merge-request->specification
+    (gitlab-event-value
+     (json->gitlab-event custom-mr-json-multiple-packages)))
+   (specification
+    (name 'gitlab-merge-requests-test-project-test-branch-1)
+    (build '(packages . ("package1" "package2" "package3")))
+    (channels
+     (cons* (channel
+             (name 'test-project)
+             (url "https://gitlab.instance.test/source-repo/fork-name.git";)
+             (branch "test-branch"))
+            %default-channels))
+    (priority 1)
+    (period 3600)
+    (systems (list "x86_64-linux")))))
+
+(test-assert "custom-json-name-prefix"
+  (specifications-equal?
+   (gitlab-merge-request->specification
+    (gitlab-event-value
+     (json->gitlab-event custom-mr-json-name-prefix)))
+   (specification
+    (name 'prefix-test-project-test-branch-1)
+    (build '(channels . (test-project)))
+    (channels
+     (cons* (channel
+             (name 'test-project)
+             (url "https://gitlab.instance.test/source-repo/fork-name.git";)
+             (branch "test-branch"))
+            %default-channels))
+    (priority 1)
+    (period 3600)
+    (systems (list "x86_64-linux")))))
+
+(test-assert "custom-json-build-all"
+  (specifications-equal?
+   (gitlab-merge-request->specification
+    (gitlab-event-value
+     (json->gitlab-event custom-mr-json-build-all)))
+   (specification
+    (name 'gitlab-merge-requests-test-project-test-branch-2)
+    (build 'all)
+    (channels
+     (cons* (channel
+             (name 'test-project)
+             (url "https://gitlab.instance.test/source-repo/fork-name.git";)
+             (branch "test-branch"))
+            %default-channels))
+    (priority 3)
+    (period 25)
+    (systems (list "x86_64-linux"
+                   "aarch64-linux")))))
diff --git a/tests/http.scm b/tests/http.scm
index 2b3c794..12355b2 100644
--- a/tests/http.scm
+++ b/tests/http.scm
@@ -90,15 +90,39 @@
                      (directory . "dir2")))))))
 
 (define mr-json-open
-  
"{\"event_type\":\"merge_request\",\"object_attributes\":{\"action\":\"open\",\"merge_status\":\"can_be_merged\",\"source_branch\":\"test-branch\",\"source\":{\"git_http_url\":\"https://gitlab.instance.test/source-repo/fork-name.git\",\"name\":\"test-project\"}}}";)
+  "{
+    \"event_type\": \"merge_request\",
+    \"object_attributes\": {
+        \"action\": \"open\",
+        \"merge_status\": \"can_be_merged\",
+        \"iid\": \"1\",
+        \"source_branch\": \"test-branch\",
+        \"source\": {
+            \"git_http_url\": 
\"https://gitlab.instance.test/source-repo/fork-name.git\";,
+            \"name\": \"test-project\"
+        }
+    }
+  }")
 
 (define mr-json-close
-  
"{\"event_type\":\"merge_request\",\"object_attributes\":{\"action\":\"close\",\"merge_status\":\"can_be_merged\",\"source_branch\":\"test-branch\",\"source\":{\"git_http_url\":\"https://gitlab.instance.test/source-repo/fork-name.git\",\"name\":\"test-project\"}}}";)
+  "{
+    \"event_type\": \"merge_request\",
+    \"object_attributes\": {
+        \"action\": \"close\",
+        \"merge_status\": \"can_be_merged\",
+        \"iid\": \"1\",
+        \"source_branch\": \"test-branch\",
+        \"source\": {
+            \"git_http_url\": 
\"https://gitlab.instance.test/source-repo/fork-name.git\";,
+            \"name\": \"test-project\"
+        }
+    }
+  }")
 
 (define mr-spec
   (gitlab-merge-request->specification
    (gitlab-event-value
-    (pk 'json (json->gitlab-event mr-json-open)))))
+    (json->gitlab-event mr-json-open))))
 
 (test-group-with-cleanup "http"
   (test-assert "db-init"