02/03: Improve chunking when inserting derivation inputs

[Christopher Baines]

cbaines pushed a commit to branch master
in repository data-service.

commit 78a5abee21d9e507fbacf443c411cfe095889b03
Author: Christopher Baines <mail@cbaines.net>
AuthorDate: Sat Sep 17 00:41:14 2022 +0200

    Improve chunking when inserting derivation inputs
    
    Chunk the values inserted in the query, rather than the derivations 
involved,
    as this is more consistent.
---
 guix-data-service/model/derivation.scm | 73 ++++++++++++++++------------------
 1 file changed, 35 insertions(+), 38 deletions(-)

diff --git a/guix-data-service/model/derivation.scm 
b/guix-data-service/model/derivation.scm
index f037b50..7a4b214 100644
--- a/guix-data-service/model/derivation.scm
+++ b/guix-data-service/model/derivation.scm
@@ -1381,51 +1381,48 @@ WHERE derivation_source_files.store_path = $1"
         #f)))
 
 (define (insert-derivation-inputs conn derivation-ids derivations)
-  (define (process-chunk derivation-ids derivations)
-    (let ((query-parts
-           (append-map!
-            (lambda (derivation-id derivation)
-              (append-map!
-               (match-lambda
-                 (($ <derivation-input> derivation-or-path sub-derivations)
-                  (let ((path
-                         (match derivation-or-path
-                           ((? derivation? d)
-                            ;; The first field changed to a derivation (from 
the file
-                            ;; name) in 
5cf4b26d52bcea382d98fb4becce89be9ee37b55
-                            (derivation-file-name d))
-                           ((? string? s)
-                            s))))
-                    (map (lambda (sub-derivation)
-                           (string-append "("
-                                          (number->string derivation-id)
-                                          ", '" path
-                                          "', '" sub-derivation "')"))
-                         sub-derivations))))
-               (derivation-inputs derivation)))
-            derivation-ids
-            derivations)))
-
-      (unless (null? query-parts)
-        (exec-query
-         conn
-         (string-append
-          "
+  (let ((query-parts
+         (append-map!
+          (lambda (derivation-id derivation)
+            (append-map!
+             (match-lambda
+               (($ <derivation-input> derivation-or-path sub-derivations)
+                (let ((path
+                       (match derivation-or-path
+                         ((? derivation? d)
+                          ;; The first field changed to a derivation (from the 
file
+                          ;; name) in 5cf4b26d52bcea382d98fb4becce89be9ee37b55
+                          (derivation-file-name d))
+                         ((? string? s)
+                          s))))
+                  (map (lambda (sub-derivation)
+                         (string-append "("
+                                        (number->string derivation-id)
+                                        ", '" path
+                                        "', '" sub-derivation "')"))
+                       sub-derivations))))
+             (derivation-inputs derivation)))
+          derivation-ids
+          derivations)))
+
+    (chunk-for-each!
+     (lambda (query-parts-chunk)
+       (exec-query
+        conn
+        (string-append
+         "
 INSERT INTO derivation_inputs (derivation_id, derivation_output_id)
 SELECT vals.derivation_id, derivation_outputs.id
 FROM (VALUES "
-          (string-join query-parts ", ")
-          ") AS vals (derivation_id, file_name, output_name)
+         (string-join query-parts-chunk ", ")
+         ") AS vals (derivation_id, file_name, output_name)
 INNER JOIN derivations
   ON derivations.file_name = vals.file_name
 INNER JOIN derivation_outputs
   ON derivation_outputs.derivation_id = derivations.id
- AND vals.output_name = derivation_outputs.name")))))
-
-  (chunk-for-each! process-chunk
-                   200
-                   (list-copy derivation-ids)
-                   (list-copy derivations)))
+ AND vals.output_name = derivation_outputs.name")))
+     1000
+     query-parts)))
 
 (define (select-from-derivation-source-files store-paths)
   (string-append