branch master updated: Tweak fetching narinfos

[Christopher Baines]

This is an automated email from the git hooks/post-receive script.

cbaines pushed a commit to branch master
in repository data-service.

The following commit(s) were added to refs/heads/master by this push:
     new 8a19bcb  Tweak fetching narinfos
8a19bcb is described below

commit 8a19bcba1398a719fefce0cb2bcf2676ef775606
Author: Christopher Baines <mail@cbaines.net>
AuthorDate: Fri Apr 28 20:42:31 2023 +0200

    Tweak fetching narinfos
    
    Currently I'm seeing failures due to guile-gnutls not supporting suspendable
    ports (write_wait_fd), so batch the requested outputs to try and avoid this.
---
 guix-data-service/substitutes.scm | 54 ++++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/guix-data-service/substitutes.scm 
b/guix-data-service/substitutes.scm
index 87b6bbb..3867568 100644
--- a/guix-data-service/substitutes.scm
+++ b/guix-data-service/substitutes.scm
@@ -22,6 +22,7 @@
   #:use-module (ice-9 threads)
   #:use-module (guix substitutes)
   #:use-module (guix narinfo)
+  #:use-module (guix-data-service utils)
   #:use-module (guix-data-service database)
   #:use-module (guix-data-service model build-server)
   #:use-module (guix-data-service model git-branch)
@@ -85,34 +86,39 @@
   (simple-format #t "Querying ~A outputs\n"
                  (length outputs))
 
-  (let ((narinfos
-         (lookup-narinfos (string-trim-right build-server-url #\/) outputs)))
+  (chunk-for-each!
+   (lambda (outputs-chunk)
+     (let ((narinfos
+            (lookup-narinfos (string-trim-right build-server-url #\/)
+                             outputs-chunk)))
 
-    (simple-format #t "Got ~A narinfo files\n"
-                   (length narinfos))
+       (simple-format #t "Got ~A narinfo files\n"
+                      (length narinfos))
 
-    (let ((filtered-narinfos
-           (filter-map
-            (lambda (narinfo)
-              (if (> (narinfo-size narinfo)
-                     %narinfo-max-size)
-                  (begin
-                    (simple-format (current-error-port)
-                                   "narinfo ~A has excessive size ~A\n"
-                                   (narinfo-path narinfo)
-                                   (narinfo-size narinfo))
-                    #f)
-                  narinfo))
-            narinfos)))
+       (let ((filtered-narinfos
+              (filter-map
+               (lambda (narinfo)
+                 (if (> (narinfo-size narinfo)
+                        %narinfo-max-size)
+                     (begin
+                       (simple-format (current-error-port)
+                                      "narinfo ~A has excessive size ~A\n"
+                                      (narinfo-path narinfo)
+                                      (narinfo-size narinfo))
+                       #f)
+                     narinfo))
+               narinfos)))
 
-      (unless (null? filtered-narinfos)
-        (with-postgresql-transaction
-         conn
-         (lambda (conn)
-           (record-narinfo-details-and-return-ids
+         (unless (null? filtered-narinfos)
+           (with-postgresql-transaction
             conn
-            build-server-id
-            filtered-narinfos)))))))
+            (lambda (conn)
+              (record-narinfo-details-and-return-ids
+               conn
+               build-server-id
+               filtered-narinfos)))))))
+   2000
+   outputs))
 
 (define (start-substitute-query-thread)
   (call-with-new-thread