emacs-elpa-diffs
[Top][All Lists]
Advanced

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[elpa] externals/llm 39e462d512 2/2: Fixes to llm-vertex token counting


From: ELPA Syncer
Subject: [elpa] externals/llm 39e462d512 2/2: Fixes to llm-vertex token counting
Date: Mon, 30 Oct 2023 00:58:34 -0400 (EDT)

branch: externals/llm
commit 39e462d5125e615d8bf33edae6d69b851bc2d4fd
Author: Andrew Hyatt <ahyatt@gmail.com>
Commit: Andrew Hyatt <ahyatt@gmail.com>

    Fixes to llm-vertex token counting
    
    The previous version didn't account for the fact that the request is 
basically
    just a version of the chat (or embedding) request.
---
 llm-vertex.el | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/llm-vertex.el b/llm-vertex.el
index 557507f3e3..dbd13fa144 100644
--- a/llm-vertex.el
+++ b/llm-vertex.el
@@ -313,6 +313,9 @@ If STREAMING is non-nil, use the URL for the streaming API."
                                  (llm-request-callback-in-buffer buf 
error-callback 'error
                                                                  
(llm-vertex--error-message data))))))
 
+;; Token counts
+;; https://cloud.google.com/vertex-ai/docs/generative-ai/get-token-count
+
 (defun llm-vertex--count-token-url (provider)
   "Return the URL to use for the Vertex API.
 PROVIDER is the llm provider.
@@ -321,14 +324,11 @@ MODEL "
           llm-vertex-gcloud-region
           (llm-vertex-project provider)
           llm-vertex-gcloud-region
-          (or (llm-vertex-embedding-model provider) "chat-bison")))
-
-;; Token counts
-;; https://cloud.google.com/vertex-ai/docs/generative-ai/get-token-count
+          (or (llm-vertex-chat-model provider) "chat-bison")))
 
-(defun llm-vertex--count-token-request (string)
-  "Create the data payload to count tokens in STRING."
-  `((instances . [((prompt . ,string))])))
+(defun llm-vertex--to-count-token-request (request)
+  "Return a version of REQUEST that is suitable for counting tokens."
+  (seq-filter (lambda (c) (not (equal (car c) "parameters"))) request))
 
 (defun llm-vertex--count-tokens-extract-response (response)
   "Extract the token count from the response."
@@ -339,7 +339,8 @@ MODEL "
   (llm-vertex--handle-response
    (llm-request-sync (llm-vertex--count-token-url provider)
                      :headers `(("Authorization" . ,(format "Bearer %s" 
(llm-vertex-key provider))))
-                     :data (llm-vertex--count-token-request string))
+                     :data (llm-vertex--to-count-token-request
+                            (llm-vertex--chat-request-v1 
(llm-make-simple-chat-prompt string))))
    #'llm-vertex--count-tokens-extract-response))
 
 (provide 'llm-vertex)



reply via email to

[Prev in Thread] Current Thread [Next in Thread]