From 368d0c8a9ebae16a20e1c8971b21ee888bdefad5 Mon Sep 17 00:00:00 2001
From: tjohnman <tjohnman@users.noreply.github.com>
Date: Sun, 19 Mar 2023 19:31:17 +0100
Subject: [PATCH] Respect the maximum number of tokens in interactive. (#298)

Co-authored-by: Johnman <johnman@github>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
---
 main.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/main.cpp b/main.cpp
index e181056..57e9249 100644
--- a/main.cpp
+++ b/main.cpp
@@ -1062,7 +1062,6 @@ int main(int argc, char ** argv) {
         }
 
         // end of text token
-
         if (embd.back() == EOS_TOKEN_ID) {
             if (params.interactive) {
                 is_interacting = true;
@@ -1071,6 +1070,12 @@ int main(int argc, char ** argv) {
                 break;
             }
         }
+
+        // In interactive mode, respect the maximum number of tokens and drop back to user input when reached.
+        if (params.interactive && remaining_tokens <= 0) {
+            remaining_tokens = params.n_predict;
+            is_interacting = true;
+        }
     }
 
 #if defined (_WIN32)