diff --git a/integration/generate_jinja_test.go b/integration/generate_jinja_test.go new file mode 100644 index 00000000..4234e4f1 --- /dev/null +++ b/integration/generate_jinja_test.go @@ -0,0 +1,260 @@ +//go:build integration && generate + +package integration + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/ollama/ollama/api" +) + +func TestGenerateNativeJinjaDebugRender(t *testing.T) { + if os.Getenv("OLLAMA_TEST_EXISTING") != "" { + t.Skip("requires a test-managed server started with OLLAMA_GO_TEMPLATE=0") + } + t.Setenv("OLLAMA_GO_TEMPLATE", "0") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + client, _, cleanup := InitServerConnection(ctx, t) + defer cleanup() + pullOrSkip(ctx, t, client, smol) + + modelName := createGenerateJinjaModel(ctx, t, client, smol) + defer func() { + if err := client.Delete(context.Background(), &api.DeleteRequest{Model: modelName}); err != nil { + t.Logf("failed to delete %s: %v", modelName, err) + } + }() + + t.Run("generate matches chat", func(t *testing.T) { + prompt := "Reply with one short sentence about blue skies." + generateRender := debugGenerateRender(ctx, t, client, api.GenerateRequest{ + Model: modelName, + Prompt: prompt, + }) + chatRender := debugChatRender(ctx, t, client, api.ChatRequest{ + Model: modelName, + Messages: []api.Message{ + {Role: "user", Content: prompt}, + }, + }) + + if generateRender == "" { + t.Fatal("generate debug render was empty") + } + if strings.Contains(generateRender, "GO_TEMPLATE_SENTINEL") { + t.Fatalf("generate used Go TEMPLATE instead of native chat template: %q", generateRender) + } + if generateRender != chatRender { + t.Fatalf("generate render does not match chat render\ngenerate: %q\nchat: %q", generateRender, chatRender) + } + }) + + t.Run("context prepends prior tokens", func(t *testing.T) { + first := generateOnce(ctx, t, client, api.GenerateRequest{ + Model: modelName, + Prompt: "Say hello.", + Options: map[string]any{ + "num_predict": 1, + "temperature": 0, + }, + }) + if len(first.Context) == 0 { + t.Fatal("generate response did not include context") + } + + next := api.GenerateRequest{ + Model: modelName, + Prompt: "Say goodbye.", + } + withoutContext := debugGenerateRender(ctx, t, client, next) + next.Context = first.Context + withContext := debugGenerateRender(ctx, t, client, next) + + if withContext == withoutContext { + t.Fatal("context debug render did not change the prompt") + } + if !strings.HasSuffix(withContext, withoutContext) { + t.Fatalf("context render should preserve current native template as suffix\nwith context: %q\nwithout: %q", withContext, withoutContext) + } + }) + + t.Run("images use completion media markers", func(t *testing.T) { + resp := debugGenerate(ctx, t, client, api.GenerateRequest{ + Model: modelName, + Prompt: "compare [img] and [img]", + Images: []api.ImageData{ + []byte("first-test-image"), + []byte("second-test-image"), + }, + }) + + if resp.DebugInfo == nil { + t.Fatal("missing debug info") + } + if resp.DebugInfo.ImageCount != 2 { + t.Fatalf("image count = %d, want 2", resp.DebugInfo.ImageCount) + } + rendered := resp.DebugInfo.RenderedTemplate + if !strings.Contains(rendered, "[img-0]") || !strings.Contains(rendered, "[img-1]") { + t.Fatalf("rendered template missing image markers: %q", rendered) + } + }) +} + +func TestGenerateNativeJinjaImages(t *testing.T) { + if os.Getenv("OLLAMA_TEST_EXISTING") != "" { + t.Skip("requires a test-managed server started with OLLAMA_GO_TEMPLATE=0") + } + t.Setenv("OLLAMA_GO_TEMPLATE", "0") + skipUnderMinVRAM(t, 6) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + client, _, cleanup := InitServerConnection(ctx, t) + defer cleanup() + _, _, ollamaHome := decodeTestImages(t) + + t.Run("actual image processing", func(t *testing.T) { + for _, baseModel := range testModels([]string{"gemma3:4b"}) { + t.Run(baseModel, func(t *testing.T) { + setupVisionModel(ctx, t, client, baseModel) + + modelName := createGenerateJinjaModel(ctx, t, client, baseModel) + defer func() { + if err := client.Delete(context.Background(), &api.DeleteRequest{Model: modelName}); err != nil { + t.Logf("failed to delete %s: %v", modelName, err) + } + }() + + req := api.GenerateRequest{ + Model: modelName, + Prompt: "Describe what you see in this image briefly.", + Images: []api.ImageData{ollamaHome}, + Options: map[string]any{ + "seed": 42, + "temperature": 0.0, + }, + KeepAlive: &api.Duration{Duration: 10 * time.Second}, + } + + resp := debugGenerate(ctx, t, client, req) + if resp.DebugInfo == nil { + t.Fatal("missing debug info") + } + if resp.DebugInfo.ImageCount != 1 { + t.Fatalf("image count = %d, want 1", resp.DebugInfo.ImageCount) + } + rendered := resp.DebugInfo.RenderedTemplate + if strings.Contains(rendered, "GO_TEMPLATE_SENTINEL") { + t.Fatalf("generate with images used Go TEMPLATE instead of native chat template: %q", rendered) + } + if !strings.Contains(rendered, "[img-0]") { + t.Fatalf("rendered template missing image marker: %q", rendered) + } + + DoGenerate(ctx, t, client, req, []string{ + "llama", "animal", "build", "model", "open", "cartoon", "character", + }, 120*time.Second, 30*time.Second) + }) + } + }) +} + +func createGenerateJinjaModel(ctx context.Context, t *testing.T, client *api.Client, baseModel string) string { + t.Helper() + + modelName := fmt.Sprintf("test-generate-jinja-%d", time.Now().UnixNano()) + stream := false + err := client.Create(ctx, &api.CreateRequest{ + Model: modelName, + From: baseModel, + Stream: &stream, + Template: `GO_TEMPLATE_SENTINEL {{ if .System }}{{ .System }} {{ end }}{{ if .Prompt }}{{ .Prompt }}{{ end }}{{ range .Messages }}{{ .Content }}{{ end }}`, + }, func(api.ProgressResponse) error { + return nil + }) + if err != nil { + t.Fatalf("create %s from %s: %v", modelName, baseModel, err) + } + return modelName +} + +func debugGenerateRender(ctx context.Context, t *testing.T, client *api.Client, req api.GenerateRequest) string { + t.Helper() + resp := debugGenerate(ctx, t, client, req) + if resp.DebugInfo == nil { + t.Fatal("missing debug info") + } + return resp.DebugInfo.RenderedTemplate +} + +func debugGenerate(ctx context.Context, t *testing.T, client *api.Client, req api.GenerateRequest) api.GenerateResponse { + t.Helper() + + stream := false + req.Stream = &stream + req.DebugRenderOnly = true + + var final api.GenerateResponse + if err := client.Generate(ctx, &req, func(resp api.GenerateResponse) error { + final = resp + return nil + }); err != nil { + if strings.Contains(err.Error(), "requires more system memory") { + t.Skipf("model is too large for this test system: %v", err) + } + t.Fatalf("generate debug render failed: %v", err) + } + return final +} + +func debugChatRender(ctx context.Context, t *testing.T, client *api.Client, req api.ChatRequest) string { + t.Helper() + + stream := false + req.Stream = &stream + req.DebugRenderOnly = true + + var final api.ChatResponse + if err := client.Chat(ctx, &req, func(resp api.ChatResponse) error { + final = resp + return nil + }); err != nil { + if strings.Contains(err.Error(), "requires more system memory") { + t.Skipf("model is too large for this test system: %v", err) + } + t.Fatalf("chat debug render failed: %v", err) + } + if final.DebugInfo == nil { + t.Fatal("missing debug info") + } + return final.DebugInfo.RenderedTemplate +} + +func generateOnce(ctx context.Context, t *testing.T, client *api.Client, req api.GenerateRequest) api.GenerateResponse { + t.Helper() + + stream := false + req.Stream = &stream + + var final api.GenerateResponse + if err := client.Generate(ctx, &req, func(resp api.GenerateResponse) error { + final = resp + return nil + }); err != nil { + if strings.Contains(err.Error(), "requires more system memory") { + t.Skipf("model is too large for this test system: %v", err) + } + t.Fatalf("generate failed: %v", err) + } + return final +} diff --git a/server/images.go b/server/images.go index 50342035..b6ce6a77 100644 --- a/server/images.go +++ b/server/images.go @@ -272,11 +272,30 @@ func hasMoreCapabilities(candidate, current []model.Capability) bool { return len(candidate) > len(current) } -func shouldPreferChatTemplate(chatTemplate string, chatTemplateCaps []model.Capability, goTemplate *template.Template, goTemplateCaps []model.Capability) bool { - if !hasMoreCapabilities(chatTemplateCaps, goTemplateCaps) { +func sameCapabilities(candidate, current []model.Capability) bool { + if len(candidate) != len(current) { return false } - return !goTemplateHasToolRoundTrip(goTemplate) || chatTemplateHasToolRoundTrip(chatTemplate) + for _, c := range candidate { + if !slices.Contains(current, c) { + return false + } + } + return true +} + +func shouldPreferChatTemplate(chatTemplate string, chatTemplateCaps []model.Capability, goTemplate *template.Template, goTemplateCaps []model.Capability) bool { + if hasMoreCapabilities(chatTemplateCaps, goTemplateCaps) { + return !goTemplateHasToolRoundTrip(goTemplate) || chatTemplateHasToolRoundTrip(chatTemplate) + } + + if !sameCapabilities(chatTemplateCaps, goTemplateCaps) || + !slices.Contains(chatTemplateCaps, model.CapabilityTools) || + !slices.Contains(goTemplateCaps, model.CapabilityTools) { + return false + } + + return chatTemplateHasToolRoundTrip(chatTemplate) && !goTemplateHasToolRoundTrip(goTemplate) } func goTemplateEnvSet() bool { diff --git a/server/images_test.go b/server/images_test.go index 2aaf217f..3cc462a8 100644 --- a/server/images_test.go +++ b/server/images_test.go @@ -132,6 +132,37 @@ func TestGetModelTemplateMetadata(t *testing.T) { } }) + t.Run("prefers chat template with stronger tool round trip", func(t *testing.T) { + t.Setenv("OLLAMA_MODELS", t.TempDir()) + t.Setenv("OLLAMA_GO_TEMPLATE", "") + + _, digest := createBinFile(t, ggml.KV{ + "general.architecture": "llama", + "tokenizer.chat_template": `{% if tools %}{{ tools }}{% endif %} +{% for message in messages %} +{% if message.tool_calls %} +{% for tool_call in message.tool_calls %}{{ tool_call.function.name }}{% endfor %} +{% endif %} +{% if message.role == 'tool' %}tool_response {{ message.content }}{% endif %} +{% endfor %}`, + }, nil) + writeTestModelManifest(t, "chat-template-tool-round-trip", digest, `{{ if .Tools }}tools{{ end }} +{{ range .Messages }} +{{ range .ToolCalls }}{{ .Function.Name }}{{ end }} +{{ end }}`) + + m, err := GetModel("chat-template-tool-round-trip") + if err != nil { + t.Fatal(err) + } + if !m.PreferChatTemplate { + t.Fatal("expected chat template to be preferred") + } + if got := m.CheckCapabilities(model.CapabilityTools); got != nil { + t.Fatalf("expected tools capability, got %v", got) + } + }) + t.Run("keeps Go TEMPLATE when chat template has weaker tool support", func(t *testing.T) { t.Setenv("OLLAMA_MODELS", t.TempDir()) t.Setenv("OLLAMA_GO_TEMPLATE", "") diff --git a/server/prompt.go b/server/prompt.go index 68e5a8db..17b50efc 100644 --- a/server/prompt.go +++ b/server/prompt.go @@ -77,11 +77,27 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api. slog.Debug("truncating input messages which exceed context length", "truncated", len(msgs[currMsgIdx:])) } - renderMsgs := slices.Clone(msgs) + renderMsgs, media, err := imageTaggedMessages(m, msgs, currMsgIdx, false) + if err != nil { + return "", nil, err + } - for cnt, msg := range renderMsgs[currMsgIdx:] { + // truncate any messages that do not fit into the context window + p, err := renderPrompt(m, append(system, renderMsgs[currMsgIdx:]...), tools, think) + if err != nil { + return "", nil, err + } + + return p, media, nil +} + +func imageTaggedMessages(m *Model, msgs []api.Message, start int, clearImages bool) ([]api.Message, []llm.MediaData, error) { + renderMsgs := slices.Clone(msgs) + var media []llm.MediaData + + for cnt, msg := range renderMsgs[start:] { if slices.Contains(m.Config.ModelFamilies, "mllama") && len(msg.Images) > 1 { - return "", nil, errors.New("this model only supports one image while more than one image requested") + return nil, nil, errors.New("this model only supports one image while more than one image requested") } var prefix string @@ -105,20 +121,15 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api. } } - if m.Config.Renderer != "" { - continue + if m.Config.Renderer == "" { + renderMsgs[start+cnt].Content = prefix + prompt + } + if clearImages { + renderMsgs[start+cnt].Images = nil } - - renderMsgs[currMsgIdx+cnt].Content = prefix + prompt } - // truncate any messages that do not fit into the context window - p, err := renderPrompt(m, append(system, renderMsgs[currMsgIdx:]...), tools, think) - if err != nil { - return "", nil, err - } - - return p, media, nil + return renderMsgs, media, nil } func renderPrompt(m *Model, msgs []api.Message, tools []api.Tool, think *api.ThinkValue) (string, error) { diff --git a/server/routes.go b/server/routes.go index f8d06b76..432dc408 100644 --- a/server/routes.go +++ b/server/routes.go @@ -567,17 +567,59 @@ func (s *Server) GenerateHandler(c *gin.Context) { // support for generate if values.Messages != nil && values.Suffix == "" && req.Template == "" { genTruncate := (req.Truncate == nil || *req.Truncate) && !m.IsMLX() - prompt, media, err = chatPrompt(c.Request.Context(), m, r.Tokenize, optionsForPrompt(opts, r), values.Messages, []api.Tool{}, req.Think, genTruncate) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return + if m.HasChatTemplate && chatModeForModel(m) == chatExecutionModeNative { + nativeReq, err := prepareNativeChatRequest(c.Request.Context(), m, r, opts, llm.ChatRequest{ + Messages: values.Messages, + Format: req.Format, + Options: opts, + Think: req.Think, + Shift: req.Shift == nil || *req.Shift, + Logprobs: req.Logprobs, + TopLogprobs: req.TopLogprobs, + }, genTruncate) + if err != nil { + slog.Error("chat template prompt error", "error", err) + var serr api.StatusError + if errors.As(err, &serr) { + c.JSON(serr.StatusCode, gin.H{"error": serr.ErrorMessage}) + } else { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + } + return + } + nativeReq.Messages, media, err = imageTaggedMessages(m, nativeReq.Messages, 0, true) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + prompt, err = r.ApplyChatTemplate(c.Request.Context(), nativeReq) + if err != nil { + var serr api.StatusError + if errors.As(err, &serr) { + c.JSON(serr.StatusCode, gin.H{"error": serr.ErrorMessage}) + } else { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + } + return + } + // TEMP(drifkin): req.Context will be removed very soon, but we're temporarily supporting it in this flow here + if req.Context != nil { + b.WriteString(prompt) + prompt = b.String() + } + } else { + prompt, media, err = chatPrompt(c.Request.Context(), m, r.Tokenize, optionsForPrompt(opts, r), values.Messages, []api.Tool{}, req.Think, genTruncate) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + // TEMP(drifkin): req.Context will be removed very soon, but we're temporarily supporting it in this flow here + if req.Context != nil { + b.WriteString(prompt) + prompt = b.String() + } + leadingBOS = leadingBOSForModel(m) } - // TEMP(drifkin): req.Context will be removed very soon, but we're temporarily supporting it in this flow here - if req.Context != nil { - b.WriteString(prompt) - prompt = b.String() - } - leadingBOS = leadingBOSForModel(m) } else { // Direct template execution flow. if err := tmpl.Execute(&b, values); err != nil { @@ -2917,8 +2959,15 @@ func (s *Server) ChatHandler(c *gin.Context) { writeChatResponse(c, req, ch) } +func prepareNativeChatRequest(ctx context.Context, m *Model, r llm.LlamaServer, opts *api.Options, nativeReq llm.ChatRequest, truncate bool) (llm.ChatRequest, error) { + var err error + nativeReq.Messages, err = truncateNativeChatMessages(ctx, m, r, optionsForPrompt(opts, r), nativeReq, truncate) + return nativeReq, err +} + func (s *Server) handleNativeChat(c *gin.Context, req api.ChatRequest, m *Model, r llm.LlamaServer, opts *api.Options, msgs []api.Message, checkpointStart, checkpointLoaded time.Time) { - nativeReq := llm.ChatRequest{ + truncate := req.Truncate == nil || *req.Truncate + nativeReq, err := prepareNativeChatRequest(c.Request.Context(), m, r, opts, llm.ChatRequest{ Messages: msgs, Tools: req.Tools, Format: req.Format, @@ -2927,10 +2976,7 @@ func (s *Server) handleNativeChat(c *gin.Context, req api.ChatRequest, m *Model, Shift: req.Shift == nil || *req.Shift, Logprobs: req.Logprobs, TopLogprobs: req.TopLogprobs, - } - truncate := req.Truncate == nil || *req.Truncate - var err error - nativeReq.Messages, err = truncateNativeChatMessages(c.Request.Context(), m, r, optionsForPrompt(opts, r), nativeReq, truncate) + }, truncate) if err != nil { slog.Error("chat template prompt error", "error", err) var serr api.StatusError diff --git a/server/routes_generate_test.go b/server/routes_generate_test.go index 2727a490..9dd5f186 100644 --- a/server/routes_generate_test.go +++ b/server/routes_generate_test.go @@ -62,6 +62,7 @@ type mockRunner struct { ChatFn func(context.Context, llm.ChatRequest, func(llm.ChatResponse)) error Template string TemplateFn func(context.Context, llm.ChatRequest) (string, error) + DetokenizeFn func(context.Context, []int) (string, error) contextLength int } @@ -91,6 +92,13 @@ func (m *mockRunner) ApplyChatTemplate(ctx context.Context, r llm.ChatRequest) ( return m.Template, nil } +func (m *mockRunner) Detokenize(ctx context.Context, tokens []int) (string, error) { + if m.DetokenizeFn != nil { + return m.DetokenizeFn(ctx, tokens) + } + return "", nil +} + func (mockRunner) Tokenize(_ context.Context, s string) (tokens []int, err error) { for range strings.Fields(s) { tokens = append(tokens, len(tokens)) @@ -465,6 +473,200 @@ func TestChatHandlerHarmonyPreservesStructuralTokens(t *testing.T) { } } +func TestGenerateHandlerChatTemplateRoute(t *testing.T) { + t.Setenv("OLLAMA_CONTEXT_LENGTH", "4096") + t.Setenv("OLLAMA_GO_TEMPLATE", "") + gin.SetMode(gin.TestMode) + + t.Run("uses GGUF chat_template when no Go TEMPLATE exists", func(t *testing.T) { + mock := mockRunner{ + TemplateFn: func(_ context.Context, req llm.ChatRequest) (string, error) { + if len(req.Messages) != 1 || req.Messages[0].Role != "user" || req.Messages[0].Content != "hello" { + t.Fatalf("chat template messages = %#v", req.Messages) + } + return "native-template: hello", nil + }, + CompletionResponse: llm.CompletionResponse{ + Content: "ok", + Done: true, + DoneReason: llm.DoneReasonStop, + PromptEvalCount: 1, + PromptEvalDuration: time.Millisecond, + EvalCount: 1, + EvalDuration: time.Millisecond, + }, + } + s := newServerWithMockRunner(t, &mock) + createMinimalGGUFModel(t, s, "generate-chat-template", ggml.KV{ + "tokenizer.chat_template": "{{ messages[0]['content'] }}", + }, "", nil) + + stream := false + w := createRequest(t, s.GenerateHandler, api.GenerateRequest{ + Model: "generate-chat-template", + Prompt: "hello", + Stream: &stream, + }) + if w.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) + } + if mock.CompletionRequest.Prompt != "native-template: hello" { + t.Fatalf("completion prompt = %q, want native chat_template prompt", mock.CompletionRequest.Prompt) + } + if mock.CompletionRequest.LeadingBOS != "" { + t.Fatalf("native chat_template prompt should not add leading BOS, got %q", mock.CompletionRequest.LeadingBOS) + } + }) + + t.Run("uses preferred GGUF chat_template over Go TEMPLATE", func(t *testing.T) { + mock := mockRunner{ + TemplateFn: func(_ context.Context, req llm.ChatRequest) (string, error) { + if len(req.Messages) != 1 || req.Messages[0].Content != "hello" { + t.Fatalf("chat template messages = %#v", req.Messages) + } + return "preferred-native-template: hello", nil + }, + } + s := newServerWithMockRunner(t, &mock) + createMinimalGGUFModel(t, s, "generate-preferred-chat-template", ggml.KV{ + "tokenizer.chat_template": "{% if tools %}{{ tools }}{% endif %}{{ messages[0]['content'] }}", + }, "{{ range .Messages }}go-template: {{ .Content }}{{ end }}", nil) + + w := createRequest(t, s.GenerateHandler, api.GenerateRequest{ + Model: "generate-preferred-chat-template", + Prompt: "hello", + DebugRenderOnly: true, + }) + if w.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) + } + + var actual api.GenerateResponse + if err := json.Unmarshal(w.Body.Bytes(), &actual); err != nil { + t.Fatal(err) + } + if actual.DebugInfo == nil || actual.DebugInfo.RenderedTemplate != "preferred-native-template: hello" { + t.Fatalf("rendered template = %#v, want preferred native chat_template", actual.DebugInfo) + } + }) + + t.Run("falls back to default raw-like template without chat_template", func(t *testing.T) { + mock := mockRunner{ + TemplateFn: func(context.Context, llm.ChatRequest) (string, error) { + t.Fatal("native chat template should not be used without tokenizer.chat_template") + return "", nil + }, + } + s := newServerWithMockRunner(t, &mock) + createMinimalGGUFModel(t, s, "generate-no-chat-template", nil, "", nil) + + w := createRequest(t, s.GenerateHandler, api.GenerateRequest{ + Model: "generate-no-chat-template", + Prompt: "plain prompt", + DebugRenderOnly: true, + }) + if w.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) + } + + var actual api.GenerateResponse + if err := json.Unmarshal(w.Body.Bytes(), &actual); err != nil { + t.Fatal(err) + } + if actual.DebugInfo == nil || actual.DebugInfo.RenderedTemplate != "plain prompt" { + t.Fatalf("rendered template = %#v, want raw-like default template", actual.DebugInfo) + } + }) + + t.Run("prepends deprecated context to native chat template prompt", func(t *testing.T) { + mock := mockRunner{ + TemplateFn: func(_ context.Context, req llm.ChatRequest) (string, error) { + if len(req.Messages) != 1 || req.Messages[0].Content != "next" { + t.Fatalf("chat template messages = %#v", req.Messages) + } + return "native-template: next", nil + }, + DetokenizeFn: func(_ context.Context, tokens []int) (string, error) { + if !slices.Equal(tokens, []int{1, 2}) { + t.Fatalf("detokenize tokens = %#v", tokens) + } + return "prior context ", nil + }, + CompletionResponse: llm.CompletionResponse{ + Content: "ok", + Done: true, + DoneReason: llm.DoneReasonStop, + }, + } + s := newServerWithMockRunner(t, &mock) + createMinimalGGUFModel(t, s, "generate-chat-template-context", ggml.KV{ + "tokenizer.chat_template": "{{ messages[0]['content'] }}", + }, "", nil) + + stream := false + w := createRequest(t, s.GenerateHandler, api.GenerateRequest{ + Model: "generate-chat-template-context", + Prompt: "next", + Context: []int{1, 2}, + Stream: &stream, + }) + if w.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) + } + if mock.CompletionRequest.Prompt != "prior context native-template: next" { + t.Fatalf("completion prompt = %q, want context-prefixed native chat_template prompt", mock.CompletionRequest.Prompt) + } + }) + + t.Run("keeps generate media markers with native chat template", func(t *testing.T) { + image1 := []byte("image-one") + image2 := []byte("image-two") + mock := mockRunner{ + TemplateFn: func(_ context.Context, req llm.ChatRequest) (string, error) { + if len(req.Messages) != 1 { + t.Fatalf("chat template messages = %#v", req.Messages) + } + if len(req.Messages[0].Images) != 0 { + t.Fatalf("native generate render request should carry image markers, not image payloads: %#v", req.Messages[0]) + } + return req.Messages[0].Content, nil + }, + CompletionResponse: llm.CompletionResponse{ + Content: "ok", + Done: true, + DoneReason: llm.DoneReasonStop, + }, + } + s := newServerWithMockRunner(t, &mock) + createMinimalGGUFModel(t, s, "generate-chat-template-images", ggml.KV{ + "tokenizer.chat_template": "{{ messages[0]['content'] }}", + }, "", nil) + + stream := false + w := createRequest(t, s.GenerateHandler, api.GenerateRequest{ + Model: "generate-chat-template-images", + Prompt: "compare [img] and [img]", + Images: []api.ImageData{image1, image2}, + Stream: &stream, + }) + if w.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) + } + if mock.CompletionRequest.Prompt != "compare [img-0] and [img-1]" { + t.Fatalf("completion prompt = %q, want image marker prompt", mock.CompletionRequest.Prompt) + } + if len(mock.CompletionRequest.Media) != 2 { + t.Fatalf("completion media = %#v, want two images", mock.CompletionRequest.Media) + } + if mock.CompletionRequest.Media[0].ID != 0 || !bytes.Equal(mock.CompletionRequest.Media[0].Data, image1) { + t.Fatalf("first image media = %#v", mock.CompletionRequest.Media[0]) + } + if mock.CompletionRequest.Media[1].ID != 1 || !bytes.Equal(mock.CompletionRequest.Media[1].Data, image2) { + t.Fatalf("second image media = %#v", mock.CompletionRequest.Media[1]) + } + }) +} + func TestGenerateChatRemote(t *testing.T) { gin.SetMode(gin.TestMode)