fix renderer prompt format rather rewrite the prompt itself

anthropic/anthropic.go

@@ -78,6 +78,11 @@ type MessagesRequest struct {
 	ToolChoice    *ToolChoice     `json:"tool_choice,omitempty"`
 	Thinking      *ThinkingConfig `json:"thinking,omitempty"`
 	Metadata      *Metadata       `json:"metadata,omitempty"`
+	OutputConfig  *OutputConfig   `json:"output_config,omitempty"`
+}
+
+type OutputConfig struct {
+	Effort string `json:"effort,omitempty"`
 }
 
 // MessageParam represents a message in the request
@@ -161,7 +166,7 @@ type WebSearchToolResultError struct {
 
 // ImageSource represents the source of an image
 type ImageSource struct {
-	Type      string `json:"type"` // "base64" or "url"
+	Type      string `json:"type"` // "base64"
 	MediaType string `json:"media_type,omitempty"`
 	Data      string `json:"data,omitempty"`
 	URL       string `json:"url,omitempty"`
@@ -373,9 +378,25 @@ func FromMessagesRequest(r MessagesRequest) (*api.ChatRequest, error) {
 	}
 
 	var think *api.ThinkValue
+	normalizedEffort := ""
+	if r.OutputConfig != nil {
+		normalizedEffort = strings.ToLower(strings.TrimSpace(r.OutputConfig.Effort))
+	}
+
 	if r.Thinking != nil && r.Thinking.Type == "enabled" {
 		think = &api.ThinkValue{Value: true}
 	}
+	if r.Thinking != nil && r.Thinking.Type == "disabled" {
+		think = &api.ThinkValue{Value: false}
+	}
+	if think == nil && r.OutputConfig != nil {
+		switch normalizedEffort {
+		case "high", "medium", "low", "max":
+			think = &api.ThinkValue{Value: normalizedEffort}
+		case "none":
+			think = &api.ThinkValue{Value: false}
+		}
+	}
 
 	stream := r.Stream
 	convertedRequest := &api.ChatRequest{
@@ -425,17 +446,12 @@ func convertMessage(msg MessageParam) ([]api.Message, error) {
 				return nil, errors.New("invalid image source")
 			}
 
-			if block.Source.Type == "base64" {
-				decoded, err := base64.StdEncoding.DecodeString(block.Source.Data)
-				if err != nil {
-					logutil.Trace("anthropic: invalid base64 image data", "role", role, "error", err)
-					return nil, fmt.Errorf("invalid base64 image data: %w", err)
-				}
-				images = append(images, decoded)
-			} else {
-				logutil.Trace("anthropic: unsupported image source type", "role", role, "source_type", block.Source.Type)
-				return nil, fmt.Errorf("invalid image source type: %s. Only base64 images are supported.", block.Source.Type)
+			decoded, err := resolveImageSource(block.Source)
+			if err != nil {
+				logutil.Trace("anthropic: unsupported image source", "role", role, "source_type", block.Source.Type, "error", err)
+				return nil, err
 			}
+			images = append(images, decoded)
 
 		case "tool_use":
 			toolUseBlocks++
@@ -457,26 +473,16 @@ func convertMessage(msg MessageParam) ([]api.Message, error) {
 
 		case "tool_result":
 			toolResultBlocks++
-			var resultContent string
-
-			switch c := block.Content.(type) {
-			case string:
-				resultContent = c
-			case []any:
-				for _, cb := range c {
-					if cbMap, ok := cb.(map[string]any); ok {
-						if cbMap["type"] == "text" {
-							if text, ok := cbMap["text"].(string); ok {
-								resultContent += text
-							}
-						}
-					}
-				}
+			resultContent, resultImages, err := convertToolResultContent(block.Content)
+			if err != nil {
+				logutil.Trace("anthropic: invalid tool_result content", "role", role, "error", err)
+				return nil, err
 			}
 
 			toolResults = append(toolResults, api.Message{
 				Role:       "tool",
 				Content:    resultContent,
+				Images:     resultImages,
 				ToolCallID: block.ToolUseID,
 			})
 
@@ -508,6 +514,10 @@ func convertMessage(msg MessageParam) ([]api.Message, error) {
 		}
 	}
 
+	if role == "user" && len(toolResults) > 0 {
+		messages = append(messages, toolResults...)
+	}
+
 	if textContent.Len() > 0 || len(images) > 0 || len(toolCalls) > 0 || thinking != "" {
 		m := api.Message{
 			Role:      role,
@@ -519,8 +529,10 @@ func convertMessage(msg MessageParam) ([]api.Message, error) {
 		messages = append(messages, m)
 	}
 
-	// Add tool results as separate messages
-	messages = append(messages, toolResults...)
+	// Add tool results as separate messages.
+	if role != "user" || len(toolResults) == 0 {
+		messages = append(messages, toolResults...)
+	}
 	logutil.Trace("anthropic: converted block message",
 		"role", role,
 		"blocks", len(msg.Content),
@@ -969,6 +981,71 @@ func GenerateMessageID() string {
 	return generateID("msg")
 }
 
+func resolveImageSource(source *ImageSource) (api.ImageData, error) {
+	if source.Type != "base64" {
+		return nil, fmt.Errorf("invalid image source type: %s. Only base64 images are supported.", source.Type)
+	}
+
+	decoded, err := base64.StdEncoding.DecodeString(source.Data)
+	if err != nil {
+		return nil, fmt.Errorf("invalid base64 image data: %w", err)
+	}
+
+	return decoded, nil
+}
+
+func convertToolResultContent(content any) (string, []api.ImageData, error) {
+	switch c := content.(type) {
+	case nil:
+		return "", nil, nil
+	case string:
+		return c, nil, nil
+	case []any:
+		var text strings.Builder
+		var images []api.ImageData
+
+		for _, cb := range c {
+			cbMap, ok := cb.(map[string]any)
+			if !ok {
+				continue
+			}
+
+			switch cbMap["type"] {
+			case "text":
+				if t, ok := cbMap["text"].(string); ok {
+					text.WriteString(t)
+				}
+			case "image":
+				rawSource, ok := cbMap["source"].(map[string]any)
+				if !ok {
+					return "", nil, errors.New("invalid tool_result image source")
+				}
+
+				var source ImageSource
+				if rawType, ok := rawSource["type"].(string); ok {
+					source.Type = rawType
+				}
+				if rawMediaType, ok := rawSource["media_type"].(string); ok {
+					source.MediaType = rawMediaType
+				}
+				if rawData, ok := rawSource["data"].(string); ok {
+					source.Data = rawData
+				}
+
+				img, err := resolveImageSource(&source)
+				if err != nil {
+					return "", nil, err
+				}
+				images = append(images, img)
+			}
+		}
+
+		return text.String(), images, nil
+	default:
+		return "", nil, nil
+	}
+}
+
 // ptr returns a pointer to the given string value
 func ptr(s string) *string {
 	return &s

anthropic/anthropic_test.go

@@ -271,6 +271,212 @@ func TestFromMessagesRequest_WithToolResult(t *testing.T) {
 	}
 }
 
+func TestFromMessagesRequest_WithToolResultImage(t *testing.T) {
+	imgData, _ := base64.StdEncoding.DecodeString(testImage)
+
+	req := MessagesRequest{
+		Model:     "test-model",
+		MaxTokens: 1024,
+		Messages: []MessageParam{
+			{
+				Role: "user",
+				Content: []ContentBlock{
+					{
+						Type:      "tool_result",
+						ToolUseID: "call_img",
+						Content: []any{
+							map[string]any{"type": "text", "text": "Attached image"},
+							map[string]any{
+								"type": "image",
+								"source": map[string]any{
+									"type":       "base64",
+									"media_type": "image/png",
+									"data":       testImage,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	result, err := FromMessagesRequest(req)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if len(result.Messages) != 1 {
+		t.Fatalf("expected 1 message, got %d", len(result.Messages))
+	}
+
+	msg := result.Messages[0]
+	if msg.Role != "tool" {
+		t.Errorf("expected role 'tool', got %q", msg.Role)
+	}
+	if msg.ToolCallID != "call_img" {
+		t.Errorf("expected tool_call_id 'call_img', got %q", msg.ToolCallID)
+	}
+	if msg.Content != "Attached image" {
+		t.Errorf("unexpected content: %q", msg.Content)
+	}
+	if len(msg.Images) != 1 {
+		t.Fatalf("expected 1 image, got %d", len(msg.Images))
+	}
+	if string(msg.Images[0]) != string(imgData) {
+		t.Error("image data mismatch")
+	}
+}
+
+func TestFromMessagesRequest_WithToolResultFollowedByUserText(t *testing.T) {
+	req := MessagesRequest{
+		Model:     "test-model",
+		MaxTokens: 1024,
+		Messages: []MessageParam{
+			{
+				Role: "assistant",
+				Content: []ContentBlock{
+					{
+						Type:  "tool_use",
+						ID:    "call_read",
+						Name:  "Read",
+						Input: makeArgs("file_path", "/Users/hoyyeva/Desktop/aaa.png"),
+					},
+				},
+			},
+			{
+				Role: "user",
+				Content: []ContentBlock{
+					{
+						Type:      "tool_result",
+						ToolUseID: "call_read",
+						Content:   "Read image (311.5KB)",
+					},
+					{
+						Type: "text",
+						Text: ptr("Please describe it."),
+					},
+				},
+			},
+		},
+	}
+
+	result, err := FromMessagesRequest(req)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if len(result.Messages) != 3 {
+		t.Fatalf("expected 3 messages, got %d", len(result.Messages))
+	}
+
+	if result.Messages[1].Role != "tool" {
+		t.Fatalf("expected second message to be tool, got %q", result.Messages[1].Role)
+	}
+	if result.Messages[1].ToolCallID != "call_read" {
+		t.Fatalf("expected tool_call_id 'call_read', got %q", result.Messages[1].ToolCallID)
+	}
+	if result.Messages[2].Role != "user" {
+		t.Fatalf("expected third message to be user, got %q", result.Messages[2].Role)
+	}
+	if result.Messages[2].Content != "Please describe it." {
+		t.Fatalf("unexpected user content: %q", result.Messages[2].Content)
+	}
+}
+
+func TestFromMessagesRequest_WithOutputConfigEffort(t *testing.T) {
+	req := MessagesRequest{
+		Model:     "gemma4",
+		MaxTokens: 32000,
+		Messages: []MessageParam{
+			{
+				Role:    "user",
+				Content: textContent("Describe the image."),
+			},
+		},
+		OutputConfig: &OutputConfig{
+			Effort: "high",
+		},
+	}
+
+	result, err := FromMessagesRequest(req)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if result.Think == nil {
+		t.Fatal("expected think to be set from output_config.effort")
+	}
+
+	if got := result.Think.String(); got != "high" {
+		t.Fatalf("expected think level 'high', got %q", got)
+	}
+}
+
+func TestFromMessagesRequest_ThinkingDisabledOverridesOutputConfigEffort(t *testing.T) {
+	req := MessagesRequest{
+		Model:     "gemma4",
+		MaxTokens: 32000,
+		Messages: []MessageParam{
+			{
+				Role:    "user",
+				Content: textContent("Describe the image."),
+			},
+		},
+		Thinking: &ThinkingConfig{
+			Type: "disabled",
+		},
+		OutputConfig: &OutputConfig{
+			Effort: "high",
+		},
+	}
+
+	result, err := FromMessagesRequest(req)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if result.Think == nil {
+		t.Fatal("expected think to be set")
+	}
+
+	if got := result.Think.Value; got != false {
+		t.Fatalf("expected think=false when thinking is disabled, got %v", got)
+	}
+}
+
+func TestFromMessagesRequest_ThinkingAdaptiveUsesOutputConfigEffort(t *testing.T) {
+	req := MessagesRequest{
+		Model:     "gemma4",
+		MaxTokens: 32000,
+		Messages: []MessageParam{
+			{
+				Role:    "user",
+				Content: textContent("Describe the image."),
+			},
+		},
+		Thinking: &ThinkingConfig{
+			Type: "adaptive",
+		},
+		OutputConfig: &OutputConfig{
+			Effort: "high",
+		},
+	}
+
+	result, err := FromMessagesRequest(req)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if result.Think == nil {
+		t.Fatal("expected think to be set from output_config.effort")
+	}
+
+	if got := result.Think.String(); got != "high" {
+		t.Fatalf("expected think level 'high' for adaptive thinking, got %q", got)
+	}
+}
+
 func TestFromMessagesRequest_WithTools(t *testing.T) {
 	req := MessagesRequest{
 		Model:     "test-model",

model/renderers/gemma4.go

@@ -98,7 +98,8 @@ func (r *Gemma4Renderer) Render(messages []api.Message, tools []api.Tool, thinkV
 		toolResponsesEmitted := false
 		if len(message.ToolCalls) > 0 {
 			for k := i + 1; k < len(loopMessages) && loopMessages[k].Role == "tool"; k++ {
-				sb.WriteString(r.formatToolResponseBlock(r.toolResponseName(loopMessages[k], message.ToolCalls), loopMessages[k].Content))
+				response := r.renderToolResponseContent(loopMessages[k], &imageOffset)
+				sb.WriteString(r.formatToolResponseBlock(r.toolResponseName(loopMessages[k], message.ToolCalls), response))
 				toolResponsesEmitted = true
 				prevMessageType = "tool_response"
 			}
@@ -173,6 +174,12 @@ func (r *Gemma4Renderer) renderContent(sb *strings.Builder, msg api.Message, ima
 	sb.WriteString(content)
 }
 
+func (r *Gemma4Renderer) renderToolResponseContent(msg api.Message, imageOffset *int) string {
+	var sb strings.Builder
+	r.renderContent(&sb, msg, imageOffset, false)
+	return sb.String()
+}
+
 func (r *Gemma4Renderer) previousNonToolRole(messages []api.Message, idx int) string {
 	for i := idx - 1; i >= 0; i-- {
 		if messages[i].Role != "tool" {

model/renderers/glmocr.go

@@ -85,8 +85,10 @@ func (r *GlmOcrRenderer) Render(messages []api.Message, tools []api.Tool, thinkV
 			if i == 0 || messages[i-1].Role != "tool" {
 				sb.WriteString("<|observation|>")
 			}
+			content, nextOffset := r.renderContent(message, imageOffset)
+			imageOffset = nextOffset
 			sb.WriteString("\n<tool_response>\n")
-			sb.WriteString(message.Content)
+			sb.WriteString(content)
 			sb.WriteString("\n</tool_response>\n")
 		case "system":
 			sb.WriteString("<|system|>\n")

model/renderers/nemotron3nano.go

@@ -79,12 +79,14 @@ func (r *Nemotron3NanoRenderer) Render(messages []api.Message, tools []api.Tool,
 			// Check if previous message was also a tool message
 			prevWasTool := i > 0 && loopMessages[i-1].Role == "tool"
 			nextIsTool := i+1 < len(loopMessages) && loopMessages[i+1].Role == "tool"
+			content := r.renderMessageContent(message, imageOffset)
+			imageOffset += len(message.Images)
 
 			if !prevWasTool {
 				sb.WriteString("<|im_start|>user\n")
 			}
 			sb.WriteString("<tool_response>\n")
-			sb.WriteString(message.Content)
+			sb.WriteString(content)
 			sb.WriteString("\n</tool_response>\n")
 
 			if !nextIsTool {

model/renderers/qwen3vl.go

@@ -126,7 +126,7 @@ func (r *Qwen3VLRenderer) Render(messages []api.Message, tools []api.Tool, think
 			if i == 0 || messages[i-1].Role != "tool" {
 				sb.WriteString("<|im_start|>user")
 			}
-			sb.WriteString("\n<tool_response>\n" + message.Content + "\n</tool_response>")
+			sb.WriteString("\n<tool_response>\n" + content + "\n</tool_response>")
 			if i == len(messages)-1 || messages[i+1].Role != "tool" {
 				sb.WriteString("<|im_end|>\n")
 			}

server/prompt.go

@@ -75,7 +75,9 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.
 		slog.Debug("truncating input messages which exceed context length", "truncated", len(msgs[currMsgIdx:]))
 	}
 
-	for cnt, msg := range msgs[currMsgIdx:] {
+	renderMsgs := slices.Clone(msgs)
+
+	for cnt, msg := range renderMsgs[currMsgIdx:] {
 		if slices.Contains(m.Config.ModelFamilies, "mllama") && len(msg.Images) > 1 {
 			return "", nil, errors.New("this model only supports one image while more than one image requested")
 		}
@@ -101,11 +103,16 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.
 				prompt = strings.Replace(prompt, "[img]", imgTag, 1)
 			}
 		}
-		msgs[currMsgIdx+cnt].Content = prefix + prompt
+
+		if m.Config.Renderer != "" {
+			continue
+		}
+
+		renderMsgs[currMsgIdx+cnt].Content = prefix + prompt
 	}
 
 	// truncate any messages that do not fit into the context window
-	p, err := renderPrompt(m, append(system, msgs[currMsgIdx:]...), tools, think)
+	p, err := renderPrompt(m, append(system, renderMsgs[currMsgIdx:]...), tools, think)
 	if err != nil {
 		return "", nil, err
 	}

server/prompt_test.go

@@ -406,6 +406,99 @@ func TestChatPromptGLMOcrRendererAddsImageTags(t *testing.T) {
 	}
 }
 
+func TestChatPromptRendererAddsToolImageTags(t *testing.T) {
+	msgs := []api.Message{
+		{
+			Role:    "user",
+			Content: "look at this file",
+			Images:  []api.ImageData{[]byte("img-1")},
+		},
+		{
+			Role: "assistant",
+			ToolCalls: []api.ToolCall{
+				{
+					ID: "call_read",
+					Function: api.ToolCallFunction{
+						Name: "Read",
+					},
+				},
+			},
+		},
+		{
+			Role:       "tool",
+			Content:    "attached image",
+			Images:     []api.ImageData{[]byte("img-2")},
+			ToolCallID: "call_read",
+		},
+	}
+
+	tests := []struct {
+		name            string
+		renderer        string
+		wantUserTag     string
+		wantToolContent string
+	}{
+		{
+			name:            "gemma4",
+			renderer:        "gemma4",
+			wantUserTag:     "<|turn>user\n[img-0]look at this file<turn|>\n",
+			wantToolContent: "[img-1]attached image",
+		},
+		{
+			name:            "qwen3-vl",
+			renderer:        "qwen3-vl-instruct",
+			wantUserTag:     "<|im_start|>user\n[img-0]look at this file<|im_end|>\n",
+			wantToolContent: "<tool_response>\n[img-1]attached image\n</tool_response>",
+		},
+		{
+			name:            "qwen3.5",
+			renderer:        "qwen3.5",
+			wantUserTag:     "<|im_start|>user\n[img-0]look at this file<|im_end|>\n",
+			wantToolContent: "<tool_response>\n[img-1]attached image\n</tool_response>",
+		},
+		{
+			name:            "glm-ocr",
+			renderer:        "glm-ocr",
+			wantUserTag:     "<|user|>\n[img-0]look at this file",
+			wantToolContent: "<tool_response>\n[img-1]attached image\n</tool_response>",
+		},
+		{
+			name:            "nemotron-3-nano",
+			renderer:        "nemotron-3-nano",
+			wantUserTag:     "<|im_start|>user\n[img-0]look at this file<|im_end|>\n",
+			wantToolContent: "<tool_response>\n[img-1]attached image\n</tool_response>",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := Model{
+				Config:         model.ConfigV2{Renderer: tt.renderer},
+				ProjectorPaths: []string{"vision"},
+			}
+			opts := api.Options{Runner: api.Runner{NumCtx: 8192}}
+			think := false
+
+			prompt, images, err := chatPrompt(t.Context(), &m, mockRunner{}.Tokenize, &opts, msgs, nil, &api.ThinkValue{Value: think}, true)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if got, want := len(images), 2; got != want {
+				t.Fatalf("len(images) = %d, want %d", got, want)
+			}
+
+			if !strings.Contains(prompt, tt.wantUserTag) {
+				t.Fatalf("prompt missing user image tag, got: %q", prompt)
+			}
+
+			if !strings.Contains(prompt, tt.wantToolContent) {
+				t.Fatalf("prompt missing tool image tag, got: %q", prompt)
+			}
+		})
+	}
+}
+
 func TestRenderPromptResolvesDynamicGemma4Renderer(t *testing.T) {
 	msgs := []api.Message{{Role: "user", Content: "Hello"}}