Add image and audio to prompt/exec

README.md

@@ -101,17 +101,21 @@ codeye --agent "my-custom-agent --stdio" prompt <session-id> "review security po
 
 ### Global Options
 
-| Flag              | Description                                                 |
-| ----------------- | ----------------------------------------------------------- |
-| `--cwd <path>`    | Run in a specific working directory                         |
-| `--agent "<cmd>"` | Use a custom ACP-compatible agent command                   |
-| `--format <mode>` | Output format: `text`, `json`, `json-strict`, `quiet`       |
-| `--json-strict`   | Shorthand for `--format json-strict`                        |
-| `--approve-all`   | Allow all agent tool requests                               |
-| `--approve-reads` | Allow read-only tool requests, deny writes                  |
-| `--deny-all`      | Deny all agent tool requests                                |
-| `--ask`           | Prompt to approve or reject each tool request (interactive) |
-| `--version`, `-V` | Print version                                               |
+| Flag                | Description                                                 |
+| ------------------- | ----------------------------------------------------------- |
+| `--cwd <path>`      | Run in a specific working directory                         |
+| `--agent "<cmd>"`   | Use a custom ACP-compatible agent command                   |
+| `--format <mode>`   | Output format: `text`, `json`, `json-strict`, `quiet`       |
+| `--json-strict`     | Shorthand for `--format json-strict`                        |
+| `--audio <path>`    | Add audio file(s) to prompt/exec (repeatable). Supports .wav, .mp3, .ogg, .flac, .m4a |
+| `--image <path>`    | Add image file(s) to prompt/exec (repeatable). Supports .png, .jpg, .gif, .webp |
+| `--approve-all`     | Allow all agent tool requests                               |
+| `--approve-reads`   | Allow read-only tool requests, deny writes                  |
+| `--deny-all`        | Deny all agent tool requests                                |
+| `--ask`             | Prompt to approve or reject each tool request (interactive) |
+| `--version`, `-V`   | Print version                                               |
+
+For `prompt` and `exec`, place `--audio` and `--image` before the command (e.g. `codeye --image diagram.png prompt <session-id> "describe this"`). Agents must advertise the corresponding prompt capabilities (image/audio) in initialization.
 
 ## Configuration
 

internal/acp/payloads.go

@@ -46,14 +46,23 @@ type SessionListResponse struct {
 	Sessions []SessionListEntry `json:"sessions"`
 }
 
-type PromptTextPart struct {
-	Type string `json:"type"`
-	Text string `json:"text"`
+// PromptPart is one content block in a session/prompt request.
+// For text: Type="text", Text set. For image/audio: Type="image"|"audio", MimeType and Data (base64) set.
+type PromptPart struct {
+	Type     string `json:"type"`
+	Text     string `json:"text,omitempty"`
+	MimeType string `json:"mimeType,omitempty"`
+	Data     string `json:"data,omitempty"` // base64-encoded for image/audio
+}
+
+// TextPrompt returns a single text prompt part (convenience for callers that have only text).
+func TextPrompt(text string) []PromptPart {
+	return []PromptPart{{Type: "text", Text: text}}
 }
 
 type SessionPromptRequest struct {
-	SessionID string           `json:"sessionId"`
-	Prompt    []PromptTextPart `json:"prompt"`
+	SessionID string       `json:"sessionId"`
+	Prompt    []PromptPart `json:"prompt"`
 }
 
 type SessionPromptResponse struct {

internal/cli/prompt_parts.go

@@ -0,0 +1,69 @@
+package cli
+
+import (
+	"encoding/base64"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/one710/codeye/internal/acp"
+)
+
+// BuildPromptParts returns ACP prompt parts: one text block (if text is non-empty), then image blocks, then audio blocks.
+// Image and audio files are read and base64-encoded; mime types are inferred from extension.
+func BuildPromptParts(text string, imagePaths, audioPaths []string) ([]acp.PromptPart, error) {
+	var parts []acp.PromptPart
+	text = strings.TrimSpace(text)
+	if text != "" {
+		parts = append(parts, acp.PromptPart{Type: "text", Text: text})
+	}
+	for _, p := range imagePaths {
+		mime, data, err := readFileAsBase64(p, imageMimeTypes)
+		if err != nil {
+			return nil, fmt.Errorf("image %s: %w", p, err)
+		}
+		parts = append(parts, acp.PromptPart{Type: "image", MimeType: mime, Data: data})
+	}
+	for _, p := range audioPaths {
+		mime, data, err := readFileAsBase64(p, audioMimeTypes)
+		if err != nil {
+			return nil, fmt.Errorf("audio %s: %w", p, err)
+		}
+		parts = append(parts, acp.PromptPart{Type: "audio", MimeType: mime, Data: data})
+	}
+	if len(parts) == 0 {
+		parts = []acp.PromptPart{{Type: "text", Text: ""}}
+	}
+	return parts, nil
+}
+
+func readFileAsBase64(path string, mimeMap map[string]string) (mimeType, b64 string, err error) {
+	ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(path), "."))
+	mimeType = mimeMap[ext]
+	if mimeType == "" {
+		return "", "", fmt.Errorf("unsupported extension .%s", ext)
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return "", "", err
+	}
+	return mimeType, base64.StdEncoding.EncodeToString(raw), nil
+}
+
+var imageMimeTypes = map[string]string{
+	"png":  "image/png",
+	"jpg":  "image/jpeg",
+	"jpeg": "image/jpeg",
+	"gif":  "image/gif",
+	"webp": "image/webp",
+}
+
+var audioMimeTypes = map[string]string{
+	"wav":  "audio/wav",
+	"mp3":  "audio/mpeg",
+	"mpeg": "audio/mpeg",
+	"ogg":  "audio/ogg",
+	"flac": "audio/flac",
+	"m4a":  "audio/mp4",
+}

internal/cli/root.go

@@ -32,6 +32,8 @@ type globalFlags struct {
 	AgentCommand string
 	Format       output.Format
 	PermMode     permissions.Mode
+	AudioPaths   []string
+	ImagePaths   []string
 }
 
 func Run(argv []string) int {
@@ -114,7 +116,7 @@ func Run(argv []string) int {
 	)
 
 	ctx := context.Background()
-	code := dispatch(ctx, out, repo, rt, adapter, agent, flags.Cwd, cmd, cmdArgs, flags.PermMode, flags.Format)
+	code := dispatch(ctx, out, repo, rt, adapter, agent, flags.Cwd, cmd, cmdArgs, flags.PermMode, flags.Format, flags.AudioPaths, flags.ImagePaths)
 	return code
 }
 
@@ -130,15 +132,21 @@ func dispatch(
 	args []string,
 	permMode permissions.Mode,
 	format output.Format,
+	audioPaths, imagePaths []string,
 ) int {
 	switch cmd {
 	case "prompt":
 		if len(args) < 2 {
-			out.PrintRPCError(-32602, "usage: prompt <session-id> <text...>", map[string]interface{}{"errorCode": cerr.CodeUsage})
+			out.PrintRPCError(-32602, "usage: prompt <session-id> <text...> [--audio <path>...] [--image <path>...]", map[string]interface{}{"errorCode": cerr.CodeUsage})
 			return 2
 		}
 		sessionID := args[0]
 		promptText := strings.Join(args[1:], " ")
+		parts, err := BuildPromptParts(promptText, imagePaths, audioPaths)
+		if err != nil {
+			out.PrintError(err.Error())
+			return 1
+		}
 		rec, err := repo.Load(sessionID)
 		if err != nil {
 			out.PrintError("session not found: " + sessionID)
@@ -149,7 +157,7 @@ func dispatch(
 			return 1
 		}
 		// ACP: session/load (by agent) restores conversation history; then session/prompt. All in-process.
-		stopReason, responseText, err := rt.PromptWithOutput(ctx, rec, promptText)
+		stopReason, responseText, err := rt.PromptWithOutput(ctx, rec, parts)
 		if err != nil {
 			out.PrintError(err.Error())
 			return 1
@@ -163,11 +171,16 @@ func dispatch(
 		}, "")
 		return 0
 	case "exec":
-		if len(args) == 0 {
-			out.PrintRPCError(-32602, "prompt is required", map[string]interface{}{"errorCode": cerr.CodeUsage})
+		if len(args) == 0 && len(audioPaths) == 0 && len(imagePaths) == 0 {
+			out.PrintRPCError(-32602, "prompt text or --audio/--image is required", map[string]interface{}{"errorCode": cerr.CodeUsage})
 			return 2
 		}
-		stopReason, responseText, err := rt.RunOnceWithOutput(ctx, cwd, strings.Join(args, " "))
+		parts, err := BuildPromptParts(strings.Join(args, " "), imagePaths, audioPaths)
+		if err != nil {
+			out.PrintError(err.Error())
+			return 1
+		}
+		stopReason, responseText, err := rt.RunOnceWithOutput(ctx, cwd, parts)
 		if err != nil {
 			out.PrintError(err.Error())
 			return 1
@@ -428,6 +441,16 @@ func parseGlobals(args []string) (globalFlags, []string, error) {
 			flags.PermMode = permissions.DenyAll
 		case "--ask":
 			flags.PermMode = permissions.Ask
+		case "--audio":
+			i++
+			if i < len(args) {
+				flags.AudioPaths = append(flags.AudioPaths, args[i])
+			}
+		case "--image":
+			i++
+			if i < len(args) {
+				flags.ImagePaths = append(flags.ImagePaths, args[i])
+			}
 		default:
 			if strings.HasPrefix(arg, "--") {
 				return flags, nil, fmt.Errorf("unknown flag: %s", arg)

internal/client/client.go

@@ -242,7 +242,10 @@ func (c *Client) ListSessions(ctx context.Context, cwd string) ([]string, error)
 	return out, nil
 }
 
-func (c *Client) Prompt(ctx context.Context, sessionID, text string) (PromptResult, error) {
+func (c *Client) Prompt(ctx context.Context, sessionID string, parts []acp.PromptPart) (PromptResult, error) {
+	if len(parts) == 0 {
+		parts = []acp.PromptPart{{Type: "text", Text: ""}}
+	}
 	c.updateMu.Lock()
 	c.activePromptSessionID = sessionID
 	c.activePromptChunks = nil
@@ -256,9 +259,7 @@ func (c *Client) Prompt(ctx context.Context, sessionID, text string) (PromptResu
 
 	res, err := c.call(ctx, acp.MethodSessionPrompt, acp.SessionPromptRequest{
 		SessionID: sessionID,
-		Prompt: []acp.PromptTextPart{
-			{Type: "text", Text: text},
-		},
+		Prompt:    parts,
 	})
 	if err != nil {
 		return PromptResult{}, err

internal/queue/protocol.go

@@ -1,5 +1,7 @@
 package queue
 
+import "github.com/one710/codeye/internal/acp"
+
 type Command string
 
 const (
@@ -11,13 +13,14 @@ const (
 )
 
 type Request struct {
-	RequestID string  `json:"requestId,omitempty"`
-	Command   Command `json:"command"`
-	SessionID string  `json:"sessionId"`
-	Prompt    string  `json:"prompt,omitempty"`
-	Mode      string  `json:"mode,omitempty"`
-	Key       string  `json:"key,omitempty"`
-	Value     string  `json:"value,omitempty"`
+	RequestID   string           `json:"requestId,omitempty"`
+	Command     Command          `json:"command"`
+	SessionID   string           `json:"sessionId"`
+	Prompt      string           `json:"prompt,omitempty"`      // legacy: single text part
+	PromptParts []acp.PromptPart `json:"promptParts,omitempty"` // when set, used instead of Prompt
+	Mode        string           `json:"mode,omitempty"`
+	Key         string           `json:"key,omitempty"`
+	Value       string           `json:"value,omitempty"`
 }
 
 type Response struct {

internal/queue/server.go

@@ -8,10 +8,12 @@ import (
 	"os"
 	"sync"
 	"time"
+
+	"github.com/one710/codeye/internal/acp"
 )
 
 type Handler interface {
-	Prompt(ctx context.Context, sessionID, prompt string) (PromptResult, error)
+	Prompt(ctx context.Context, sessionID string, parts []acp.PromptPart) (PromptResult, error)
 	Cancel(ctx context.Context, sessionID string) error
 	SetMode(ctx context.Context, sessionID, mode string) error
 	SetConfigOption(ctx context.Context, sessionID, key, value string) error
@@ -122,7 +124,11 @@ func (s *Server) Run(ctx context.Context) error {
 func (s *Server) dispatch(ctx context.Context, req Request) (Response, error) {
 	switch req.Command {
 	case CmdPrompt:
-		result, err := s.Handler.Prompt(ctx, req.SessionID, req.Prompt)
+		parts := req.PromptParts
+		if len(parts) == 0 {
+			parts = []acp.PromptPart{{Type: "text", Text: req.Prompt}}
+		}
+		result, err := s.Handler.Prompt(ctx, req.SessionID, parts)
 		if err != nil {
 			return Response{}, err
 		}

internal/session/runtime.go

@@ -11,6 +11,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/one710/codeye/internal/acp"
 	"github.com/one710/codeye/internal/client"
 	"github.com/one710/codeye/internal/queue"
 	"github.com/one710/codeye/internal/session/persistence"
@@ -59,18 +60,18 @@ func (r *Runtime) CreateSession(ctx context.Context, agent, cwd, name string) (p
 	return rec, nil
 }
 
-func (r *Runtime) Prompt(ctx context.Context, rec persistence.Record, prompt string) (string, error) {
-	stopReason, _, err := r.PromptWithOutput(ctx, rec, prompt)
+func (r *Runtime) Prompt(ctx context.Context, rec persistence.Record, parts []acp.PromptPart) (string, error) {
+	stopReason, _, err := r.PromptWithOutput(ctx, rec, parts)
 	return stopReason, err
 }
 
-func (r *Runtime) PromptWithOutput(ctx context.Context, rec persistence.Record, prompt string) (string, string, error) {
-	return r.PromptInProcess(ctx, rec, prompt)
+func (r *Runtime) PromptWithOutput(ctx context.Context, rec persistence.Record, parts []acp.PromptPart) (string, string, error) {
+	return r.PromptInProcess(ctx, rec, parts)
 }
 
 // PromptInProcess runs the prompt in the current process with the same client as exec,
 // so streaming and permission prompts behave identically to exec.
-func (r *Runtime) PromptInProcess(ctx context.Context, rec persistence.Record, prompt string) (string, string, error) {
+func (r *Runtime) PromptInProcess(ctx context.Context, rec persistence.Record, parts []acp.PromptPart) (string, string, error) {
 	c := r.ClientFactory()
 	if err := c.Start(ctx); err != nil {
 		return "", "", err
@@ -86,7 +87,7 @@ func (r *Runtime) PromptInProcess(ctx context.Context, rec persistence.Record, p
 		rec.ACPSession = newSid
 		_ = r.Repo.Save(rec)
 	}
-	result, err := c.Prompt(ctx, sid, prompt)
+	result, err := c.Prompt(ctx, sid, parts)
 	if err != nil {
 		return "", "", err
 	}
@@ -193,12 +194,12 @@ func (r *Runtime) RunWorkingSession(ctx context.Context, initial persistence.Rec
 	return server.Run(ctx)
 }
 
-func (r *Runtime) RunOnce(ctx context.Context, cwd, prompt string) (string, error) {
-	stopReason, _, err := r.RunOnceWithOutput(ctx, cwd, prompt)
+func (r *Runtime) RunOnce(ctx context.Context, cwd string, parts []acp.PromptPart) (string, error) {
+	stopReason, _, err := r.RunOnceWithOutput(ctx, cwd, parts)
 	return stopReason, err
 }
 
-func (r *Runtime) RunOnceWithOutput(ctx context.Context, cwd, prompt string) (string, string, error) {
+func (r *Runtime) RunOnceWithOutput(ctx context.Context, cwd string, parts []acp.PromptPart) (string, string, error) {
 	c := r.ClientFactory()
 	if err := c.Start(ctx); err != nil {
 		return "", "", err
@@ -208,7 +209,7 @@ func (r *Runtime) RunOnceWithOutput(ctx context.Context, cwd, prompt string) (st
 	if err != nil {
 		return "", "", err
 	}
-	result, err := c.Prompt(ctx, sid, prompt)
+	result, err := c.Prompt(ctx, sid, parts)
 	if err != nil {
 		return "", "", err
 	}
@@ -250,14 +251,14 @@ type workingSessionHandler struct {
 	fullMessage   string // set when we get a non-chunk final message to avoid duplicating chunk content
 }
 
-func (w *workingSessionHandler) Prompt(ctx context.Context, sessionID, prompt string) (queue.PromptResult, error) {
+func (w *workingSessionHandler) Prompt(ctx context.Context, sessionID string, parts []acp.PromptPart) (queue.PromptResult, error) {
 	w.mu.Lock()
 	activeSessionID := w.liveSessionID
 	w.chunks = nil
 	w.fullMessage = ""
 	w.mu.Unlock()
 
-	result, err := w.client.Prompt(ctx, activeSessionID, prompt)
+	result, err := w.client.Prompt(ctx, activeSessionID, parts)
 	if err != nil && shouldRecreateSessionOnPromptError(err) {
 		// Agent rejected stale/unknown session; recreate once and retry.
 		if sid, createErr := w.client.CreateSession(ctx, w.cwd); createErr == nil {
@@ -268,7 +269,7 @@ func (w *workingSessionHandler) Prompt(ctx context.Context, sessionID, prompt st
 				_ = w.repo.Save(w.record)
 			}
 			w.mu.Unlock()
-			result, err = w.client.Prompt(ctx, sid, prompt)
+			result, err = w.client.Prompt(ctx, sid, parts)
 		}
 	}
 	if err != nil {