diff --git a/README.md b/README.md index e032623..f203a03 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ mmx text chat --model MiniMax-M3 --message "Hello" --stream mmx text chat --system "You are a coding assistant" --message "Fizzbuzz in Go" mmx text chat --message "user:Hi" --message "assistant:Hey!" --message "How are you?" cat messages.json | mmx text chat --messages-file - --output json +mmx text chat --image photo.jpg --message "What breed is this dog?" +mmx text chat --image before.png --image after.png --message "What changed?" ``` ### `mmx image` diff --git a/README_CN.md b/README_CN.md index 0b7c5f0..3c8ea97 100644 --- a/README_CN.md +++ b/README_CN.md @@ -72,6 +72,8 @@ mmx text chat --model MiniMax-M3 --message "你好" --stream mmx text chat --system "你是编程助手" --message "用 Go 写 Fizzbuzz" mmx text chat --message "user:你好" --message "assistant:嗨！" --message "你叫什么名字？" cat messages.json | mmx text chat --messages-file - --output json +mmx text chat --image photo.jpg --message "这是什么品种的狗？" +mmx text chat --image before.png --image after.png --message "这两张图有什么不同？" ``` ### `mmx image` diff --git a/skill/SKILL.md b/skill/SKILL.md index 5c7c3cf..9c34eec 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -57,6 +57,7 @@ mmx text chat --message [flags] | `--message ` | string, **required**, repeatable | Message text. Prefix with `role:` to set role (e.g. `"system:You are helpful"`, `"user:Hello"`) | | `--messages-file ` | string | JSON file with messages array. Use `-` for stdin | | `--system ` | string | System prompt | +| `--image ` | string, repeatable | Image to send with the message (auto base64-encoded). Forces `MiniMax-M3` unless `--model` is set | | `--model ` | string | Model ID (default: `MiniMax-M3`) | | `--max-tokens ` | number | Max tokens (default: 4096) | | `--temperature ` | number | Sampling temperature (0.0, 1.0] | @@ -76,8 +77,17 @@ mmx text chat \ # From file cat conversation.json | mmx text chat --messages-file - --output json + +# With images (M3 is multimodal; --image is repeatable) +mmx text chat --image photo.jpg --message "What breed is this dog?" --quiet +mmx text chat --image before.png --image after.png \ + --message "List every visual difference between these two." --quiet ``` +`text chat` posts to the Anthropic-compatible `/messages` endpoint, so hand-written +`--messages-file` image blocks must use `{"type":"image","source":{"type":"base64",...}}`. +The OpenAI `image_url` shape is rejected. `--image` emits the correct shape for you. + **stdout**: response text (text mode) or full response object (json mode). --- diff --git a/src/commands/text/chat.ts b/src/commands/text/chat.ts index fe2702d..df0264c 100644 --- a/src/commands/text/chat.ts +++ b/src/commands/text/chat.ts @@ -17,6 +17,13 @@ import type { import { readFileSync } from 'fs'; import { isInteractive } from '../../utils/env'; import { promptText, failIfMissing } from '../../utils/prompt'; +import { toImageBlock } from '../../utils/image'; + +// MiniMax Anthropic API contract (platform.minimax.io/docs/api-reference/text-anthropic-api): +// per-image cap, supported formats, and the whole-request-body cap. +const CHAT_IMAGE_MAX_BYTES = 10 * 1024 * 1024; +const CHAT_IMAGE_ALLOWED_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp']; +const CHAT_MAX_REQUEST_BYTES = 64 * 1024 * 1024; // --------------------------------------------------------------------------- // Thinking indicator — dynamic spinner + color-cycling label @@ -146,6 +153,27 @@ function parseMessages(flags: GlobalFlags): ParsedMessages { return { system, messages }; } +/** + * Attach image blocks to the last user message, promoting its content from a + * bare string to a block array. Images land after the text so the model reads + * the instruction first. + */ +function attachImages(messages: ChatMessage[], images: ContentBlock[]): void { + let idx = messages.length - 1; + while (idx >= 0 && messages[idx]!.role !== 'user') idx--; + + if (idx < 0) { + messages.push({ role: 'user', content: images }); + return; + } + + const target = messages[idx]!; + const content = typeof target.content === 'string' + ? (target.content ? [{ type: 'text' as const, text: target.content }] : []) + : target.content; + target.content = [...content, ...images]; +} + function extractText(content: ContentBlock[]): string { return content .filter((b): b is Extract => b.type === 'text') @@ -163,6 +191,7 @@ export default defineCommand({ { flag: '--message ', description: 'Message text (repeatable, prefix role: to set role)', required: true, type: 'array' }, { flag: '--messages-file ', description: 'JSON file with messages array (use - for stdin)' }, { flag: '--system ', description: 'System prompt' }, + { flag: '--image ', description: 'Image to send with the message (repeatable, base64 encoded automatically)', type: 'array' }, { flag: '--max-tokens ', description: 'Maximum tokens to generate (default: 4096)', type: 'number' }, { flag: '--temperature ', description: 'Sampling temperature (0.0, 1.0]', type: 'number' }, { flag: '--top-p ', description: 'Nucleus sampling threshold', type: 'number' }, @@ -173,14 +202,17 @@ export default defineCommand({ 'mmx text chat --message "What is MiniMax?"', 'mmx text chat --model MiniMax-M3 --system "You are a coding assistant." --message "Write fizzbuzz in Python"', 'mmx text chat --message "Hello" --message "assistant:Hi!" --message "How are you?"', + 'mmx text chat --image photo.jpg --message "What breed is this dog?"', + 'mmx text chat --image before.png --image after.png --message "List every visual difference."', 'cat conversation.json | mmx text chat --messages-file - --stream', 'mmx text chat --message "Hello" --output json', ], async run(config: Config, flags: GlobalFlags) { const { system, messages: parsedMessages } = parseMessages(flags); let messages = parsedMessages; + const imageInputs = (flags.image as string[] | undefined) ?? []; - if (messages.length === 0) { + if (messages.length === 0 && imageInputs.length === 0) { if (isInteractive({ nonInteractive: config.nonInteractive })) { const hint = await promptText({ message: 'Enter your message:', @@ -195,8 +227,36 @@ export default defineCommand({ } } + if (imageInputs.length > 0) { + const images: ContentBlock[] = []; + // Track the running request-body size so a run bails as soon as it's + // clear the request would exceed the cap, not after every image is + // fetched and encoded. + let requestBytes = Buffer.byteLength(JSON.stringify(messages), 'utf-8') + + Buffer.byteLength(system ?? '', 'utf-8'); + + for (const input of imageInputs) { + const block = await toImageBlock(input, { + maxBytes: CHAT_IMAGE_MAX_BYTES, + allowedMediaTypes: CHAT_IMAGE_ALLOWED_TYPES, + }); + requestBytes += block.source.data.length; + if (requestBytes > CHAT_MAX_REQUEST_BYTES) { + throw new CLIError( + `Request body exceeds the ${CHAT_MAX_REQUEST_BYTES / 1024 / 1024} MB limit once this image is attached.`, + ExitCode.USAGE, + 'Send fewer or smaller --image inputs.', + ); + } + images.push(block); + } + + attachImages(messages, images); + } + + // Images require a multimodal model, so they override a text-only config default. const model = (flags.model as string) - || config.defaultTextModel + || (imageInputs.length > 0 ? 'MiniMax-M3' : config.defaultTextModel) || 'MiniMax-M3'; const format = detectOutputFormat(config.output); const shouldStream = flags.stream === true || ( diff --git a/src/types/api.ts b/src/types/api.ts index badf507..8538e03 100644 --- a/src/types/api.ts +++ b/src/types/api.ts @@ -4,7 +4,8 @@ export type ContentBlock = | { type: 'text'; text: string } | { type: 'thinking'; thinking: string } | { type: 'tool_use'; id: string; name: string; input: Record } - | { type: 'tool_result'; tool_use_id: string; content: string }; + | { type: 'tool_result'; tool_use_id: string; content: string } + | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } }; export interface ChatMessage { role: 'user' | 'assistant'; diff --git a/src/utils/image.ts b/src/utils/image.ts index ab53b63..df6238f 100644 --- a/src/utils/image.ts +++ b/src/utils/image.ts @@ -2,6 +2,10 @@ import { readFileSync, existsSync, statSync } from 'fs'; import { extname } from 'path'; import { CLIError } from '../errors/base'; import { ExitCode } from '../errors/codes'; +import type { ContentBlock } from '../types/api'; +import { dataUriDecodedSize } from './media'; + +type ImageBlock = Extract; export const IMAGE_MIME_TYPES: Record = { '.jpg': 'image/jpeg', @@ -12,7 +16,13 @@ export const IMAGE_MIME_TYPES: Record = { '.heif': 'image/heif', }; -export function localFileToDataUri(filePath: string, maxBytes?: number): string { +/** Per-image and format constraints for a specific caller (e.g. chat's Anthropic-API contract). */ +export interface ImageValidationOptions { + maxBytes: number; + allowedMediaTypes: readonly string[]; +} + +export function localFileToDataUri(filePath: string, maxBytes?: number, mimeOverride?: string): string { if (maxBytes !== undefined) { const size = statSync(filePath).size; if (size > maxBytes) { @@ -24,7 +34,7 @@ export function localFileToDataUri(filePath: string, maxBytes?: number): string } } const ext = extname(filePath).toLowerCase(); - const mime = IMAGE_MIME_TYPES[ext] || 'image/jpeg'; + const mime = mimeOverride || IMAGE_MIME_TYPES[ext] || 'image/jpeg'; const data = readFileSync(filePath); return `data:${mime};base64,${data.toString('base64')}`; } @@ -37,18 +47,56 @@ export function resolveImageInput(input: string, maxBytes?: number): string { const MAX_IMAGE_SIZE_BYTES = 50 * 1024 * 1024; -export async function toDataUri(image: string): Promise { - if (image.startsWith('data:')) return image; +export async function toDataUri(image: string, opts?: ImageValidationOptions): Promise { + if (image.startsWith('data:')) { + if (opts) { + const mime = /^data:([^;,]+)[;,]/.exec(image)?.[1]; + if (!mime || !opts.allowedMediaTypes.includes(mime)) { + throw new CLIError( + `Unsupported image type "${mime ?? 'unknown'}". Supported: ${opts.allowedMediaTypes.join(', ')}`, + ExitCode.USAGE, + ); + } + const size = dataUriDecodedSize(image); + if (size !== undefined && size > opts.maxBytes) { + throw new CLIError( + `Image too large (${(size / 1024 / 1024).toFixed(1)} MB). Maximum is ${(opts.maxBytes / 1024 / 1024).toFixed(0)} MB.`, + ExitCode.USAGE, + ); + } + } + return image; + } if (image.startsWith('http://') || image.startsWith('https://')) { const res = await fetch(image); if (!res.ok) throw new CLIError(`Failed to download image: HTTP ${res.status}`, ExitCode.GENERAL); const contentType = res.headers.get('content-type') || 'image/jpeg'; const mime = contentType.split(';')[0]!.trim(); + const maxBytes = opts?.maxBytes ?? MAX_IMAGE_SIZE_BYTES; + + if (opts) { + if (!opts.allowedMediaTypes.includes(mime)) { + throw new CLIError( + `Unsupported image type "${mime}". Supported: ${opts.allowedMediaTypes.join(', ')}`, + ExitCode.USAGE, + ); + } + // content-length can lie or be absent, but checking it first avoids + // buffering an oversized body just to reject it a moment later. + const contentLength = Number(res.headers.get('content-length')); + if (contentLength > maxBytes) { + throw new CLIError( + `Image too large (${(contentLength / 1024 / 1024).toFixed(1)} MB). Maximum is ${(maxBytes / 1024 / 1024).toFixed(0)} MB.`, + ExitCode.USAGE, + ); + } + } + const buf = await res.arrayBuffer(); - if (buf.byteLength > MAX_IMAGE_SIZE_BYTES) { + if (buf.byteLength > maxBytes) { throw new CLIError( - `Image too large (${(buf.byteLength / 1024 / 1024).toFixed(1)} MB). Maximum is 50 MB.`, + `Image too large (${(buf.byteLength / 1024 / 1024).toFixed(1)} MB). Maximum is ${(maxBytes / 1024 / 1024).toFixed(0)} MB.`, ExitCode.USAGE, ); } @@ -57,6 +105,41 @@ export async function toDataUri(image: string): Promise { if (!existsSync(image)) throw new CLIError(`File not found: ${image}`, ExitCode.USAGE); const ext = extname(image).toLowerCase(); - if (!IMAGE_MIME_TYPES[ext]) throw new CLIError(`Unsupported image format "${ext}". Supported: jpg, jpeg, png, webp`, ExitCode.USAGE); + const mime = opts + ? (IMAGE_MIME_TYPES[ext] ?? (ext === '.gif' ? 'image/gif' : undefined)) + : IMAGE_MIME_TYPES[ext]; + if (!mime || (opts && !opts.allowedMediaTypes.includes(mime))) { + throw new CLIError( + `Unsupported image format "${ext}". Supported: ${opts ? opts.allowedMediaTypes.join(', ') : 'jpg, jpeg, png, webp'}`, + ExitCode.USAGE, + ); + } + if (opts) { + const size = statSync(image).size; + if (size > opts.maxBytes) { + throw new CLIError( + `Image too large (${(size / 1024 / 1024).toFixed(1)} MB). Maximum is ${(opts.maxBytes / 1024 / 1024).toFixed(0)} MB.`, + ExitCode.USAGE, + ); + } + return localFileToDataUri(image, undefined, mime); + } return localFileToDataUri(image); } + +/** + * Convert a path / URL / data URI into an Anthropic-shaped image content block. + * The Messages API rejects the OpenAI `image_url` shape, so callers targeting + * `/anthropic/v1/messages` must use this instead of a raw data URI. + */ +export async function toImageBlock(image: string, opts?: ImageValidationOptions): Promise { + const uri = await toDataUri(image, opts); + const match = /^data:([^;,]+);base64,(.*)$/s.exec(uri); + if (!match) { + throw new CLIError( + `Unsupported image source "${image}": expected a base64 data URI, file path, or http(s) URL.`, + ExitCode.USAGE, + ); + } + return { type: 'image', source: { type: 'base64', media_type: match[1]!, data: match[2]! } }; +} diff --git a/test/commands/text/chat.test.ts b/test/commands/text/chat.test.ts index 2c592b6..c67a048 100644 --- a/test/commands/text/chat.test.ts +++ b/test/commands/text/chat.test.ts @@ -1,4 +1,7 @@ import { describe, it, expect, afterEach } from 'bun:test'; +import { mkdtempSync, writeFileSync, truncateSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; import { createMockServer, jsonResponse, sseResponse, type MockServer } from '../../helpers/mock-server'; import textChatResponse from '../../fixtures/text-chat-response.json'; import type { Config } from '../../../src/config/schema'; @@ -302,4 +305,176 @@ describe('text chat command', () => { console.log = originalLog; } }); + + describe('--image', () => { + // 1x1 transparent PNG + const PNG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + const dir = mkdtempSync(join(tmpdir(), 'mmx-chat-image-')); + const imgA = join(dir, 'a.png'); + const imgB = join(dir, 'b.png'); + writeFileSync(imgA, Buffer.from(PNG_BASE64, 'base64')); + writeFileSync(imgB, Buffer.from(PNG_BASE64, 'base64')); + + const baseConfig: Config = { + apiKey: 'test-key', + region: 'global' as const, + baseUrl: 'https://api.mmx.io', + output: 'json', + timeout: 10, + verbose: false, + quiet: false, + noColor: true, + yes: false, + dryRun: true, + nonInteractive: true, + async: false, + }; + + const baseFlags = { + quiet: false, + verbose: false, + noColor: true, + yes: false, + dryRun: true, + help: false, + nonInteractive: true, + async: false, + }; + + async function dryRunRequest(config: Config, flags: Record) { + const { default: chatCommand } = await import('../../../src/commands/text/chat'); + const originalLog = console.log; + let output = ''; + console.log = (msg: string) => { output += msg; }; + try { + await chatCommand.execute(config, { ...baseFlags, ...flags } as never); + } finally { + console.log = originalLog; + } + return JSON.parse(output).request; + } + + it('appends Anthropic-shaped image blocks to the user message', async () => { + const request = await dryRunRequest(baseConfig, { + message: ['What is this?'], + image: [imgA], + }); + + expect(request.messages).toHaveLength(1); + expect(request.messages[0].content).toEqual([ + { type: 'text', text: 'What is this?' }, + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG_BASE64 } }, + ]); + }); + + it('supports multiple images in one message', async () => { + const request = await dryRunRequest(baseConfig, { + message: ['Compare these.'], + image: [imgA, imgB], + }); + + const blocks = request.messages[0].content; + expect(blocks).toHaveLength(3); + expect(blocks.filter((b: { type: string }) => b.type === 'image')).toHaveLength(2); + }); + + it('sends images with no --message', async () => { + const request = await dryRunRequest(baseConfig, { image: [imgA] }); + + expect(request.messages).toHaveLength(1); + expect(request.messages[0].role).toBe('user'); + expect(request.messages[0].content).toEqual([ + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG_BASE64 } }, + ]); + }); + + it('overrides a text-only defaultTextModel with MiniMax-M3', async () => { + const request = await dryRunRequest( + { ...baseConfig, defaultTextModel: 'MiniMax-Text-01' }, + { message: ['What is this?'], image: [imgA] }, + ); + + expect(request.model).toBe('MiniMax-M3'); + }); + + it('still honours an explicit --model', async () => { + const request = await dryRunRequest( + { ...baseConfig, defaultTextModel: 'MiniMax-Text-01' }, + { message: ['What is this?'], image: [imgA], model: 'MiniMax-VL-01' }, + ); + + expect(request.model).toBe('MiniMax-VL-01'); + }); + + it('errors on a missing image file', async () => { + await expect( + dryRunRequest(baseConfig, { message: ['hi'], image: [join(dir, 'nope.png')] }), + ).rejects.toThrow(/File not found/); + }); + + it('rejects an oversized local image', async () => { + const big = join(dir, 'big.png'); + writeFileSync(big, ''); + truncateSync(big, 11 * 1024 * 1024); + + await expect( + dryRunRequest(baseConfig, { message: ['hi'], image: [big] }), + ).rejects.toThrow(/too large/i); + }); + + it('rejects HEIC images for chat', async () => { + const heic = join(dir, 'photo.heic'); + writeFileSync(heic, Buffer.from('not really heic')); + + await expect( + dryRunRequest(baseConfig, { message: ['hi'], image: [heic] }), + ).rejects.toThrow(/Unsupported image format/); + }); + + it('accepts GIF images for chat', async () => { + const gif = join(dir, 'a.gif'); + writeFileSync(gif, Buffer.from('GIF89a')); + + const request = await dryRunRequest(baseConfig, { message: ['hi'], image: [gif] }); + const block = request.messages[0].content.find((b: { type: string }) => b.type === 'image'); + expect(block.source.media_type).toBe('image/gif'); + }); + + it('rejects a remote image over 10 MB via content-length', async () => { + const imgServer = createMockServer({ + routes: { + '/big.png': () => new Response(Buffer.alloc(11 * 1024 * 1024), { + headers: { 'Content-Type': 'image/png' }, + }), + }, + }); + + try { + await expect( + dryRunRequest(baseConfig, { message: ['hi'], image: [`${imgServer.url}/big.png`] }), + ).rejects.toThrow(/too large/i); + } finally { + imgServer.close(); + } + }); + + it('rejects a data: URI image over the per-image cap', async () => { + const oversized = 'A'.repeat(Math.ceil((11 * 1024 * 1024) / 3) * 4); + + await expect( + dryRunRequest(baseConfig, { message: ['hi'], image: [`data:image/png;base64,${oversized}`] }), + ).rejects.toThrow(/too large/i); + }); + + it('rejects when images cumulatively exceed the 64 MB request cap', async () => { + // ~7.15 MB decoded each — under the 10 MB per-image cap, but seven of + // them push the whole request past the 64 MB aggregate cap. + const chunk = 'A'.repeat(10_000_000); + const images = Array.from({ length: 7 }, () => `data:image/png;base64,${chunk}`); + + await expect( + dryRunRequest(baseConfig, { message: ['hi'], image: images }), + ).rejects.toThrow(/64 MB/); + }); + }); });