From 1c538067af7a58ed9dff650342011240460fd9d4 Mon Sep 17 00:00:00 2001 From: alejandro-angulo Date: Fri, 6 Jun 2025 23:45:45 -0700 Subject: [PATCH 1/3] Switched to streaming responses --- main.go | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 5310c50..881bac7 100644 --- a/main.go +++ b/main.go @@ -75,7 +75,7 @@ func (a *Agent) Run(ctx context.Context) error { for _, content := range message.Content { switch content.Type { case "text": - fmt.Printf("\u001b[93mClaude\u001b[0m: %s\n", content.Text) + print("\n") case "tool_use": result := a.executeTool(content.ID, content.Name, content.Input) toolResults = append(toolResults, result) @@ -106,7 +106,7 @@ func (a *Agent) executeTool(id, name string, input json.RawMessage) anthropic.Co return anthropic.NewToolResultBlock(id, "tool not found", true) } - fmt.Printf("\u001b[92mtool\u001b[0m: %s(%s)\n", name, input) + fmt.Printf("\n\u001b[92mtool\u001b[0m: %s(%s)\n", name, input) response, err := toolDef.Function(input) if err != nil { return anthropic.NewToolResultBlock(id, err.Error(), true) @@ -126,13 +126,32 @@ func (a *Agent) runInference(ctx context.Context, conversation []anthropic.Messa }) } - message, err := a.client.Messages.New(ctx, anthropic.MessageNewParams{ + stream := a.client.Messages.NewStreaming(ctx, anthropic.MessageNewParams{ Model: anthropic.ModelClaude3_7SonnetLatest, MaxTokens: int64(1024), Messages: conversation, Tools: anthropicTools, }) - return message, err + + print("\u001b[93mClaude\u001b[0m: ") + message := anthropic.Message{} + for stream.Next() { + event := stream.Current() + err := message.Accumulate(event) + if err != nil { + return nil, err + } + + switch eventVariant := event.AsAny().(type) { + case anthropic.ContentBlockDeltaEvent: + switch deltaVariant := eventVariant.Delta.AsAny().(type) { + case anthropic.TextDelta: + print(deltaVariant.Text) + } + } + } + + return &message, stream.Err() } type ToolDefinition struct { From b2e6d24fed8582179a68a89eb9d849a37be1df0d Mon Sep 17 00:00:00 2001 From: alejandro-angulo Date: Sat, 7 Jun 2025 15:08:46 -0700 Subject: [PATCH 2/3] Added base64 encode tool --- main.go | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 881bac7..c0c1425 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "bufio" "context" + "encoding/base64" "encoding/json" "fmt" "os" @@ -24,7 +25,12 @@ func main() { } return scanner.Text(), true } - tools := []ToolDefinition{ReadFileDefinition, ListFilesDefinition, EditFileDefinition} + tools := []ToolDefinition{ + ReadFileDefinition, + ListFilesDefinition, + EditFileDefinition, + Base64EncodeFileDefinition, + } agent := NewAgent(&client, getUserMessage, tools) err := agent.Run(context.TODO()) @@ -336,3 +342,36 @@ func createNewFile(filePath, content string) (string, error) { return fmt.Sprintf("Successfully created file %s", filePath), nil } + +var Base64EncodeFileDefinition = ToolDefinition{ + Name: "base64_encode", + Description: `Generates a base64 encoding of a file. + + This is especially useful when asked to describe an image file (you can use + this get a base64 encoded representation of the image file). +`, + InputSchema: Base64EncodeFileInputSchema, + Function: Base64EncodeFile, +} + +type Base64EncodeFileInput struct { + Path string `json:"path" jsonschema_description:"The path to the image"` +} + +var Base64EncodeFileInputSchema = GenerateSchema[EditFileInput]() + +func Base64EncodeFile(input json.RawMessage) (string, error) { + analyzeImageInput := Base64EncodeFileInput{} + err := json.Unmarshal(input, &analyzeImageInput) + if err != nil { + panic(err) + } + + content, err := os.ReadFile(analyzeImageInput.Path) + if err != nil { + return "", err + } + encoded := base64.StdEncoding.EncodeToString([]byte(content)) + + return encoded, nil +} From ed240e2f40c8d24da33a9437a017727db9e734cc Mon Sep 17 00:00:00 2001 From: alejandro-angulo Date: Sat, 7 Jun 2025 15:46:19 -0700 Subject: [PATCH 3/3] Added webcam tool --- flake.nix | 4 ++++ go.mod | 1 + go.sum | 2 ++ main.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+) diff --git a/flake.nix b/flake.nix index d0978fe..e52490a 100644 --- a/flake.nix +++ b/flake.nix @@ -41,6 +41,10 @@ # https://devenv.sh/reference/options/ languages.go.enable = true; languages.javascript.enable = true; + + packages = with pkgs; [ + opencv + ]; } ]; }; diff --git a/go.mod b/go.mod index 6d7a035..ef98a3f 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.24.2 require ( github.com/anthropics/anthropic-sdk-go v1.3.0 github.com/invopop/jsonschema v0.13.0 + gocv.io/x/gocv v0.41.0 ) require ( diff --git a/go.sum b/go.sum index 6de236b..9b5ca94 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +gocv.io/x/gocv v0.41.0 h1:KM+zRXUP28b6dHfhy+4JxDODbCNQNtLg8kio+YE7TqA= +gocv.io/x/gocv v0.41.0/go.mod h1:zYdWMj29WAEznM3Y8NsU3A0TRq/wR/cy75jeUypThqU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main.go b/main.go index c0c1425..6ab0cdd 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "fmt" "os" "path" @@ -13,6 +14,8 @@ import ( "github.com/anthropics/anthropic-sdk-go" "github.com/invopop/jsonschema" + + "gocv.io/x/gocv" ) func main() { @@ -30,6 +33,7 @@ func main() { ListFilesDefinition, EditFileDefinition, Base64EncodeFileDefinition, + WebcamDefinition, } agent := NewAgent(&client, getUserMessage, tools) @@ -375,3 +379,50 @@ func Base64EncodeFile(input json.RawMessage) (string, error) { return encoded, nil } + +var WebcamDefinition = ToolDefinition{ + Name: "webcam", + Description: `Take a picture using the computer's webcam. + + This way you can see what the user sees and provide a description of what + you see. + `, + InputSchema: WebcamDefinitionInputSchema, + Function: Webcam, +} + +type WebcamDefinitionInput struct{} + +var WebcamDefinitionInputSchema = GenerateSchema[WebcamDefinitionInput]() + +func Webcam(input json.RawMessage) (string, error) { + webcam, err := gocv.OpenVideoCapture(0) + if err != nil { + return "", err + } + defer webcam.Close() + + if !webcam.IsOpened() { + return "", errors.New("Unable to open video capture device") + } + + img := gocv.NewMat() + defer img.Close() + + if ok := webcam.Read(&img); !ok { + return "", errors.New("Cannot read from video capture device") + } + + if img.Empty() { + return "", errors.New("Capture image is empty") + } + + jpegData, err := gocv.IMEncode(".jpg", img) + if err != nil { + return "", err + } + + encoded := base64.StdEncoding.EncodeToString(jpegData.GetBytes()) + + return encoded, nil +}