Added webcam tool

flake.nix

@@ -41,6 +41,10 @@
                 # https://devenv.sh/reference/options/
                 languages.go.enable = true;
                 languages.javascript.enable = true;
+
+                packages = with pkgs; [
+                  opencv
+                ];
               }
             ];
           };

go.mod

@@ -5,6 +5,7 @@ go 1.24.2
 require (
 	github.com/anthropics/anthropic-sdk-go v1.3.0
 	github.com/invopop/jsonschema v0.13.0
+	gocv.io/x/gocv v0.41.0
 )
 
 require (

go.sum

@@ -27,6 +27,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
 github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
+gocv.io/x/gocv v0.41.0 h1:KM+zRXUP28b6dHfhy+4JxDODbCNQNtLg8kio+YE7TqA=
+gocv.io/x/gocv v0.41.0/go.mod h1:zYdWMj29WAEznM3Y8NsU3A0TRq/wR/cy75jeUypThqU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

main.go

@@ -3,7 +3,9 @@ package main
 import (
 	"bufio"
 	"context"
+	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path"
@@ -12,6 +14,8 @@ import (
 
 	"github.com/anthropics/anthropic-sdk-go"
 	"github.com/invopop/jsonschema"
+
+	"gocv.io/x/gocv"
 )
 
 func main() {
@@ -24,7 +28,13 @@ func main() {
 		}
 		return scanner.Text(), true
 	}
-	tools := []ToolDefinition{ReadFileDefinition, ListFilesDefinition, EditFileDefinition}
+	tools := []ToolDefinition{
+		ReadFileDefinition,
+		ListFilesDefinition,
+		EditFileDefinition,
+		Base64EncodeFileDefinition,
+		WebcamDefinition,
+	}
 
 	agent := NewAgent(&client, getUserMessage, tools)
 	err := agent.Run(context.TODO())
@@ -75,7 +85,7 @@ func (a *Agent) Run(ctx context.Context) error {
 		for _, content := range message.Content {
 			switch content.Type {
 			case "text":
-				fmt.Printf("\u001b[93mClaude\u001b[0m: %s\n", content.Text)
+				print("\n")
 			case "tool_use":
 				result := a.executeTool(content.ID, content.Name, content.Input)
 				toolResults = append(toolResults, result)
@@ -106,7 +116,7 @@ func (a *Agent) executeTool(id, name string, input json.RawMessage) anthropic.Co
 		return anthropic.NewToolResultBlock(id, "tool not found", true)
 	}
 
-	fmt.Printf("\u001b[92mtool\u001b[0m: %s(%s)\n", name, input)
+	fmt.Printf("\n\u001b[92mtool\u001b[0m: %s(%s)\n", name, input)
 	response, err := toolDef.Function(input)
 	if err != nil {
 		return anthropic.NewToolResultBlock(id, err.Error(), true)
@@ -126,13 +136,32 @@ func (a *Agent) runInference(ctx context.Context, conversation []anthropic.Messa
 		})
 	}
 
-	message, err := a.client.Messages.New(ctx, anthropic.MessageNewParams{
+	stream := a.client.Messages.NewStreaming(ctx, anthropic.MessageNewParams{
 		Model:     anthropic.ModelClaude3_7SonnetLatest,
 		MaxTokens: int64(1024),
 		Messages:  conversation,
 		Tools:     anthropicTools,
 	})
-	return message, err
+
+	print("\u001b[93mClaude\u001b[0m: ")
+	message := anthropic.Message{}
+	for stream.Next() {
+		event := stream.Current()
+		err := message.Accumulate(event)
+		if err != nil {
+			return nil, err
+		}
+
+		switch eventVariant := event.AsAny().(type) {
+		case anthropic.ContentBlockDeltaEvent:
+			switch deltaVariant := eventVariant.Delta.AsAny().(type) {
+			case anthropic.TextDelta:
+				print(deltaVariant.Text)
+			}
+		}
+	}
+
+	return &message, stream.Err()
 }
 
 type ToolDefinition struct {
@@ -317,3 +346,83 @@ func createNewFile(filePath, content string) (string, error) {
 
 	return fmt.Sprintf("Successfully created file %s", filePath), nil
 }
+
+var Base64EncodeFileDefinition = ToolDefinition{
+	Name: "base64_encode",
+	Description: `Generates a base64 encoding of a file.
+
+	This is especially useful when asked to describe an image file (you can use
+	this get a base64 encoded representation of the image file).
+`,
+	InputSchema: Base64EncodeFileInputSchema,
+	Function:    Base64EncodeFile,
+}
+
+type Base64EncodeFileInput struct {
+	Path string `json:"path" jsonschema_description:"The path to the image"`
+}
+
+var Base64EncodeFileInputSchema = GenerateSchema[EditFileInput]()
+
+func Base64EncodeFile(input json.RawMessage) (string, error) {
+	analyzeImageInput := Base64EncodeFileInput{}
+	err := json.Unmarshal(input, &analyzeImageInput)
+	if err != nil {
+		panic(err)
+	}
+
+	content, err := os.ReadFile(analyzeImageInput.Path)
+	if err != nil {
+		return "", err
+	}
+	encoded := base64.StdEncoding.EncodeToString([]byte(content))
+
+	return encoded, nil
+}
+
+var WebcamDefinition = ToolDefinition{
+	Name: "webcam",
+	Description: `Take a picture using the computer's webcam.
+
+	This way you can see what the user sees and provide a description of what
+	you see.
+	`,
+	InputSchema: WebcamDefinitionInputSchema,
+	Function:    Webcam,
+}
+
+type WebcamDefinitionInput struct{}
+
+var WebcamDefinitionInputSchema = GenerateSchema[WebcamDefinitionInput]()
+
+func Webcam(input json.RawMessage) (string, error) {
+	webcam, err := gocv.OpenVideoCapture(0)
+	if err != nil {
+		return "", err
+	}
+	defer webcam.Close()
+
+	if !webcam.IsOpened() {
+		return "", errors.New("Unable to open video capture device")
+	}
+
+	img := gocv.NewMat()
+	defer img.Close()
+
+	if ok := webcam.Read(&img); !ok {
+		return "", errors.New("Cannot read from video capture device")
+	}
+
+	if img.Empty() {
+		return "", errors.New("Capture image is empty")
+	}
+
+	jpegData, err := gocv.IMEncode(".jpg", img)
+	if err != nil {
+		return "", err
+	}
+
+	encoded := base64.StdEncoding.EncodeToString(jpegData.GetBytes())
+
+	return encoded, nil
+}