Skip to content

APP-10189 - Add logic for passing through annotations from Images #5462

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

APP-10189 - Add logic for passing through annotations from Images #5462

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e240709
Add logic for passing through annotations
tahiyasalam Nov 10, 2025
fb34750
Change to function
tahiyasalam Nov 10, 2025
91d2f7f
Add to NamedImageFromImage
tahiyasalam Nov 10, 2025
a38ecec
Update webcame Images to include annotations
tahiyasalam Nov 10, 2025
e9d8563
Add tests for camera server, client, collectors, Images, and NamedImage
tahiyasalam Nov 11, 2025
f340f9d
Update stream camera tests
tahiyasalam Nov 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 27 additions & 14 deletions components/camera/camera.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,32 +87,35 @@ type Properties struct {

// NamedImage is a struct that associates the source from where the image came from to the Image.
type NamedImage struct {
data []byte
img image.Image
SourceName string
mimeType string
data []byte
img image.Image
SourceName string
mimeType string
annotations data.Annotations
}

// NamedImageFromBytes constructs a NamedImage from a byte slice, source name, and mime type.
func NamedImageFromBytes(data []byte, sourceName, mimeType string) (NamedImage, error) {
// NamedImageFromBytes constructs a NamedImage from a byte slice, source name, mime type, and annotations.
func NamedImageFromBytes(data []byte, sourceName, mimeType string, annotations data.Annotations,
) (NamedImage, error) {
if data == nil {
return NamedImage{}, fmt.Errorf("must provide image bytes to construct a named image from bytes")
}
if mimeType == "" {
return NamedImage{}, fmt.Errorf("must provide a mime type to construct a named image")
}
return NamedImage{data: data, SourceName: sourceName, mimeType: mimeType}, nil
return NamedImage{data: data, SourceName: sourceName, mimeType: mimeType, annotations: annotations}, nil
}

// NamedImageFromImage constructs a NamedImage from an image.Image, source name, and mime type.
func NamedImageFromImage(img image.Image, sourceName, mimeType string) (NamedImage, error) {
// NamedImageFromImage constructs a NamedImage from an image.Image, source name, mime type, and annotations.
func NamedImageFromImage(img image.Image, sourceName, mimeType string, annotations data.Annotations,
) (NamedImage, error) {
if img == nil {
return NamedImage{}, fmt.Errorf("must provide image to construct a named image from image")
}
if mimeType == "" {
return NamedImage{}, fmt.Errorf("must provide a mime type to construct a named image")
}
return NamedImage{img: img, SourceName: sourceName, mimeType: mimeType}, nil
return NamedImage{img: img, SourceName: sourceName, mimeType: mimeType, annotations: annotations}, nil
}

// Image returns the image.Image of the NamedImage.
Expand Down Expand Up @@ -164,9 +167,16 @@ func (ni *NamedImage) MimeType() string {
return ni.mimeType
}

// ImageMetadata contains useful information about returned image bytes such as its mimetype.
// Annotations returns the annotations of the NamedImage.
func (ni *NamedImage) Annotations() data.Annotations {
Copy link
Member

@hexbabe hexbabe Nov 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

any scenarios where RDK may want to set annotations after initialization?

return ni.annotations
}

// ImageMetadata contains useful information about returned image bytes such as its mimetype
// and any annotations associated with the image.
type ImageMetadata struct {
MimeType string
MimeType string
Annotations data.Annotations
}

// A Camera is a resource that can capture frames.
Expand Down Expand Up @@ -284,12 +294,14 @@ func GetImageFromGetImages(

var img image.Image
var mimeType string
var annotations data.Annotations
if sourceName == nil {
img, err = namedImages[0].Image(ctx)
if err != nil {
return nil, ImageMetadata{}, fmt.Errorf("could not get image from named image: %w", err)
}
mimeType = namedImages[0].MimeType()
annotations = namedImages[0].Annotations()
} else {
for _, i := range namedImages {
if i.SourceName == *sourceName {
Expand All @@ -298,6 +310,7 @@ func GetImageFromGetImages(
return nil, ImageMetadata{}, fmt.Errorf("could not get image from named image: %w", err)
}
mimeType = i.MimeType()
annotations = i.Annotations()
break
}
}
Expand All @@ -314,7 +327,7 @@ func GetImageFromGetImages(
if err != nil {
return nil, ImageMetadata{}, fmt.Errorf("could not encode image with encoding %s: %w", mimeType, err)
}
return imgBytes, ImageMetadata{MimeType: mimeType}, nil
return imgBytes, ImageMetadata{MimeType: mimeType, Annotations: annotations}, nil
}

// GetImagesFromGetImage will be deprecated after RSDK-11726.
Expand Down Expand Up @@ -345,7 +358,7 @@ func GetImagesFromGetImage(
logger.Warnf("requested mime type %s, but received %s", mimeType, resMimetype)
}

namedImg, err := NamedImageFromBytes(resBytes, "", resMetadata.MimeType)
namedImg, err := NamedImageFromBytes(resBytes, "", resMetadata.MimeType, resMetadata.Annotations)
if err != nil {
return nil, resource.ResponseMetadata{}, fmt.Errorf("could not create named image: %w", err)
}
Expand Down
77 changes: 53 additions & 24 deletions components/camera/camera_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"go.viam.com/utils/artifact"

"go.viam.com/rdk/components/camera"
"go.viam.com/rdk/data"
"go.viam.com/rdk/logging"
"go.viam.com/rdk/pointcloud"
"go.viam.com/rdk/resource"
Expand Down Expand Up @@ -281,18 +282,20 @@ func verifyDecodedImage(t *testing.T, imgBytes []byte, mimeType string, original
func TestGetImageFromGetImages(t *testing.T) {
testImg1 := image.NewRGBA(image.Rect(0, 0, 100, 100))
testImg2 := image.NewRGBA(image.Rect(0, 0, 200, 200))
annotations1 := data.Annotations{BoundingBoxes: []data.BoundingBox{{Label: "object1"}}}
annotations2 := data.Annotations{Classifications: []data.Classification{{Label: "object2"}}}

rgbaCam := inject.NewCamera("rgba_cam")
rgbaCam.ImagesFunc = func(
ctx context.Context,
filterSourceNames []string,
extra map[string]interface{},
) ([]camera.NamedImage, resource.ResponseMetadata, error) {
namedImg1, err := camera.NamedImageFromImage(testImg1, source1Name, rutils.MimeTypeRawRGBA)
namedImg1, err := camera.NamedImageFromImage(testImg1, source1Name, rutils.MimeTypeRawRGBA, annotations1)
if err != nil {
return nil, resource.ResponseMetadata{}, err
}
namedImg2, err := camera.NamedImageFromImage(testImg2, source2Name, rutils.MimeTypeRawRGBA)
namedImg2, err := camera.NamedImageFromImage(testImg2, source2Name, rutils.MimeTypeRawRGBA, annotations2)
if err != nil {
return nil, resource.ResponseMetadata{}, err
}
Expand All @@ -309,7 +312,7 @@ func TestGetImageFromGetImages(t *testing.T) {
filterSourceNames []string,
extra map[string]interface{},
) ([]camera.NamedImage, resource.ResponseMetadata, error) {
namedImg, err := camera.NamedImageFromImage(dm, source1Name, rutils.MimeTypeRawDepth)
namedImg, err := camera.NamedImageFromImage(dm, source1Name, rutils.MimeTypeRawDepth, annotations1)
if err != nil {
return nil, resource.ResponseMetadata{}, err
}
Expand All @@ -322,6 +325,7 @@ func TestGetImageFromGetImages(t *testing.T) {
test.That(t, err, test.ShouldBeNil)
test.That(t, img, test.ShouldNotBeNil)
test.That(t, metadata.MimeType, test.ShouldEqual, rutils.MimeTypeRawRGBA)
test.That(t, metadata.Annotations, test.ShouldResemble, annotations1)
verifyDecodedImage(t, img, rutils.MimeTypeRawRGBA, testImg1)
})

Expand Down Expand Up @@ -358,7 +362,7 @@ func TestGetImageFromGetImages(t *testing.T) {
filterSourceNames []string,
extra map[string]interface{},
) ([]camera.NamedImage, resource.ResponseMetadata, error) {
namedImg, err := camera.NamedImageFromImage(nil, source1Name, rutils.MimeTypeRawRGBA)
namedImg, err := camera.NamedImageFromImage(nil, source1Name, rutils.MimeTypeRawRGBA, data.Annotations{})
if err != nil {
return nil, resource.ResponseMetadata{}, err
}
Expand Down Expand Up @@ -472,6 +476,7 @@ func TestImages(t *testing.T) {
ctx := context.Background()
t.Run("extra param", func(t *testing.T) {
respImg := image.NewRGBA(image.Rect(0, 0, 10, 10))
annotations1 := data.Annotations{BoundingBoxes: []data.BoundingBox{{Label: "annotation1"}}}

cam := inject.NewCamera("extra_param_cam")
cam.ImagesFunc = func(
Expand All @@ -482,7 +487,7 @@ func TestImages(t *testing.T) {
if len(extra) == 0 {
return nil, resource.ResponseMetadata{}, fmt.Errorf("extra parameters required")
}
namedImg, err := camera.NamedImageFromImage(respImg, source1Name, rutils.MimeTypeRawRGBA)
namedImg, err := camera.NamedImageFromImage(respImg, source1Name, rutils.MimeTypeRawRGBA, annotations1)
if err != nil {
return nil, resource.ResponseMetadata{}, err
}
Expand All @@ -497,6 +502,7 @@ func TestImages(t *testing.T) {
test.That(t, err, test.ShouldBeNil)
test.That(t, rimage.ImagesExactlyEqual(img, respImg), test.ShouldBeTrue)
test.That(t, images[0].SourceName, test.ShouldEqual, source1Name)
test.That(t, images[0].Annotations(), test.ShouldResemble, annotations1)
})

t.Run("error when no extra params", func(t *testing.T) {
Expand All @@ -511,11 +517,15 @@ func TestImages(t *testing.T) {
img2 := rimage.NewEmptyDepthMap(10, 10)
img3 := image.NewNRGBA(image.Rect(0, 0, 30, 30))

namedImg1, err := camera.NamedImageFromImage(img1, source1Name, rutils.MimeTypePNG)
annotations1 := data.Annotations{BoundingBoxes: []data.BoundingBox{{Label: "object1"}}}
annotations2 := data.Annotations{Classifications: []data.Classification{{Label: "object2"}}}
annotations3 := data.Annotations{BoundingBoxes: []data.BoundingBox{{Label: "object3"}}}

namedImg1, err := camera.NamedImageFromImage(img1, source1Name, rutils.MimeTypePNG, annotations1)
test.That(t, err, test.ShouldBeNil)
namedImg2, err := camera.NamedImageFromImage(img2, source2Name, rutils.MimeTypeRawDepth)
namedImg2, err := camera.NamedImageFromImage(img2, source2Name, rutils.MimeTypeRawDepth, annotations2)
test.That(t, err, test.ShouldBeNil)
namedImg3, err := camera.NamedImageFromImage(img3, source3Name, rutils.MimeTypeJPEG)
namedImg3, err := camera.NamedImageFromImage(img3, source3Name, rutils.MimeTypeJPEG, annotations3)
test.That(t, err, test.ShouldBeNil)

allImgs := []camera.NamedImage{namedImg1, namedImg2, namedImg3}
Expand Down Expand Up @@ -582,6 +592,7 @@ func TestImages(t *testing.T) {
test.That(t, len(imgs), test.ShouldEqual, 1)
test.That(t, imgs[0].SourceName, test.ShouldEqual, source2Name)
test.That(t, imgs[0].MimeType(), test.ShouldEqual, rutils.MimeTypeRawDepth)
test.That(t, imgs[0].Annotations(), test.ShouldResemble, annotations2)
Copy link
Member

@hexbabe hexbabe Nov 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

curious why resemble?

bc it's deep value equality not pointer equality?

img, err := imgs[0].Image(ctx)
test.That(t, err, test.ShouldBeNil)
test.That(t, rimage.ImagesExactlyEqual(img, img2), test.ShouldBeTrue)
Expand All @@ -599,11 +610,13 @@ func TestImages(t *testing.T) {
test.That(t, returnedSources[source1Name], test.ShouldBeTrue)

test.That(t, imgs[0].MimeType(), test.ShouldEqual, rutils.MimeTypeJPEG)
test.That(t, imgs[0].Annotations(), test.ShouldResemble, annotations3)
img, err := imgs[0].Image(ctx)
test.That(t, err, test.ShouldBeNil)
test.That(t, rimage.ImagesExactlyEqual(img, img3), test.ShouldBeTrue)

test.That(t, imgs[1].MimeType(), test.ShouldEqual, rutils.MimeTypePNG)
test.That(t, imgs[1].Annotations(), test.ShouldResemble, annotations1)
img, err = imgs[1].Image(ctx)
test.That(t, err, test.ShouldBeNil)
test.That(t, rimage.ImagesExactlyEqual(img, img1), test.ShouldBeTrue)
Expand Down Expand Up @@ -632,47 +645,53 @@ func TestNamedImage(t *testing.T) {
test.That(t, err, test.ShouldBeNil)
badBytes := []byte("trust bro i'm an image ong")
sourceName := "test_source"

annotations := data.Annotations{
BoundingBoxes: []data.BoundingBox{
{Label: "object1"},
},
}
t.Run("NamedImageFromBytes", func(t *testing.T) {
t.Run("success", func(t *testing.T) {
ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG)
ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeNil)
test.That(t, ni.SourceName, test.ShouldEqual, sourceName)
test.That(t, ni.MimeType(), test.ShouldEqual, rutils.MimeTypePNG)
test.That(t, ni.Annotations(), test.ShouldResemble, annotations)
})
t.Run("error on nil data", func(t *testing.T) {
_, err := camera.NamedImageFromBytes(nil, sourceName, rutils.MimeTypePNG)
_, err := camera.NamedImageFromBytes(nil, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeError, errors.New("must provide image bytes to construct a named image from bytes"))
})
t.Run("error on empty mime type", func(t *testing.T) {
_, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, "")
_, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, "", annotations)
test.That(t, err, test.ShouldBeError, errors.New("must provide a mime type to construct a named image"))
})
})

t.Run("NamedImageFromImage", func(t *testing.T) {
t.Run("success", func(t *testing.T) {
ni, err := camera.NamedImageFromImage(testImg, sourceName, rutils.MimeTypePNG)
ni, err := camera.NamedImageFromImage(testImg, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeNil)
test.That(t, ni.SourceName, test.ShouldEqual, sourceName)
test.That(t, ni.MimeType(), test.ShouldEqual, rutils.MimeTypePNG)
test.That(t, ni.Annotations(), test.ShouldResemble, annotations)
img, err := ni.Image(ctx)
test.That(t, err, test.ShouldBeNil)
test.That(t, rimage.ImagesExactlyEqual(img, testImg), test.ShouldBeTrue)
})
t.Run("error on nil image", func(t *testing.T) {
_, err := camera.NamedImageFromImage(nil, sourceName, rutils.MimeTypePNG)
_, err := camera.NamedImageFromImage(nil, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeError, errors.New("must provide image to construct a named image from image"))
})
t.Run("error on empty mime type", func(t *testing.T) {
_, err := camera.NamedImageFromImage(testImg, sourceName, "")
_, err := camera.NamedImageFromImage(testImg, sourceName, "", annotations)
test.That(t, err, test.ShouldBeError, errors.New("must provide a mime type to construct a named image"))
})
})

t.Run("Image method", func(t *testing.T) {
t.Run("when image is already populated, it should return the image and cache it", func(t *testing.T) {
ni, err := camera.NamedImageFromImage(testImg, sourceName, rutils.MimeTypePNG)
ni, err := camera.NamedImageFromImage(testImg, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeNil)
img, err := ni.Image(ctx)
test.That(t, err, test.ShouldBeNil)
Expand All @@ -685,7 +704,7 @@ func TestNamedImage(t *testing.T) {
})

t.Run("when only data is populated, it should decode the data and cache it", func(t *testing.T) {
ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG)
ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeNil)

// first call should decode
Expand All @@ -707,15 +726,15 @@ func TestNamedImage(t *testing.T) {
})

t.Run("error when data is invalid", func(t *testing.T) {
ni, err := camera.NamedImageFromBytes(badBytes, sourceName, rutils.MimeTypePNG)
ni, err := camera.NamedImageFromBytes(badBytes, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeNil)
_, err = ni.Image(ctx)
test.That(t, err, test.ShouldBeError)
test.That(t, err.Error(), test.ShouldEqual, "could not decode image config: image: unknown format")
})

t.Run("error when mime type mismatches and decode fails", func(t *testing.T) {
ni, err := camera.NamedImageFromBytes(testImgJPEGBytes, sourceName, rutils.MimeTypePNG)
ni, err := camera.NamedImageFromBytes(testImgJPEGBytes, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeNil)
_, err = ni.Image(ctx)
test.That(t, err, test.ShouldBeError)
Expand All @@ -726,7 +745,7 @@ func TestNamedImage(t *testing.T) {
corruptedPNGBytes := append([]byte(nil), testImgPNGBytes...)
corruptedPNGBytes[len(corruptedPNGBytes)-5] = 0 // corrupt it

ni, err := camera.NamedImageFromBytes(corruptedPNGBytes, sourceName, rutils.MimeTypePNG)
ni, err := camera.NamedImageFromBytes(corruptedPNGBytes, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeNil)
_, err = ni.Image(ctx)
test.That(t, err, test.ShouldBeError)
Expand All @@ -736,7 +755,7 @@ func TestNamedImage(t *testing.T) {

t.Run("Bytes method", func(t *testing.T) {
t.Run("when data is already populated, it should return the data and cache it", func(t *testing.T) {
ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG)
ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeNil)
data, err := ni.Bytes(ctx)
test.That(t, err, test.ShouldBeNil)
Expand All @@ -749,7 +768,7 @@ func TestNamedImage(t *testing.T) {
})

t.Run("when only image is populated, it should encode the image and cache it", func(t *testing.T) {
ni, err := camera.NamedImageFromImage(testImg, sourceName, rutils.MimeTypePNG)
ni, err := camera.NamedImageFromImage(testImg, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeNil)

// first call should encode
Expand All @@ -771,7 +790,7 @@ func TestNamedImage(t *testing.T) {
})

t.Run("error when encoding fails", func(t *testing.T) {
ni, err := camera.NamedImageFromImage(testImg, sourceName, "bad-mime-type")
ni, err := camera.NamedImageFromImage(testImg, sourceName, "bad-mime-type", annotations)
test.That(t, err, test.ShouldBeNil)
_, err = ni.Bytes(ctx)
test.That(t, err, test.ShouldBeError)
Expand All @@ -781,8 +800,18 @@ func TestNamedImage(t *testing.T) {
})

t.Run("MimeType method", func(t *testing.T) {
ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG)
ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeNil)
test.That(t, ni.MimeType(), test.ShouldEqual, rutils.MimeTypePNG)
})

t.Run("Annotations method", func(t *testing.T) {
ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, annotations)
test.That(t, err, test.ShouldBeNil)
test.That(t, ni.Annotations(), test.ShouldResemble, annotations)

ni, err = camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{})
test.That(t, err, test.ShouldBeNil)
test.That(t, ni.Annotations().Empty(), test.ShouldBeTrue)
})
}
3 changes: 2 additions & 1 deletion components/camera/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"golang.org/x/exp/slices"

"go.viam.com/rdk/components/camera/rtppassthrough"
"go.viam.com/rdk/data"
"go.viam.com/rdk/gostream"
"go.viam.com/rdk/grpc"
"go.viam.com/rdk/logging"
Expand Down Expand Up @@ -236,7 +237,7 @@ func (c *client) Images(
// format. We will remove this once we remove the format field from the proto.
img.MimeType = utils.FormatToMimeType[img.Format]
}
namedImg, err := NamedImageFromBytes(img.Image, img.SourceName, img.MimeType)
namedImg, err := NamedImageFromBytes(img.Image, img.SourceName, img.MimeType, data.AnnotationsFromProto(img.Annotations))
if err != nil {
return nil, resource.ResponseMetadata{}, err
}
Expand Down
Loading
Loading