diff --git a/robot/web/stream/camera/camera.go b/robot/web/stream/camera/camera.go index a3c4d171645..8a80be8d0b7 100644 --- a/robot/web/stream/camera/camera.go +++ b/robot/web/stream/camera/camera.go @@ -2,16 +2,58 @@ package camera import ( + "bytes" "context" + "fmt" "image" "github.com/pion/mediadevices/pkg/prop" "go.viam.com/rdk/components/camera" "go.viam.com/rdk/gostream" + "go.viam.com/rdk/rimage" "go.viam.com/rdk/robot" + rutils "go.viam.com/rdk/utils" ) +// StreamableImageMIMETypes is the set of all mime types the stream server supports. +// StreamableImageMIMETypes represents all mime types the stream server supports. +// The order of the slice defines the priority of the mime types. +var StreamableImageMIMETypes = []string{ + rutils.MimeTypeJPEG, + rutils.MimeTypePNG, + rutils.MimeTypeRawRGBA, + rutils.MimeTypeRawDepth, + rutils.MimeTypeQOI, +} + +// cropToEvenDimensions crops an image to even dimensions for x264 compatibility. +// x264 only supports even resolutions. This ensures all streamed images work with x264. +func cropToEvenDimensions(img image.Image) (image.Image, error) { + if img, ok := img.(*rimage.LazyEncodedImage); ok { + if err := img.DecodeConfig(); err != nil { + return nil, err + } + } + + hasOddWidth := img.Bounds().Dx()%2 != 0 + hasOddHeight := img.Bounds().Dy()%2 != 0 + if !hasOddWidth && !hasOddHeight { + return img, nil + } + + rImg := rimage.ConvertImage(img) + newWidth := rImg.Width() + newHeight := rImg.Height() + if hasOddWidth { + newWidth-- + } + if hasOddHeight { + newHeight-- + } + return rImg.SubImage(image.Rect(0, 0, newWidth, newHeight)), nil +} + // Camera returns the camera from the robot (derived from the stream) or // an error if it has no camera. func Camera(robot robot.Robot, stream gostream.Stream) (camera.Camera, error) { @@ -24,14 +66,119 @@ func Camera(robot robot.Robot, stream gostream.Stream) (camera.Camera, error) { return cam, nil } +// FormatStringToMimeType takes a format string returned from image.DecodeConfig and converts +// it to a utils mime type. +func FormatStringToMimeType(format string) string { + return fmt.Sprintf("image/%s", format) +} + +// GetStreamableNamedImageFromCamera returns the first named image it finds from the camera that is supported for streaming. +// It prioritizes images based on the order of StreamableImageMIMETypes. +func GetStreamableNamedImageFromCamera(ctx context.Context, cam camera.Camera) (camera.NamedImage, error) { + namedImages, _, err := cam.Images(ctx, nil, nil) + if err != nil { + return camera.NamedImage{}, err + } + if len(namedImages) == 0 { + return camera.NamedImage{}, fmt.Errorf("no images received for camera %q", cam.Name()) + } + + for _, streamableMimeType := range StreamableImageMIMETypes { + for _, namedImage := range namedImages { + if namedImage.MimeType() == streamableMimeType { + return namedImage, nil + } + + imgBytes, err := namedImage.Bytes(ctx) + if err != nil { + continue + } + + _, format, err := image.DecodeConfig(bytes.NewReader(imgBytes)) + if err != nil { + continue + } + if FormatStringToMimeType(format) == streamableMimeType { + return namedImage, nil + } + } + } + return camera.NamedImage{}, fmt.Errorf("no images were found with a streamable mime type for camera %q", cam.Name()) +} + +// getImageBySourceName retrieves a specific named image from the camera by source name. +func getImageBySourceName(ctx context.Context, cam camera.Camera, sourceName string) (camera.NamedImage, error) { + filterSourceNames := []string{sourceName} + namedImages, _, err := cam.Images(ctx, filterSourceNames, nil) + if err != nil { + return camera.NamedImage{}, err + } + + switch len(namedImages) { + case 0: + return camera.NamedImage{}, fmt.Errorf("no images found for requested source name: %s", sourceName) + case 1: + namedImage := namedImages[0] + if namedImage.SourceName != sourceName { + return camera.NamedImage{}, fmt.Errorf("mismatched source name: requested %q, got %q", sourceName, namedImage.SourceName) + } + return namedImage, nil + default: + // At this point, multiple images were returned. This can happen if the camera is on an older version of the API and does not support + // filtering by source name, or if there is a bug in the camera resource's filtering logic. In this unfortunate case, we'll match the + // requested source name and tank the performance costs. + responseSourceNames := []string{} + for _, namedImage := range namedImages { + if namedImage.SourceName == sourceName { + return namedImage, nil + } + responseSourceNames = append(responseSourceNames, namedImage.SourceName) + } + return camera.NamedImage{}, + fmt.Errorf("no matching source name found for multiple returned images: requested %q, got %q", sourceName, responseSourceNames) + } +} + // VideoSourceFromCamera converts a camera resource into a gostream VideoSource. // This is useful for streaming video from a camera resource. func VideoSourceFromCamera(ctx context.Context, cam camera.Camera) (gostream.VideoSource, error) { + // The reader callback uses a small state machine to determine which image to request from the camera. + // A `sourceName` is used to track the selected image source. On the first call, `sourceName` is nil, + // so the first available streamable image is chosen. On subsequent successful calls, the same `sourceName` + // is used. If any errors occur while getting an image, `sourceName` is reset to nil, and the selection + // process starts over on the next call. This allows the stream to recover if a source becomes unavailable. + var sourceName *string reader := gostream.VideoReaderFunc(func(ctx context.Context) (image.Image, func(), error) { - img, err := camera.DecodeImageFromCamera(ctx, "", nil, cam) + var respNamedImage camera.NamedImage + + if sourceName == nil { + namedImage, err := GetStreamableNamedImageFromCamera(ctx, cam) + if err != nil { + return nil, func() {}, err + } + respNamedImage = namedImage + sourceName = &namedImage.SourceName + } else { + var err error + respNamedImage, err = getImageBySourceName(ctx, cam, *sourceName) + if err != nil { + sourceName = nil + return nil, func() {}, err + } + } + + img, err := respNamedImage.Image(ctx) if err != nil { + sourceName = nil return nil, func() {}, err } + + img, err = cropToEvenDimensions(img) + if err != nil { + sourceName = nil + return nil, func() {}, err + } + return img, func() {}, nil }) diff --git a/robot/web/stream/camera/camera_test.go b/robot/web/stream/camera/camera_test.go index 1b9b899be3f..bac3d380cbc 100644 --- a/robot/web/stream/camera/camera_test.go +++ b/robot/web/stream/camera/camera_test.go @@ -1,27 +1,285 @@ package camera_test import ( + "bytes" "context" + "errors" + "fmt" "image" "testing" "go.viam.com/test" "go.viam.com/rdk/components/camera" + "go.viam.com/rdk/data" "go.viam.com/rdk/gostream" + "go.viam.com/rdk/resource" "go.viam.com/rdk/rimage" camerautils "go.viam.com/rdk/robot/web/stream/camera" "go.viam.com/rdk/testutils/inject" "go.viam.com/rdk/utils" ) +func TestFormatStringToMimeType(t *testing.T) { + testRect := image.Rect(0, 0, 100, 100) + + type testCase struct { + mimeType string + createImage func() image.Image + expectedFormat string + } + + testCases := []testCase{ + { + mimeType: utils.MimeTypeRawRGBA, + createImage: func() image.Image { + return image.NewRGBA(testRect) + }, + expectedFormat: "vnd.viam.rgba", + }, + { + mimeType: utils.MimeTypeRawDepth, + createImage: func() image.Image { + return image.NewGray16(testRect) + }, + expectedFormat: "vnd.viam.dep", + }, + { + mimeType: utils.MimeTypeJPEG, + createImage: func() image.Image { + return image.NewRGBA(testRect) + }, + expectedFormat: "jpeg", + }, + { + mimeType: utils.MimeTypePNG, + createImage: func() image.Image { + return image.NewRGBA(testRect) + }, + expectedFormat: "png", + }, + { + mimeType: utils.MimeTypeQOI, + createImage: func() image.Image { + return image.NewRGBA(testRect) + }, + expectedFormat: "qoi", + }, + } + + for _, tc := range testCases { + t.Run(tc.mimeType, func(t *testing.T) { + sourceImg := tc.createImage() + imgBytes, err := rimage.EncodeImage(context.Background(), sourceImg, tc.mimeType) + test.That(t, err, test.ShouldBeNil) + + _, format, err := image.DecodeConfig(bytes.NewReader(imgBytes)) + test.That(t, err, test.ShouldBeNil) + test.That(t, format, test.ShouldEqual, tc.expectedFormat) + + resultMimeType := camerautils.FormatStringToMimeType(format) + test.That(t, resultMimeType, test.ShouldEqual, tc.mimeType) + + var ok bool + for _, m := range camerautils.StreamableImageMIMETypes { + if m == resultMimeType { + ok = true + break + } + } + test.That(t, ok, test.ShouldBeTrue) + }) + } +} + +func TestGetStreamableNamedImageFromCamera(t *testing.T) { + sourceImg := image.NewRGBA(image.Rect(0, 0, 1, 1)) + unstreamableImg, err := camera.NamedImageFromImage(sourceImg, "unstreamable", "image/undefined", data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + streamableImg, err := camera.NamedImageFromImage(sourceImg, "streamable", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + t.Run("no images", func(t *testing.T) { + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return []camera.NamedImage{}, resource.ResponseMetadata{}, nil + }, + } + _, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeError, errors.New(`no images received for camera "::/"`)) + }) + + t.Run("no streamable images", func(t *testing.T) { + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return []camera.NamedImage{unstreamableImg}, resource.ResponseMetadata{}, nil + }, + } + _, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeError, errors.New(`no images were found with a streamable mime type for camera "::/"`)) + }) + + t.Run("one streamable image", func(t *testing.T) { + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return []camera.NamedImage{streamableImg}, resource.ResponseMetadata{}, nil + }, + } + img, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + test.That(t, img.SourceName, test.ShouldEqual, "streamable") + }) + + t.Run("first image is not streamable", func(t *testing.T) { + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return []camera.NamedImage{unstreamableImg, streamableImg}, resource.ResponseMetadata{}, nil + }, + } + img, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + test.That(t, img.SourceName, test.ShouldEqual, "streamable") + }) + + t.Run("camera Images returns an error", func(t *testing.T) { + expectedErr := errors.New("camera error") + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return nil, resource.ResponseMetadata{}, expectedErr + }, + } + _, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeError, expectedErr) + }) + + t.Run("all streamable mime types are accepted", func(t *testing.T) { + sourceImg := image.NewRGBA(image.Rect(0, 0, 1, 1)) + + streamableMIMETypes := camerautils.StreamableImageMIMETypes + + for _, mimeType := range streamableMIMETypes { + t.Run(mimeType, func(t *testing.T) { + streamableImg, err := camera.NamedImageFromImage(sourceImg, "streamable", mimeType, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return []camera.NamedImage{streamableImg}, resource.ResponseMetadata{}, nil + }, + } + img, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + test.That(t, img.SourceName, test.ShouldEqual, "streamable") + test.That(t, img.MimeType(), test.ShouldEqual, mimeType) + }) + } + }) + + t.Run("image.DecodeConfig fails and continues to next streamable image", func(t *testing.T) { + malformedBytes := []byte("malformed") + + malformedNamedImage, err := camera.NamedImageFromBytes(malformedBytes, "malformed", "image/wtf", data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + goodImg := image.NewRGBA(image.Rect(0, 0, 2, 2)) + goodImgBytes, err := rimage.EncodeImage(context.Background(), goodImg, utils.MimeTypeJPEG) + test.That(t, err, test.ShouldBeNil) + streamableNamedImage, err := camera.NamedImageFromBytes(goodImgBytes, "good", "image/wtf", data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return []camera.NamedImage{malformedNamedImage, streamableNamedImage}, resource.ResponseMetadata{}, nil + }, + } + + img, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + test.That(t, img.SourceName, test.ShouldEqual, "good") + resBytes, err := img.Bytes(context.Background()) + test.That(t, err, test.ShouldBeNil) + test.That(t, resBytes, test.ShouldResemble, goodImgBytes) + }) + + t.Run("prioritizes images based on StreamableImageMIMETypes order", func(t *testing.T) { + sourceImg := image.NewRGBA(image.Rect(0, 0, 1, 1)) + // PNG is higher priority than Depth in our slice + depthImg, err := camera.NamedImageFromImage(sourceImg, "depth", utils.MimeTypeRawDepth, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + pngImg, err := camera.NamedImageFromImage(sourceImg, "png", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + // Return depth first, but PNG should be selected as it's higher priority + return []camera.NamedImage{depthImg, pngImg}, resource.ResponseMetadata{}, nil + }, + } + img, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + test.That(t, img.SourceName, test.ShouldEqual, "png") + test.That(t, img.MimeType(), test.ShouldEqual, utils.MimeTypePNG) + + camReverse := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + // Return PNG first, still should be PNG + return []camera.NamedImage{pngImg, depthImg}, resource.ResponseMetadata{}, nil + }, + } + img, err = camerautils.GetStreamableNamedImageFromCamera(context.Background(), camReverse) + test.That(t, err, test.ShouldBeNil) + test.That(t, img.SourceName, test.ShouldEqual, "png") + test.That(t, img.MimeType(), test.ShouldEqual, utils.MimeTypePNG) + }) +} + func TestVideoSourceFromCamera(t *testing.T) { - sourceImg := image.NewRGBA(image.Rect(0, 0, 3, 3)) + sourceImg := image.NewRGBA(image.Rect(0, 0, 4, 4)) + namedImg, err := camera.NamedImageFromImage(sourceImg, "test", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) cam := &inject.Camera{ - ImageFunc: func(ctx context.Context, mimeType string, extra map[string]interface{}) ([]byte, camera.ImageMetadata, error) { - imgBytes, err := rimage.EncodeImage(ctx, sourceImg, utils.MimeTypePNG) - test.That(t, err, test.ShouldBeNil) - return imgBytes, camera.ImageMetadata{MimeType: utils.MimeTypePNG}, nil + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return []camera.NamedImage{namedImg}, resource.ResponseMetadata{}, nil }, } vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) @@ -39,9 +297,13 @@ func TestVideoSourceFromCamera(t *testing.T) { } func TestVideoSourceFromCameraFalsyVideoProps(t *testing.T) { - malformedCam := &inject.Camera{ - ImageFunc: func(ctx context.Context, mimeType string, extra map[string]interface{}) ([]byte, camera.ImageMetadata, error) { - return []byte("not a valid image"), camera.ImageMetadata{MimeType: utils.MimeTypePNG}, nil + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return nil, resource.ResponseMetadata{}, errors.New("this should not be called") }, } @@ -50,7 +312,7 @@ func TestVideoSourceFromCameraFalsyVideoProps(t *testing.T) { // See: https://viam.atlassian.net/browse/RSDK-12744 // // Instead, it should return a VideoSource with empty video props. - vs, err := camerautils.VideoSourceFromCamera(context.Background(), malformedCam) + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) test.That(t, err, test.ShouldBeNil) test.That(t, vs, test.ShouldNotBeNil) @@ -63,23 +325,539 @@ func TestVideoSourceFromCameraFalsyVideoProps(t *testing.T) { } func TestVideoSourceFromCameraWithNonsenseMimeType(t *testing.T) { - sourceImg := image.NewRGBA(image.Rect(0, 0, 3, 3)) + sourceImg := image.NewRGBA(image.Rect(0, 0, 4, 4)) + namedImg, err := camera.NamedImageFromImage(sourceImg, "test", "image/undefined", data.Annotations{}) + test.That(t, err, test.ShouldBeNil) camWithNonsenseMimeType := &inject.Camera{ - ImageFunc: func(ctx context.Context, mimeType string, extra map[string]interface{}) ([]byte, camera.ImageMetadata, error) { - imgBytes, err := rimage.EncodeImage(ctx, sourceImg, utils.MimeTypePNG) - test.That(t, err, test.ShouldBeNil) - return imgBytes, camera.ImageMetadata{MimeType: "rand"}, nil + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return []camera.NamedImage{namedImg}, resource.ResponseMetadata{}, nil }, } - // Should log a warning though due to the nonsense MIME type vs, err := camerautils.VideoSourceFromCamera(context.Background(), camWithNonsenseMimeType) test.That(t, err, test.ShouldBeNil) test.That(t, vs, test.ShouldNotBeNil) - stream, _ := vs.Stream(context.Background()) + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + + _, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeError, errors.New(`no images were found with a streamable mime type for camera "::/"`)) +} + +func TestVideoSourceFromCamera_SourceSelection(t *testing.T) { + sourceImg := image.NewRGBA(image.Rect(0, 0, 4, 4)) + unstreamableImg, err := camera.NamedImageFromImage(sourceImg, "unstreamable", "image/undefined", data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + streamableImg, err := camera.NamedImageFromImage(sourceImg, "streamable", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + if len(sourceNames) == 0 { + return []camera.NamedImage{unstreamableImg, streamableImg}, resource.ResponseMetadata{}, nil + } + if len(sourceNames) == 1 && sourceNames[0] == "streamable" { + return []camera.NamedImage{streamableImg}, resource.ResponseMetadata{}, nil + } + return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected source filter: %v", sourceNames) + }, + } + + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + + img, _, err := stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + + diffVal, _, err := rimage.CompareImages(img, sourceImg) + test.That(t, err, test.ShouldBeNil) + test.That(t, diffVal, test.ShouldEqual, 0) +} + +func TestVideoSourceFromCamera_Recovery(t *testing.T) { + sourceImg1 := image.NewRGBA(image.Rect(0, 0, 4, 4)) + sourceImg2 := image.NewRGBA(image.Rect(0, 0, 6, 6)) + + goodNamedImage, err := camera.NamedImageFromImage(sourceImg1, "good", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + fallbackNamedImage, err := camera.NamedImageFromImage(sourceImg2, "fallback", utils.MimeTypeJPEG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + goodSourceCallCount := 0 + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + if len(sourceNames) == 0 { // GetStreamableNamedImageFromCamera call + if goodSourceCallCount == 0 { + return []camera.NamedImage{goodNamedImage}, resource.ResponseMetadata{}, nil + } + return []camera.NamedImage{fallbackNamedImage}, resource.ResponseMetadata{}, nil + } + + // getImageBySourceName call + if sourceNames[0] == "good" { + goodSourceCallCount++ + if goodSourceCallCount == 1 { + return nil, resource.ResponseMetadata{}, errors.New("source 'good' is gone") + } + return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected call to source 'good' after failure") + } + if sourceNames[0] == "fallback" { + return []camera.NamedImage{fallbackNamedImage}, resource.ResponseMetadata{}, nil + } + return nil, resource.ResponseMetadata{}, fmt.Errorf("unknown source %q", sourceNames[0]) + }, + } + + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + + // First image should be the good one + img, _, err := stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + diffVal, _, err := rimage.CompareImages(img, sourceImg1) + test.That(t, err, test.ShouldBeNil) + test.That(t, diffVal, test.ShouldEqual, 0) + + // Second call should fail, as the source is now gone. + _, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeError, errors.New("source 'good' is gone")) + + // Third image should be the fallback one, because the state machine reset. + img, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + diffVal, _, err = rimage.CompareImages(img, sourceImg2) + test.That(t, err, test.ShouldBeNil) + test.That(t, diffVal, test.ShouldEqual, 0) +} + +func TestVideoSourceFromCamera_NoImages(t *testing.T) { + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return []camera.NamedImage{}, resource.ResponseMetadata{}, nil + }, + } + + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + test.That(t, vs, test.ShouldNotBeNil) + + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + + _, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeError, errors.New(`no images received for camera "::/"`)) +} + +func TestVideoSourceFromCamera_ImagesError(t *testing.T) { + expectedErr := errors.New("camera error") + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return nil, resource.ResponseMetadata{}, expectedErr + }, + } + + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + test.That(t, vs, test.ShouldNotBeNil) + + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + + _, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeError, expectedErr) +} + +func TestVideoSourceFromCamera_MultipleStreamableSources(t *testing.T) { + imageGood1 := image.NewRGBA(image.Rect(0, 0, 4, 4)) + imageGood2 := image.NewRGBA(image.Rect(0, 0, 6, 6)) + namedA, err := camera.NamedImageFromImage(imageGood1, "good1", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + namedB, err := camera.NamedImageFromImage(imageGood2, "good2", utils.MimeTypeJPEG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + // When unfiltered, return both streamable sources. Selection should pick good1 first. + if len(sourceNames) == 0 { + return []camera.NamedImage{namedA, namedB}, resource.ResponseMetadata{}, nil + } + // When filtered to good1, return only good1 + if len(sourceNames) == 1 && sourceNames[0] == "good1" { + return []camera.NamedImage{namedA}, resource.ResponseMetadata{}, nil + } + return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected source filter: %v", sourceNames) + }, + } + + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + + // JPEG is higher priority than PNG in StreamableImageMIMETypes img, _, err := stream.Next(context.Background()) test.That(t, err, test.ShouldBeNil) - test.That(t, img, test.ShouldNotBeNil) + diffVal, _, err := rimage.CompareImages(img, imageGood2) + test.That(t, err, test.ShouldBeNil) + test.That(t, diffVal, test.ShouldEqual, 0) +} + +func TestVideoSourceFromCamera_NoStreamableSources(t *testing.T) { + src := image.NewRGBA(image.Rect(0, 0, 2, 2)) + unstream1, err := camera.NamedImageFromImage(src, "bad1", "image/undefined", data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + unstream2, err := camera.NamedImageFromImage(src, "bad2", "image/undefined", data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return []camera.NamedImage{unstream1, unstream2}, resource.ResponseMetadata{}, nil + }, + } + + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + _, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeError, errors.New(`no images were found with a streamable mime type for camera "::/"`)) +} + +func TestVideoSourceFromCamera_FilterNoImages(t *testing.T) { + src := image.NewRGBA(image.Rect(0, 0, 4, 4)) + good, err := camera.NamedImageFromImage(src, "good", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + fallback, err := camera.NamedImageFromImage(src, "fallback", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + var firstServed bool + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + if len(sourceNames) == 0 { + // Initial selection / or post-reset unfiltered selection + if !firstServed { + return []camera.NamedImage{good, fallback}, resource.ResponseMetadata{}, nil + } + // After the filtered failure, put fallback first so recovery can succeed + return []camera.NamedImage{fallback, good}, resource.ResponseMetadata{}, nil + } + // Filtered to "good": return no images to simulate empty filter result + if len(sourceNames) == 1 && sourceNames[0] == "good" { + firstServed = true + return []camera.NamedImage{}, resource.ResponseMetadata{}, nil + } + return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected sequence: %v", sourceNames) + }, + } + + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + // First Next() selects "good" source + img, _, err := stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + diffVal, _, err := rimage.CompareImages(img, src) + test.That(t, err, test.ShouldBeNil) + test.That(t, diffVal, test.ShouldEqual, 0) + // Second Next() tries to get "good" but filter returns no images; expect failure + _, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeError, errors.New("no images found for requested source name: good")) + // Third Next() should recover and serve fallback + img, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + diffVal, _, err = rimage.CompareImages(img, src) + test.That(t, err, test.ShouldBeNil) + test.That(t, diffVal, test.ShouldEqual, 0) +} + +func TestVideoSourceFromCamera_FilterMultipleImages(t *testing.T) { + src := image.NewRGBA(image.Rect(0, 0, 4, 4)) + good, err := camera.NamedImageFromImage(src, "good", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + if len(sourceNames) == 0 { + return []camera.NamedImage{good}, resource.ResponseMetadata{}, nil + } + if len(sourceNames) == 1 && sourceNames[0] == "good" { + // Return multiple images even though a single source was requested + // This simulates older camera APIs that don't support filtering + return []camera.NamedImage{good, good}, resource.ResponseMetadata{}, nil + } + return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected source filter: %v", sourceNames) + }, + } + + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + // First Next() should succeed, using the first image from the multiple returned images + img, _, err := stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + diffVal, _, err := rimage.CompareImages(img, src) + test.That(t, err, test.ShouldBeNil) + test.That(t, diffVal, test.ShouldEqual, 0) +} + +func TestVideoSourceFromCamera_FilterMultipleImages_NoMatchingSource(t *testing.T) { + src := image.NewRGBA(image.Rect(0, 0, 4, 4)) + img1, err := camera.NamedImageFromImage(src, "source1", utils.MimeTypeRawRGBA, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + img2, err := camera.NamedImageFromImage(src, "source2", utils.MimeTypeRawRGBA, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + var erroredOnce bool + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + // Initial unfiltered call returns two options. The stream will select "source1". + if len(sourceNames) == 0 { + if !erroredOnce { + return []camera.NamedImage{img1, img2}, resource.ResponseMetadata{}, nil + } + // For recovery, only return the second option. + return []camera.NamedImage{img2}, resource.ResponseMetadata{}, nil + } + + // A filtered call for "source1" will return two images that don't match, triggering an error. + if len(sourceNames) == 1 && sourceNames[0] == "source1" { + erroredOnce = true + return []camera.NamedImage{img2, img2}, resource.ResponseMetadata{}, nil + } + + // The filtered call for "source2" is used for a successful recovery. + if len(sourceNames) == 1 && sourceNames[0] == "source2" { + return []camera.NamedImage{img2}, resource.ResponseMetadata{}, nil + } + return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected source filter: %v", sourceNames) + }, + } + + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + + // First Next() performs unfiltered selection and picks "source1" + img, _, err := stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + diffVal, _, err := rimage.CompareImages(img, src) + test.That(t, err, test.ShouldBeNil) + test.That(t, diffVal, test.ShouldEqual, 0) + + // Second Next() requests "source1" but receives two "source2" images, causing an error + _, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeError, + errors.New(`no matching source name found for multiple returned images: requested "source1", got ["source2" "source2"]`)) + + // Third Next() should recover by performing an unfiltered images call + // The mock will return only the second image, and the stream should succeed + img, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + diffVal, _, err = rimage.CompareImages(img, src) + test.That(t, err, test.ShouldBeNil) + test.That(t, diffVal, test.ShouldEqual, 0) + + // Subsequent calls should also succeed + img, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + diffVal, _, err = rimage.CompareImages(img, src) + test.That(t, err, test.ShouldBeNil) + test.That(t, diffVal, test.ShouldEqual, 0) +} + +func TestVideoSourceFromCamera_LazyDecodeConfigError(t *testing.T) { + malformedImage := rimage.NewLazyEncodedImage( + []byte("not a valid image"), + utils.MimeTypePNG, + ) + + namedImg, err := camera.NamedImageFromImage(malformedImage, "lazy-image", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return []camera.NamedImage{namedImg}, resource.ResponseMetadata{}, nil + }, + } + + _, err = camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) +} + +func TestVideoSourceFromCamera_InvalidImageFirst_ThenValidAlsoAvailable(t *testing.T) { + validImg := image.NewRGBA(image.Rect(0, 0, 4, 4)) + invalidBytes := []byte("not a valid image") + invalidNamed, err := camera.NamedImageFromBytes(invalidBytes, "bad", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + validNamed, err := camera.NamedImageFromImage(validImg, "good", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + // Unfiltered call returns a valid combination, but first entry is invalid + if len(sourceNames) == 0 { + return []camera.NamedImage{invalidNamed, validNamed}, resource.ResponseMetadata{}, nil + } + // If filtered to bad, still bad + if len(sourceNames) == 1 && sourceNames[0] == "bad" { + return []camera.NamedImage{invalidNamed}, resource.ResponseMetadata{}, nil + } + // If filtered to good, return the good one + if len(sourceNames) == 1 && sourceNames[0] == "good" { + return []camera.NamedImage{validNamed}, resource.ResponseMetadata{}, nil + } + return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected filter: %v", sourceNames) + }, + } + + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + // First Next(): already failed, and unfiltered selection again chooses invalid first -> fail + _, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeError, errors.New("could not decode image config: image: unknown format")) + // Second Next(): still fails because selection continues to prioritize invalid-first combination + _, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeError, errors.New("could not decode image config: image: unknown format")) +} + +func TestVideoSourceFromCamera_FilterMismatchedSourceName(t *testing.T) { + src := image.NewRGBA(image.Rect(0, 0, 4, 4)) + good, err := camera.NamedImageFromImage(src, "good", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + mismatched, err := camera.NamedImageFromImage(src, "bad", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + var askedOnce bool + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + if len(sourceNames) == 0 { + return []camera.NamedImage{good}, resource.ResponseMetadata{}, nil + } + if len(sourceNames) == 1 && sourceNames[0] == "good" { + if !askedOnce { + askedOnce = true + // Return a single image with a different source name than requested + return []camera.NamedImage{mismatched}, resource.ResponseMetadata{}, nil + } + // After reset, success + return []camera.NamedImage{good}, resource.ResponseMetadata{}, nil + } + return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected filter: %v", sourceNames) + }, + } + + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + // First Next(): unfiltered selection picks "good" + img, _, err := stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + diffVal, _, err := rimage.CompareImages(img, src) + test.That(t, err, test.ShouldBeNil) + test.That(t, diffVal, test.ShouldEqual, 0) + // Second Next(): filtered request for "good" returns mismatched source name and should fail + _, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeError, errors.New(`mismatched source name: requested "good", got "bad"`)) + // Third Next(): should recover and deliver the correct image + img, _, err = stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + diffVal, _, err = rimage.CompareImages(img, src) + test.That(t, err, test.ShouldBeNil) + test.That(t, diffVal, test.ShouldEqual, 0) +} + +func TestVideoSourceFromCamera_OddDimensionsCropped(t *testing.T) { + // Create an image with odd dimensions + oddImg := image.NewRGBA(image.Rect(0, 0, 3, 3)) + + namedImg, err := camera.NamedImageFromImage(oddImg, "test", utils.MimeTypePNG, data.Annotations{}) + test.That(t, err, test.ShouldBeNil) + + cam := &inject.Camera{ + ImagesFunc: func( + ctx context.Context, + sourceNames []string, + extra map[string]interface{}, + ) ([]camera.NamedImage, resource.ResponseMetadata, error) { + return []camera.NamedImage{namedImg}, resource.ResponseMetadata{}, nil + }, + } + + vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) + test.That(t, err, test.ShouldBeNil) + + stream, err := vs.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + + streamedImg, _, err := stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + + // Verify dimensions were cropped from 3x3 to 2x2 for x264 compatibility + test.That(t, streamedImg.Bounds(), test.ShouldResemble, image.Rect(0, 0, 2, 2)) } diff --git a/robot/web/stream/camera2/camera.go b/robot/web/stream/camera2/camera.go deleted file mode 100644 index 60cc367227e..00000000000 --- a/robot/web/stream/camera2/camera.go +++ /dev/null @@ -1,167 +0,0 @@ -// Package camera provides utilities for working with camera resources in the context of streaming. -package camera - -import ( - "context" - "fmt" - "image" - - "github.com/pion/mediadevices/pkg/prop" - - "go.viam.com/rdk/components/camera" - "go.viam.com/rdk/gostream" - "go.viam.com/rdk/resource" - "go.viam.com/rdk/rimage" - "go.viam.com/rdk/robot" - rutils "go.viam.com/rdk/utils" -) - -var streamableImageMIMETypes = map[string]interface{}{ - rutils.MimeTypeRawRGBA: nil, - rutils.MimeTypeRawDepth: nil, - rutils.MimeTypeJPEG: nil, - rutils.MimeTypePNG: nil, - rutils.MimeTypeQOI: nil, -} - -// cropToEvenDimensions crops an image to even dimensions for x264 compatibility. -// x264 only supports even resolutions. This ensures all streamed images work with x264. -func cropToEvenDimensions(img image.Image) (image.Image, error) { - if img, ok := img.(*rimage.LazyEncodedImage); ok { - if err := img.DecodeConfig(); err != nil { - return nil, err - } - } - - hasOddWidth := img.Bounds().Dx()%2 != 0 - hasOddHeight := img.Bounds().Dy()%2 != 0 - if !hasOddWidth && !hasOddHeight { - return img, nil - } - - rImg := rimage.ConvertImage(img) - newWidth := rImg.Width() - newHeight := rImg.Height() - if hasOddWidth { - newWidth-- - } - if hasOddHeight { - newHeight-- - } - return rImg.SubImage(image.Rect(0, 0, newWidth, newHeight)), nil -} - -// Camera returns the camera from the robot (derived from the stream) or -// an error if it has no camera. -func Camera(robot robot.Robot, stream gostream.Stream) (camera.Camera, error) { - // Stream names are slightly modified versions of the resource short name - shortName := resource.SDPTrackNameToShortName(stream.Name()) - cam, err := camera.FromProvider(robot, shortName) - if err != nil { - return nil, err - } - return cam, nil -} - -// GetStreamableNamedImageFromCamera returns the first named image it finds from the camera that is supported for streaming. -func GetStreamableNamedImageFromCamera(ctx context.Context, cam camera.Camera) (camera.NamedImage, error) { - namedImages, _, err := cam.Images(ctx, nil, nil) - if err != nil { - return camera.NamedImage{}, err - } - if len(namedImages) == 0 { - return camera.NamedImage{}, fmt.Errorf("no images received for camera %q", cam.Name()) - } - - for _, namedImage := range namedImages { - if _, ok := streamableImageMIMETypes[namedImage.MimeType()]; ok { - return namedImage, nil - } - } - return camera.NamedImage{}, fmt.Errorf("no images were found with a streamable mime type for camera %q", cam.Name()) -} - -// getImageBySourceName retrieves a specific named image from the camera by source name. -func getImageBySourceName(ctx context.Context, cam camera.Camera, sourceName string) (camera.NamedImage, error) { - filterSourceNames := []string{sourceName} - namedImages, _, err := cam.Images(ctx, filterSourceNames, nil) - if err != nil { - return camera.NamedImage{}, err - } - - switch len(namedImages) { - case 0: - return camera.NamedImage{}, fmt.Errorf("no images found for requested source name: %s", sourceName) - case 1: - namedImage := namedImages[0] - if namedImage.SourceName != sourceName { - return camera.NamedImage{}, fmt.Errorf("mismatched source name: requested %q, got %q", sourceName, namedImage.SourceName) - } - return namedImage, nil - default: - // At this point, multiple images were returned. This can happen if the camera is on an older version of the API and does not support - // filtering by source name, or if there is a bug in the camera resource's filtering logic. In this unfortunate case, we'll match the - // requested source name and tank the performance costs. - responseSourceNames := []string{} - for _, namedImage := range namedImages { - if namedImage.SourceName == sourceName { - return namedImage, nil - } - responseSourceNames = append(responseSourceNames, namedImage.SourceName) - } - return camera.NamedImage{}, - fmt.Errorf("no matching source name found for multiple returned images: requested %q, got %q", sourceName, responseSourceNames) - } -} - -// VideoSourceFromCamera converts a camera resource into a gostream VideoSource. -// This is useful for streaming video from a camera resource. -func VideoSourceFromCamera(ctx context.Context, cam camera.Camera) (gostream.VideoSource, error) { - // The reader callback uses a small state machine to determine which image to request from the camera. - // A `sourceName` is used to track the selected image source. On the first call, `sourceName` is nil, - // so the first available streamable image is chosen. On subsequent successful calls, the same `sourceName` - // is used. If any errors occur while getting an image, `sourceName` is reset to nil, and the selection - // process starts over on the next call. This allows the stream to recover if a source becomes unavailable. - var sourceName *string - reader := gostream.VideoReaderFunc(func(ctx context.Context) (image.Image, func(), error) { - var respNamedImage camera.NamedImage - - if sourceName == nil { - namedImage, err := GetStreamableNamedImageFromCamera(ctx, cam) - if err != nil { - return nil, func() {}, err - } - respNamedImage = namedImage - sourceName = &namedImage.SourceName - } else { - var err error - respNamedImage, err = getImageBySourceName(ctx, cam, *sourceName) - if err != nil { - sourceName = nil - return nil, func() {}, err - } - } - - img, err := respNamedImage.Image(ctx) - if err != nil { - sourceName = nil - return nil, func() {}, err - } - - img, err = cropToEvenDimensions(img) - if err != nil { - sourceName = nil - return nil, func() {}, err - } - - return img, func() {}, nil - }) - - img, _, err := reader(ctx) - if err != nil { - // Okay to return empty prop because processInputFrames will tick and set them - return gostream.NewVideoSource(reader, prop.Video{}), nil //nolint:nilerr - } - - return gostream.NewVideoSource(reader, prop.Video{Width: img.Bounds().Dx(), Height: img.Bounds().Dy()}), nil -} diff --git a/robot/web/stream/camera2/camera_test.go b/robot/web/stream/camera2/camera_test.go deleted file mode 100644 index bd24c65a876..00000000000 --- a/robot/web/stream/camera2/camera_test.go +++ /dev/null @@ -1,691 +0,0 @@ -package camera_test - -import ( - "context" - "errors" - "fmt" - "image" - "os" - "testing" - - "go.viam.com/test" - - "go.viam.com/rdk/components/camera" - "go.viam.com/rdk/data" - "go.viam.com/rdk/resource" - "go.viam.com/rdk/rimage" - camerautils "go.viam.com/rdk/robot/web/stream/camera2" - "go.viam.com/rdk/testutils/inject" - "go.viam.com/rdk/utils" -) - -func TestGetStreamableNamedImageFromCamera(t *testing.T) { - sourceImg := image.NewRGBA(image.Rect(0, 0, 1, 1)) - unstreamableImg, err := camera.NamedImageFromImage(sourceImg, "unstreamable", "image/undefined", data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - streamableImg, err := camera.NamedImageFromImage(sourceImg, "streamable", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - t.Run("no images", func(t *testing.T) { - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return []camera.NamedImage{}, resource.ResponseMetadata{}, nil - }, - } - _, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeError, errors.New(`no images received for camera "::/"`)) - }) - - t.Run("no streamable images", func(t *testing.T) { - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return []camera.NamedImage{unstreamableImg}, resource.ResponseMetadata{}, nil - }, - } - _, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeError, errors.New(`no images were found with a streamable mime type for camera "::/"`)) - }) - - t.Run("one streamable image", func(t *testing.T) { - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return []camera.NamedImage{streamableImg}, resource.ResponseMetadata{}, nil - }, - } - img, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - test.That(t, img.SourceName, test.ShouldEqual, "streamable") - }) - - t.Run("first image is not streamable", func(t *testing.T) { - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return []camera.NamedImage{unstreamableImg, streamableImg}, resource.ResponseMetadata{}, nil - }, - } - img, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - test.That(t, img.SourceName, test.ShouldEqual, "streamable") - }) - - t.Run("camera Images returns an error", func(t *testing.T) { - expectedErr := errors.New("camera error") - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return nil, resource.ResponseMetadata{}, expectedErr - }, - } - _, err := camerautils.GetStreamableNamedImageFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeError, expectedErr) - }) -} - -func TestVideoSourceFromCamera(t *testing.T) { - sourceImg := image.NewRGBA(image.Rect(0, 0, 4, 4)) - namedImg, err := camera.NamedImageFromImage(sourceImg, "test", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return []camera.NamedImage{namedImg}, resource.ResponseMetadata{}, nil - }, - } - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - - img, _, err := stream.Next(context.Background()) - test.That(t, err, test.ShouldBeNil) - - diffVal, _, err := rimage.CompareImages(img, sourceImg) - test.That(t, err, test.ShouldBeNil) - test.That(t, diffVal, test.ShouldEqual, 0) -} - -func TestVideoSourceFromCameraFailure(t *testing.T) { - malformedNamedImage, err := camera.NamedImageFromBytes([]byte("not a valid image"), "source", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - malformedCam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return []camera.NamedImage{malformedNamedImage}, resource.ResponseMetadata{}, nil - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), malformedCam) - test.That(t, err, test.ShouldBeNil) - test.That(t, vs, test.ShouldNotBeNil) - - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - - _, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeError, errors.New("could not decode image config: image: unknown format")) -} - -func TestVideoSourceFromCameraWithNonsenseMimeType(t *testing.T) { - sourceImg := image.NewRGBA(image.Rect(0, 0, 4, 4)) - namedImg, err := camera.NamedImageFromImage(sourceImg, "test", "image/undefined", data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - camWithNonsenseMimeType := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return []camera.NamedImage{namedImg}, resource.ResponseMetadata{}, nil - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), camWithNonsenseMimeType) - test.That(t, err, test.ShouldBeNil) - test.That(t, vs, test.ShouldNotBeNil) - - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - - _, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeError, errors.New(`no images were found with a streamable mime type for camera "::/"`)) -} - -func TestVideoSourceFromCamera_SourceSelection(t *testing.T) { - sourceImg := image.NewRGBA(image.Rect(0, 0, 4, 4)) - unstreamableImg, err := camera.NamedImageFromImage(sourceImg, "unstreamable", "image/undefined", data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - streamableImg, err := camera.NamedImageFromImage(sourceImg, "streamable", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - if len(sourceNames) == 0 { - return []camera.NamedImage{unstreamableImg, streamableImg}, resource.ResponseMetadata{}, nil - } - if len(sourceNames) == 1 && sourceNames[0] == "streamable" { - return []camera.NamedImage{streamableImg}, resource.ResponseMetadata{}, nil - } - return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected source filter: %v", sourceNames) - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - - img, _, err := stream.Next(context.Background()) - test.That(t, err, test.ShouldBeNil) - - diffVal, _, err := rimage.CompareImages(img, sourceImg) - test.That(t, err, test.ShouldBeNil) - test.That(t, diffVal, test.ShouldEqual, 0) -} - -func TestVideoSourceFromCamera_Recovery(t *testing.T) { - sourceImg1 := image.NewRGBA(image.Rect(0, 0, 4, 4)) - sourceImg2 := image.NewRGBA(image.Rect(0, 0, 6, 6)) - - goodNamedImage, err := camera.NamedImageFromImage(sourceImg1, "good", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - fallbackNamedImage, err := camera.NamedImageFromImage(sourceImg2, "fallback", utils.MimeTypeJPEG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - firstSourceFailed := false - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - if len(sourceNames) == 0 { // GetStreamableNamedImageFromCamera call - if !firstSourceFailed { - return []camera.NamedImage{goodNamedImage}, resource.ResponseMetadata{}, nil - } - return []camera.NamedImage{fallbackNamedImage}, resource.ResponseMetadata{}, nil - } - - // getImageBySourceName call - if sourceNames[0] == "good" { - if !firstSourceFailed { - firstSourceFailed = true - return []camera.NamedImage{goodNamedImage}, resource.ResponseMetadata{}, nil - } - return nil, resource.ResponseMetadata{}, errors.New("source 'good' is gone") - } - if sourceNames[0] == "fallback" { - return []camera.NamedImage{fallbackNamedImage}, resource.ResponseMetadata{}, nil - } - return nil, resource.ResponseMetadata{}, fmt.Errorf("unknown source %q", sourceNames[0]) - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - - // First image should be the good one - img, _, err := stream.Next(context.Background()) - test.That(t, err, test.ShouldBeNil) - diffVal, _, err := rimage.CompareImages(img, sourceImg1) - test.That(t, err, test.ShouldBeNil) - test.That(t, diffVal, test.ShouldEqual, 0) - - // Second call should fail, as the source is now gone. - _, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeError, errors.New("source 'good' is gone")) - - // Third image should be the fallback one, because the state machine reset. - img, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeNil) - diffVal, _, err = rimage.CompareImages(img, sourceImg2) - test.That(t, err, test.ShouldBeNil) - test.That(t, diffVal, test.ShouldEqual, 0) -} - -func TestVideoSourceFromCamera_NoImages(t *testing.T) { - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return []camera.NamedImage{}, resource.ResponseMetadata{}, nil - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - test.That(t, vs, test.ShouldNotBeNil) - - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - - _, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeError, errors.New(`no images received for camera "::/"`)) -} - -func TestVideoSourceFromCamera_ImagesError(t *testing.T) { - expectedErr := errors.New("camera error") - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return nil, resource.ResponseMetadata{}, expectedErr - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - test.That(t, vs, test.ShouldNotBeNil) - - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - - _, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeError, expectedErr) -} - -func TestVideoSourceFromCamera_MultipleStreamableSources(t *testing.T) { - imageGood1 := image.NewRGBA(image.Rect(0, 0, 4, 4)) - imageGood2 := image.NewRGBA(image.Rect(0, 0, 6, 6)) - namedA, err := camera.NamedImageFromImage(imageGood1, "good1", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - namedB, err := camera.NamedImageFromImage(imageGood2, "good2", utils.MimeTypeJPEG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - // When unfiltered, return both streamable sources. Selection should pick good1 first. - if len(sourceNames) == 0 { - return []camera.NamedImage{namedA, namedB}, resource.ResponseMetadata{}, nil - } - // When filtered to good1, return only good1 - if len(sourceNames) == 1 && sourceNames[0] == "good1" { - return []camera.NamedImage{namedA}, resource.ResponseMetadata{}, nil - } - return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected source filter: %v", sourceNames) - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - - img, _, err := stream.Next(context.Background()) - test.That(t, err, test.ShouldBeNil) - diffVal, _, err := rimage.CompareImages(img, imageGood1) - test.That(t, err, test.ShouldBeNil) - test.That(t, diffVal, test.ShouldEqual, 0) -} - -func TestVideoSourceFromCamera_NoStreamableSources(t *testing.T) { - src := image.NewRGBA(image.Rect(0, 0, 2, 2)) - unstream1, err := camera.NamedImageFromImage(src, "bad1", "image/undefined", data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - unstream2, err := camera.NamedImageFromImage(src, "bad2", "image/undefined", data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return []camera.NamedImage{unstream1, unstream2}, resource.ResponseMetadata{}, nil - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - _, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeError, errors.New(`no images were found with a streamable mime type for camera "::/"`)) -} - -func TestVideoSourceFromCamera_FilterNoImages(t *testing.T) { - src := image.NewRGBA(image.Rect(0, 0, 4, 4)) - good, err := camera.NamedImageFromImage(src, "good", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - fallback, err := camera.NamedImageFromImage(src, "fallback", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - var firstServed bool - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - if len(sourceNames) == 0 { - // Initial selection / or post-reset unfiltered selection - if !firstServed { - return []camera.NamedImage{good, fallback}, resource.ResponseMetadata{}, nil - } - // After the filtered failure, put fallback first so recovery can succeed - return []camera.NamedImage{fallback, good}, resource.ResponseMetadata{}, nil - } - // Filtered to "good": return no images to simulate empty filter result - if len(sourceNames) == 1 && sourceNames[0] == "good" { - firstServed = true - return []camera.NamedImage{}, resource.ResponseMetadata{}, nil - } - return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected sequence: %v", sourceNames) - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - // First Next() corresponds to the first filtered read; expect failure - _, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeError, errors.New("no images found for requested source name: good")) - // Next Next() should recover and serve fallback - img, _, err := stream.Next(context.Background()) - test.That(t, err, test.ShouldBeNil) - diffVal, _, err := rimage.CompareImages(img, src) - test.That(t, err, test.ShouldBeNil) - test.That(t, diffVal, test.ShouldEqual, 0) -} - -func TestVideoSourceFromCamera_FilterMultipleImages(t *testing.T) { - src := image.NewRGBA(image.Rect(0, 0, 4, 4)) - good, err := camera.NamedImageFromImage(src, "good", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - if len(sourceNames) == 0 { - return []camera.NamedImage{good}, resource.ResponseMetadata{}, nil - } - if len(sourceNames) == 1 && sourceNames[0] == "good" { - // Return multiple images even though a single source was requested - // This simulates older camera APIs that don't support filtering - return []camera.NamedImage{good, good}, resource.ResponseMetadata{}, nil - } - return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected source filter: %v", sourceNames) - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - // First Next() should succeed, using the first image from the multiple returned images - img, _, err := stream.Next(context.Background()) - test.That(t, err, test.ShouldBeNil) - diffVal, _, err := rimage.CompareImages(img, src) - test.That(t, err, test.ShouldBeNil) - test.That(t, diffVal, test.ShouldEqual, 0) -} - -func TestVideoSourceFromCamera_FilterMultipleImages_NoMatchingSource(t *testing.T) { - src := image.NewRGBA(image.Rect(0, 0, 4, 4)) - img1, err := camera.NamedImageFromImage(src, "source1", utils.MimeTypeRawRGBA, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - img2, err := camera.NamedImageFromImage(src, "source2", utils.MimeTypeRawRGBA, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - var erroredOnce bool - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - // Initial unfiltered call returns two options. The stream will select "source1". - if len(sourceNames) == 0 { - if !erroredOnce { - return []camera.NamedImage{img1, img2}, resource.ResponseMetadata{}, nil - } - // For recovery, only return the second option. - return []camera.NamedImage{img2}, resource.ResponseMetadata{}, nil - } - - // A filtered call for "source1" will return two images that don't match, triggering an error. - if len(sourceNames) == 1 && sourceNames[0] == "source1" { - erroredOnce = true - return []camera.NamedImage{img2, img2}, resource.ResponseMetadata{}, nil - } - - // The filtered call for "source2" is used for a successful recovery. - if len(sourceNames) == 1 && sourceNames[0] == "source2" { - return []camera.NamedImage{img2}, resource.ResponseMetadata{}, nil - } - return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected source filter: %v", sourceNames) - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - - // The first call to `VideoSourceFromCamera` will select "source1". The first call to `stream.Next()` - // will then request "source1" and receive two "source2" images, causing an error. - _, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeError, - errors.New(`no matching source name found for multiple returned images: requested "source1", got ["source2" "source2"]`)) - - // On the next call, the stream should recover by performing an unfiltered images call. - // The mock will return only the second image, and the stream should succeed. - img, _, err := stream.Next(context.Background()) - test.That(t, err, test.ShouldBeNil) - diffVal, _, err := rimage.CompareImages(img, src) - test.That(t, err, test.ShouldBeNil) - test.That(t, diffVal, test.ShouldEqual, 0) - - // Subsequent calls should also succeed. - img, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeNil) - diffVal, _, err = rimage.CompareImages(img, src) - test.That(t, err, test.ShouldBeNil) - test.That(t, diffVal, test.ShouldEqual, 0) -} - -func TestVideoSourceFromCamera_LazyDecodeConfigError(t *testing.T) { - malformedImage := rimage.NewLazyEncodedImage( - []byte("not a valid image"), - utils.MimeTypePNG, - ) - - namedImg, err := camera.NamedImageFromImage(malformedImage, "lazy-image", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return []camera.NamedImage{namedImg}, resource.ResponseMetadata{}, nil - }, - } - - _, err = camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) -} - -func TestVideoSourceFromCamera_InvalidImageFirst_ThenValidAlsoAvailable(t *testing.T) { - validImg := image.NewRGBA(image.Rect(0, 0, 4, 4)) - invalidBytes := []byte("not a valid image") - invalidNamed, err := camera.NamedImageFromBytes(invalidBytes, "bad", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - validNamed, err := camera.NamedImageFromImage(validImg, "good", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - // Unfiltered call returns a valid combination, but first entry is invalid - if len(sourceNames) == 0 { - return []camera.NamedImage{invalidNamed, validNamed}, resource.ResponseMetadata{}, nil - } - // If filtered to bad, still bad - if len(sourceNames) == 1 && sourceNames[0] == "bad" { - return []camera.NamedImage{invalidNamed}, resource.ResponseMetadata{}, nil - } - // If filtered to good, return the good one - if len(sourceNames) == 1 && sourceNames[0] == "good" { - return []camera.NamedImage{validNamed}, resource.ResponseMetadata{}, nil - } - return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected filter: %v", sourceNames) - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - // First Next(): already failed, and unfiltered selection again chooses invalid first -> fail - _, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeError, errors.New("could not decode image config: image: unknown format")) - // Second Next(): still fails because selection continues to prioritize invalid-first combination - _, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeError, errors.New("could not decode image config: image: unknown format")) -} - -func TestVideoSourceFromCamera_FilterMismatchedSourceName(t *testing.T) { - src := image.NewRGBA(image.Rect(0, 0, 4, 4)) - good, err := camera.NamedImageFromImage(src, "good", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - mismatched, err := camera.NamedImageFromImage(src, "bad", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - var askedOnce bool - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - if len(sourceNames) == 0 { - return []camera.NamedImage{good}, resource.ResponseMetadata{}, nil - } - if len(sourceNames) == 1 && sourceNames[0] == "good" { - if !askedOnce { - askedOnce = true - // Return a single image with a different source name than requested - return []camera.NamedImage{mismatched}, resource.ResponseMetadata{}, nil - } - // After reset, success - return []camera.NamedImage{good}, resource.ResponseMetadata{}, nil - } - return nil, resource.ResponseMetadata{}, fmt.Errorf("unexpected filter: %v", sourceNames) - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - // First Next(): filtered mismatch should fail - _, _, err = stream.Next(context.Background()) - test.That(t, err, test.ShouldBeError, errors.New(`mismatched source name: requested "good", got "bad"`)) - // Second Next(): should recover and deliver the correct image - img, _, err := stream.Next(context.Background()) - test.That(t, err, test.ShouldBeNil) - diffVal, _, err := rimage.CompareImages(img, src) - test.That(t, err, test.ShouldBeNil) - test.That(t, diffVal, test.ShouldEqual, 0) -} - -func TestVideoSourceFromCamera_OddDimensionsCropped(t *testing.T) { - // Create an image with odd dimensions - oddImg := image.NewRGBA(image.Rect(0, 0, 3, 3)) - - namedImg, err := camera.NamedImageFromImage(oddImg, "test", utils.MimeTypePNG, data.Annotations{}) - test.That(t, err, test.ShouldBeNil) - - cam := &inject.Camera{ - ImagesFunc: func( - ctx context.Context, - sourceNames []string, - extra map[string]interface{}, - ) ([]camera.NamedImage, resource.ResponseMetadata, error) { - return []camera.NamedImage{namedImg}, resource.ResponseMetadata{}, nil - }, - } - - vs, err := camerautils.VideoSourceFromCamera(context.Background(), cam) - test.That(t, err, test.ShouldBeNil) - - stream, err := vs.Stream(context.Background()) - test.That(t, err, test.ShouldBeNil) - - streamedImg, _, err := stream.Next(context.Background()) - test.That(t, err, test.ShouldBeNil) - - // Verify dimensions were cropped from 3x3 to 2x2 for x264 compatibility - test.That(t, streamedImg.Bounds(), test.ShouldResemble, image.Rect(0, 0, 2, 2)) -} - -// TODO(https://viam.atlassian.net/browse/RSDK-11726): Remove this test. -func TestGetImagesInStreamServerEnvVar(t *testing.T) { - ogVal, ok := os.LookupEnv(utils.GetImagesInStreamServerEnvVar) - if ok { - defer os.Setenv(utils.GetImagesInStreamServerEnvVar, ogVal) - } else { - defer os.Unsetenv(utils.GetImagesInStreamServerEnvVar) - } - - t.Run("when env var is set to true, returns true", func(t *testing.T) { - os.Setenv(utils.GetImagesInStreamServerEnvVar, "true") - test.That(t, utils.GetImagesInStreamServer(), test.ShouldBeTrue) - }) - - t.Run("when env var is not set, returns false", func(t *testing.T) { - os.Unsetenv(utils.GetImagesInStreamServerEnvVar) - test.That(t, utils.GetImagesInStreamServer(), test.ShouldBeFalse) - }) -} diff --git a/robot/web/stream/server.go b/robot/web/stream/server.go index b155d0f16a0..285b5f8ce72 100644 --- a/robot/web/stream/server.go +++ b/robot/web/stream/server.go @@ -3,7 +3,6 @@ package webstream import ( "context" "fmt" - "image" "runtime" "sync" "time" @@ -22,8 +21,7 @@ import ( "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" "go.viam.com/rdk/robot" - camerautils1 "go.viam.com/rdk/robot/web/stream/camera" - camerautils2 "go.viam.com/rdk/robot/web/stream/camera2" + camerautils "go.viam.com/rdk/robot/web/stream/camera" "go.viam.com/rdk/robot/web/stream/state" rutils "go.viam.com/rdk/utils" ) @@ -172,7 +170,7 @@ func (server *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequ } // return error if resource is neither a camera nor audioinput - _, isCamErr := cameraUtilsCamera(server.robot, streamStateToAdd.Stream) + _, isCamErr := camerautils.Camera(server.robot, streamStateToAdd.Stream) _, isAudioErr := audioinput.FromProvider(server.robot, streamStateToAdd.Stream.Name()) if isCamErr != nil && isAudioErr != nil { return nil, errors.Errorf("stream is neither a camera nor audioinput. streamName: %v", streamStateToAdd.Stream) @@ -273,7 +271,7 @@ func (server *Server) RemoveStream(ctx context.Context, req *streampb.RemoveStre streamName := streamToRemove.Stream.Name() _, isAudioResourceErr := audioinput.FromProvider(server.robot, streamName) - _, isCameraResourceErr := cameraUtilsCamera(server.robot, streamToRemove.Stream) + _, isCameraResourceErr := camerautils.Camera(server.robot, streamToRemove.Stream) if isAudioResourceErr != nil && isCameraResourceErr != nil { return &streampb.RemoveStreamResponse{}, nil @@ -419,7 +417,7 @@ func (server *Server) resizeVideoSource(ctx context.Context, name string, width, if !ok { return fmt.Errorf("stream state not found with name %q", name) } - vs, err := cameraUtilsVideoSourceFromCamera(ctx, cam) + vs, err := camerautils.VideoSourceFromCamera(ctx, cam) if err != nil { return fmt.Errorf("failed to create video source from camera: %w", err) } @@ -451,7 +449,7 @@ func (server *Server) resetVideoSource(ctx context.Context, name string) error { return fmt.Errorf("stream state not found with name %q", name) } server.logger.Debug("resetting video source") - vs, err := cameraUtilsVideoSourceFromCamera(ctx, cam) + vs, err := camerautils.VideoSourceFromCamera(ctx, cam) if err != nil { return fmt.Errorf("failed to create video source from camera: %w", err) } @@ -643,7 +641,7 @@ func (server *Server) refreshVideoSources(ctx context.Context) { if err != nil { continue } - src, err := cameraUtilsVideoSourceFromCamera(ctx, cam) + src, err := camerautils.VideoSourceFromCamera(ctx, cam) if err != nil { server.logger.Errorf("error creating video source from camera: %v", err) continue @@ -794,56 +792,9 @@ func GenerateResolutions(width, height int32, logger logging.Logger) []Resolutio return resolutions } -func cameraUtilsCamera(robot robot.Robot, stream gostream.Stream) (camera.Camera, error) { - if rutils.GetImagesInStreamServer() { - return camerautils2.Camera(robot, stream) - } - return camerautils1.Camera(robot, stream) -} - -func cameraUtilsVideoSourceFromCamera(ctx context.Context, cam camera.Camera) (gostream.VideoSource, error) { - if rutils.GetImagesInStreamServer() { - return camerautils2.VideoSourceFromCamera(ctx, cam) - } - return camerautils1.VideoSourceFromCamera(ctx, cam) -} - // sampleFrameSize takes in a camera.Camera, starts a stream, attempts to // pull a frame, and returns the width and height. func sampleFrameSize(ctx context.Context, cam camera.Camera, logger logging.Logger) (int, int, error) { - if rutils.GetImagesInStreamServer() { - return sampleFrameSize2(ctx, cam, logger) - } - return sampleFrameSize1(ctx, cam, logger) -} - -func sampleFrameSize1(ctx context.Context, cam camera.Camera, logger logging.Logger) (int, int, error) { - logger.Debug("sampling frame size") - // Attempt to get a frame from the stream with a maximum of 5 retries. - // This is useful if cameras have a warm-up period before they can start streaming. - var frame image.Image - var err error -retryLoop: - for i := 0; i < 5; i++ { - select { - case <-ctx.Done(): - return 0, 0, ctx.Err() - default: - frame, err = camera.DecodeImageFromCamera(ctx, "", nil, cam) - if err == nil { - break retryLoop // Break out of the for loop, not just the select. - } - logger.Debugf("failed to get frame, retrying... (%d/5)", i+1) - time.Sleep(retryDelay) - } - } - if err != nil { - return 0, 0, fmt.Errorf("failed to get frame after 5 attempts: %w", err) - } - return frame.Bounds().Dx(), frame.Bounds().Dy(), nil -} - -func sampleFrameSize2(ctx context.Context, cam camera.Camera, logger logging.Logger) (int, int, error) { logger.Debug("sampling frame size") // Attempt to get a frame from the stream with a maximum of 5 retries. // This is useful if cameras have a warm-up period before they can start streaming. @@ -854,7 +805,7 @@ func sampleFrameSize2(ctx context.Context, cam camera.Camera, logger logging.Log case <-ctx.Done(): return 0, 0, ctx.Err() default: - namedImage, err := camerautils2.GetStreamableNamedImageFromCamera(ctx, cam) + namedImage, err := camerautils.GetStreamableNamedImageFromCamera(ctx, cam) if err != nil { logger.Debugf("failed to get streamable named image from camera: %v", err) lastErr = err diff --git a/utils/env.go b/utils/env.go index 31eab44c458..05b99456090 100644 --- a/utils/env.go +++ b/utils/env.go @@ -6,7 +6,6 @@ import ( "path/filepath" "regexp" "runtime" - "slices" "strconv" "strings" "time" @@ -80,9 +79,6 @@ const ( // defaults to 100. ViamResourceRequestsLimitEnvVar = "VIAM_RESOURCE_REQUESTS_LIMIT" - // GetImagesInStreamServerEnvVar is the environment variable that enables the GetImages feature flag in stream server. - GetImagesInStreamServerEnvVar = "VIAM_GET_IMAGES_IN_STREAM_SERVER" - // ViamModuleTracingEnvVar is the environment variable that configures // modules to record trace spans and send them to their parent viam-server // process. Any non-empty string other than "0" or "false" enables module @@ -214,11 +210,6 @@ func GetenvBool(v string, def bool) bool { return x[0] == 't' || x[0] == 'T' || x[0] == '1' } -// GetImagesInStreamServer returns true iff an env bool was set to use the GetImages feature flag in stream server. -func GetImagesInStreamServer() bool { - return slices.Contains(EnvTrueValues, os.Getenv(GetImagesInStreamServerEnvVar)) -} - // CleanWindowsSocketPath mutates socket paths on windows only so they // work well with the GRPC library. // It converts e.g. C:\x\y.sock to /x/y.sock