From e4f1289c7cd298614c1f6c1d65a8e17bea261eb9 Mon Sep 17 00:00:00 2001 From: Oleksandr Krutko Date: Fri, 6 Dec 2024 02:05:59 +0200 Subject: [PATCH] add filter for container command Signed-off-by: Oleksandr Krutko add a test, improve logic of command filter Signed-off-by: Oleksandr Krutko improve a test Signed-off-by: Oleksandr Krutko improve test, update a man page Signed-off-by: Oleksandr Krutko improve man page, runtime functions Signed-off-by: Oleksandr Krutko move ExternalContainerFilter type to entities package Signed-off-by: Oleksandr Krutko add external filters Signed-off-by: Oleksandr Krutko add tests for external containers Signed-off-by: Oleksandr Krutko add test for ps external id, ancestor Signed-off-by: Oleksandr Krutko add tests for ps external filters of since, before Signed-off-by: Oleksandr Krutko fix linter warnings, add completion for the name filter Signed-off-by: Oleksandr Krutko resolve conflicts Signed-off-by: Oleksandr Krutko check command length, filter containers liist by external key Signed-off-by: Oleksandr Krutko --- cmd/podman/common/completion.go | 54 +++++- docs/source/markdown/podman-ps.1.md | 2 +- libpod/runtime_ctr.go | 16 +- pkg/domain/entities/container_ps.go | 5 + pkg/domain/entities/types/container_ps.go | 28 +++ pkg/domain/filters/containers.go | 115 +++++++++++++ pkg/ps/ps.go | 47 ++++- test/e2e/ps_test.go | 199 ++++++++++++++++++++++ 8 files changed, 455 insertions(+), 11 deletions(-) diff --git a/cmd/podman/common/completion.go b/cmd/podman/common/completion.go index e46675733c..c148e86fe4 100644 --- a/cmd/podman/common/completion.go +++ b/cmd/podman/common/completion.go @@ -87,6 +87,7 @@ func setupImageEngine(cmd *cobra.Command) (entities.ImageEngine, error) { } func getContainers(cmd *cobra.Command, toComplete string, cType completeType, statuses ...string) ([]string, cobra.ShellCompDirective) { + var listContainers []entities.ListContainer suggestions := []string{} listOpts := entities.ContainerListOptions{ Filters: make(map[string][]string), @@ -108,7 +109,20 @@ func getContainers(cmd *cobra.Command, toComplete string, cType completeType, st return nil, cobra.ShellCompDirectiveNoFileComp } - for _, c := range containers { + listContainers = append(listContainers, containers...) + + // Add containers from the external storage into complete list + if ok, _ := cmd.Flags().GetBool("external"); ok { + externalContainers, err := engine.ContainerListExternal(registry.GetContext()) + if err != nil { + cobra.CompErrorln(err.Error()) + return nil, cobra.ShellCompDirectiveNoFileComp + } + + listContainers = append(listContainers, externalContainers...) + } + + for _, c := range listContainers { // include ids in suggestions if cType == completeIDs or // more then 2 chars are typed and cType == completeDefault if ((len(toComplete) > 1 && cType == completeDefault) || @@ -317,6 +331,43 @@ func getNetworks(cmd *cobra.Command, toComplete string, cType completeType) ([]s return suggestions, cobra.ShellCompDirectiveNoFileComp } +func getCommands(cmd *cobra.Command, toComplete string) ([]string, cobra.ShellCompDirective) { + suggestions := []string{} + lsOpts := entities.ContainerListOptions{} + + engine, err := setupContainerEngine(cmd) + if err != nil { + cobra.CompErrorln(err.Error()) + return nil, cobra.ShellCompDirectiveNoFileComp + } + + containers, err := engine.ContainerList(registry.GetContext(), lsOpts) + if err != nil { + cobra.CompErrorln(err.Error()) + return nil, cobra.ShellCompDirectiveNoFileComp + } + + externalContainers, err := engine.ContainerListExternal(registry.GetContext()) + if err != nil { + cobra.CompErrorln(err.Error()) + return nil, cobra.ShellCompDirectiveNoFileComp + } + containers = append(containers, externalContainers...) + + for _, container := range containers { + // taking of the first element of commands list is done intentionally + // to exclude command arguments from suggestions (e.g. exclude arguments "-g daemon" + // from "nginx -g daemon" output) + if len(container.Command) > 0 { + if strings.HasPrefix(container.Command[0], toComplete) { + suggestions = append(suggestions, container.Command[0]) + } + } + } + + return suggestions, cobra.ShellCompDirectiveNoFileComp +} + func fdIsNotDir(f *os.File) bool { stat, err := f.Stat() if err != nil { @@ -1661,6 +1712,7 @@ func AutocompletePsFilters(cmd *cobra.Command, args []string, toComplete string) kv := keyValueCompletion{ "ancestor=": func(s string) ([]string, cobra.ShellCompDirective) { return getImages(cmd, s) }, "before=": func(s string) ([]string, cobra.ShellCompDirective) { return getContainers(cmd, s, completeDefault) }, + "command=": func(s string) ([]string, cobra.ShellCompDirective) { return getCommands(cmd, s) }, "exited=": nil, "health=": func(_ string) ([]string, cobra.ShellCompDirective) { return []string{define.HealthCheckHealthy, diff --git a/docs/source/markdown/podman-ps.1.md b/docs/source/markdown/podman-ps.1.md index 30ccf98021..4689c9860d 100644 --- a/docs/source/markdown/podman-ps.1.md +++ b/docs/source/markdown/podman-ps.1.md @@ -61,7 +61,7 @@ Valid filters are listed below: | pod | [Pod] name or full or partial ID of pod | | network | [Network] name or full ID of network | | until | [DateTime] container created before the given duration or time. | - +| command | [Command] the command the container is executing, only argv[0] is taken | #### **--format**=*format* diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 87d8484783..3636a7d6d4 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -1246,11 +1246,21 @@ func (r *Runtime) GetContainers(loadState bool, filters ...ContainerFilter) ([]* return nil, err } - ctrsFiltered := make([]*Container, 0, len(ctrs)) + ctrsFiltered := applyContainersFilters(ctrs, filters...) - for _, ctr := range ctrs { + return ctrsFiltered, nil +} + +// Applies container filters on bunch of containers +func applyContainersFilters(containers []*Container, filters ...ContainerFilter) []*Container { + ctrsFiltered := make([]*Container, 0, len(containers)) + + for _, ctr := range containers { include := true for _, filter := range filters { + if filter == nil { + continue + } include = include && filter(ctr) } @@ -1259,7 +1269,7 @@ func (r *Runtime) GetContainers(loadState bool, filters ...ContainerFilter) ([]* } } - return ctrsFiltered, nil + return ctrsFiltered } // GetAllContainers is a helper function for GetContainers diff --git a/pkg/domain/entities/container_ps.go b/pkg/domain/entities/container_ps.go index 0cd8e740f9..b408eec06d 100644 --- a/pkg/domain/entities/container_ps.go +++ b/pkg/domain/entities/container_ps.go @@ -8,6 +8,11 @@ import ( "github.com/containers/podman/v5/pkg/domain/entities/types" ) +// ExternalContainerFilter is a function to determine whether a container list is included +// in command output. Container lists to be outputted are tested using the function. +// A true return will include the container list, a false return will exclude it. +type ExternalContainerFilter func(*ListContainer) bool + // ListContainer describes a container suitable for listing type ListContainer = types.ListContainer diff --git a/pkg/domain/entities/types/container_ps.go b/pkg/domain/entities/types/container_ps.go index 139a87c036..badcf73acf 100644 --- a/pkg/domain/entities/types/container_ps.go +++ b/pkg/domain/entities/types/container_ps.go @@ -118,3 +118,31 @@ func (l ListContainer) USERNS() string { func (l ListContainer) UTS() string { return l.Namespaces.UTS } + +func (l ListContainer) Commands() []string { + return l.Command +} + +func (l ListContainer) ContainerID() string { + return l.ID +} + +func (l ListContainer) LabelsList() map[string]string { + return l.Labels +} + +func (l ListContainer) NamesList() []string { + return l.Names +} + +func (l ListContainer) ImageInfo() (string, string) { + return l.ImageID, l.Image +} + +func (l ListContainer) CreatedTime() time.Time { + return l.Created +} + +func (l ListContainer) StatusInfo() string { + return l.Status +} diff --git a/pkg/domain/filters/containers.go b/pkg/domain/filters/containers.go index b4a683631c..b9c1d439e2 100644 --- a/pkg/domain/filters/containers.go +++ b/pkg/domain/filters/containers.go @@ -14,6 +14,8 @@ import ( "github.com/containers/common/pkg/util" "github.com/containers/podman/v5/libpod" "github.com/containers/podman/v5/libpod/define" + "github.com/containers/podman/v5/pkg/domain/entities/types" + "github.com/containers/storage" ) // GenerateContainerFilterFuncs return ContainerFilter functions based of filter. @@ -282,6 +284,10 @@ func GenerateContainerFilterFuncs(filter string, filterValues []string, r *libpo } return false }, filterValueError + case "command": + return func(c *libpod.Container) bool { + return util.StringMatchRegexSlice(c.Command()[0], filterValues) + }, nil } return nil, fmt.Errorf("%s is an invalid filter", filter) } @@ -315,3 +321,112 @@ func prepareUntilFilterFunc(filterValues []string) (func(container *libpod.Conta return false }, nil } + +// GenerateContainerFilterFuncs return ContainerFilter functions based of filter. +func GenerateExternalContainerFilterFuncs(filter string, filterValues []string, r *libpod.Runtime) (func(listContainer *types.ListContainer) bool, error) { + switch filter { + case "id": + return func(listContainer *types.ListContainer) bool { + return filters.FilterID(listContainer.ContainerID(), filterValues) + }, nil + case "name": + // we only have to match one name + return func(listContainer *types.ListContainer) bool { + namesList := listContainer.NamesList() + + for _, f := range filterValues { + f = strings.ReplaceAll(f, "/", "") + if util.StringMatchRegexSlice(f, namesList) { + return true + } + } + + return false + }, nil + case "command": + return func(listContainer *types.ListContainer) bool { + return util.StringMatchRegexSlice(listContainer.Commands()[0], filterValues) + }, nil + case "ancestor": + // This needs to refine to match docker + // - ancestor=([:tag]|| ⟨image@digest⟩) - containers created from an image or a descendant. + return func(listContainer *types.ListContainer) bool { + for _, filterValue := range filterValues { + rootfsImageID, rootfsImageName := listContainer.ImageInfo() + var imageTag string + var imageNameWithoutTag string + // Compare with ImageID, ImageName + // Will match ImageName if running image has tag latest for other tags exact complete filter must be given + name, tag, hasColon := strings.Cut(rootfsImageName, ":") + if hasColon { + imageNameWithoutTag = name + imageTag = tag + } + + if (rootfsImageID == filterValue) || + util.StringMatchRegexSlice(rootfsImageName, filterValues) || + (util.StringMatchRegexSlice(imageNameWithoutTag, filterValues) && imageTag == "latest") { + return true + } + } + return false + }, nil + case "before": + var createTime time.Time + var externCons []storage.Container + externCons, err := r.StorageContainers() + if err != nil { + return nil, err + } + + for _, filterValue := range filterValues { + for _, ctr := range externCons { + if slices.Contains(ctr.Names, filterValue) { + if createTime.IsZero() || createTime.After(ctr.Created) { + createTime = ctr.Created + } + } + } + } + + return func(listContainer *types.ListContainer) bool { + return createTime.After(listContainer.CreatedTime()) + }, nil + case "since": + var createTime time.Time + var externCons []storage.Container + externCons, err := r.StorageContainers() + if err != nil { + return nil, err + } + + for _, filterValue := range filterValues { + for _, ctr := range externCons { + if slices.Contains(ctr.Names, filterValue) { + if createTime.IsZero() || createTime.After(ctr.Created) { + createTime = ctr.Created + } + } + } + } + + return func(listContainer *types.ListContainer) bool { + return createTime.Before(listContainer.CreatedTime()) + }, nil + case "until": + until, err := filters.ComputeUntilTimestamp(filterValues) + if err != nil { + return nil, err + } + return func(listContainer *types.ListContainer) bool { + if !until.IsZero() && listContainer.CreatedTime().Before(until) { + return true + } + return false + }, nil + case "restart-policy", "network", "pod", "volume", "health", "label", "exited", "status": + return nil, fmt.Errorf("filter %s is not applicable for external containers", filter) + } + + return nil, fmt.Errorf("%s is an invalid filter", filter) +} diff --git a/pkg/ps/ps.go b/pkg/ps/ps.go index 7a77621e75..619aafed60 100644 --- a/pkg/ps/ps.go +++ b/pkg/ps/ps.go @@ -24,19 +24,33 @@ import ( "github.com/sirupsen/logrus" ) +// ExternalContainerFilter is a function to determine whether a container list is included +// in command output. Container lists to be outputted are tested using the function. +// A true return will include the container list, a false return will exclude it. +type ExternalContainerFilter func(*entities.ListContainer) bool + func GetContainerLists(runtime *libpod.Runtime, options entities.ContainerListOptions) ([]entities.ListContainer, error) { var ( pss = []entities.ListContainer{} ) filterFuncs := make([]libpod.ContainerFilter, 0, len(options.Filters)) + filterExtFuncs := make([]entities.ExternalContainerFilter, 0, len(options.Filters)) all := options.All || options.Last > 0 if len(options.Filters) > 0 { for k, v := range options.Filters { generatedFunc, err := filters.GenerateContainerFilterFuncs(k, v, runtime) - if err != nil { + if err != nil && !options.External { return nil, err } filterFuncs = append(filterFuncs, generatedFunc) + + if options.External { + generatedExtFunc, err := filters.GenerateExternalContainerFilterFuncs(k, v, runtime) + if err != nil { + return nil, err + } + filterExtFuncs = append(filterExtFuncs, generatedExtFunc) + } } } @@ -87,7 +101,7 @@ func GetContainerLists(runtime *libpod.Runtime, options entities.ContainerListOp } if options.External { - listCon, err := GetExternalContainerLists(runtime) + listCon, err := GetExternalContainerLists(runtime, filterExtFuncs...) if err != nil { return nil, err } @@ -107,9 +121,9 @@ func GetContainerLists(runtime *libpod.Runtime, options entities.ContainerListOp } // GetExternalContainerLists returns list of external containers for e.g. created by buildah -func GetExternalContainerLists(runtime *libpod.Runtime) ([]entities.ListContainer, error) { +func GetExternalContainerLists(runtime *libpod.Runtime, filterExtFuncs ...entities.ExternalContainerFilter) ([]entities.ListContainer, error) { var ( - pss = []entities.ListContainer{} + pss = []*entities.ListContainer{} ) externCons, err := runtime.StorageContainers() @@ -128,10 +142,31 @@ func GetExternalContainerLists(runtime *libpod.Runtime) ([]entities.ListContaine case err != nil: return nil, err default: - pss = append(pss, listCon) + pss = append(pss, &listCon) } } - return pss, nil + + filteredPss := applyExternalContainersFilters(pss, filterExtFuncs...) + + return filteredPss, nil +} + +// Apply container filters on bunch of external container lists +func applyExternalContainersFilters(containersList []*entities.ListContainer, filters ...entities.ExternalContainerFilter) []entities.ListContainer { + ctrsFiltered := make([]entities.ListContainer, 0, len(containersList)) + + for _, ctr := range containersList { + include := true + for _, filter := range filters { + include = include && filter(ctr) + } + + if include { + ctrsFiltered = append(ctrsFiltered, *ctr) + } + } + + return ctrsFiltered } // ListContainerBatch is used in ps to reduce performance hits by "batching" diff --git a/test/e2e/ps_test.go b/test/e2e/ps_test.go index c79e1e55e1..da5c826793 100644 --- a/test/e2e/ps_test.go +++ b/test/e2e/ps_test.go @@ -3,12 +3,15 @@ package integration import ( + "context" "fmt" "regexp" "sort" "strconv" + "github.com/containers/buildah" . "github.com/containers/podman/v5/test/utils" + "github.com/containers/storage" "github.com/containers/storage/pkg/stringid" "github.com/docker/go-units" . "github.com/onsi/ginkgo/v2" @@ -405,6 +408,37 @@ var _ = Describe("Podman ps", func() { Expect(actual).ToNot(ContainSubstring("NAMES")) }) + // This test checks a ps filtering by container command/entrypoint + // To improve the test reliability a container ID is also checked + It("podman ps filter by container command", func() { + matchedSession := podmanTest.Podman([]string{"run", "-d", "--name", "matched", ALPINE, "top"}) + matchedSession.WaitWithDefaultTimeout() + containedID := matchedSession.OutputToString() // save container ID returned by the run command + Expect(containedID).ShouldNot(BeEmpty()) + Expect(matchedSession).Should(ExitCleanly()) + + matchedSession = podmanTest.Podman([]string{"ps", "-a", "--no-trunc", "--noheading", "--filter", "command=top"}) + matchedSession.WaitWithDefaultTimeout() + Expect(matchedSession).Should(ExitCleanly()) + + output := matchedSession.OutputToStringArray() + Expect(output).To(HaveLen(1)) + Expect(output).Should(ContainElement(ContainSubstring(containedID))) + + unmatchedSession := podmanTest.Podman([]string{"run", "-d", "--name", "unmatched", ALPINE, "sh"}) + unmatchedSession.WaitWithDefaultTimeout() + containedID = unmatchedSession.OutputToString() // save container ID returned by the run command + Expect(containedID).ShouldNot(BeEmpty()) + Expect(unmatchedSession).Should(ExitCleanly()) + + unmatchedSession = podmanTest.Podman([]string{"ps", "-a", "--no-trunc", "--noheading", "--filter", "command=fakecommand"}) + unmatchedSession.WaitWithDefaultTimeout() + Expect(unmatchedSession).Should(ExitCleanly()) + + output = unmatchedSession.OutputToStringArray() + Expect(output).To(BeEmpty()) + }) + It("podman ps mutually exclusive flags", func() { session := podmanTest.Podman([]string{"ps", "-aqs"}) session.WaitWithDefaultTimeout() @@ -896,4 +930,169 @@ var _ = Describe("Podman ps", func() { Expect(session.OutputToString()).To(Or(Equal(net1+","+net2), Equal(net2+","+net1))) }) + // This test checks ps filtering of external container by container command/entrypoint + It("podman ps filter external by container command", func() { + opts := storage.StoreOptions{ + ImageStore: podmanTest.ImageCacheDir, + RunRoot: podmanTest.RunRoot, + GraphRoot: podmanTest.Root, + } + + storage, err := storage.GetStore(opts) + Expect(err).ShouldNot(HaveOccurred()) + + builder, err := buildah.NewBuilder(context.TODO(), storage, buildah.BuilderOptions{FromImage: ALPINE, Container: "top"}) + Expect(err).ShouldNot(HaveOccurred()) + + err = builder.Save() + Expect(err).ShouldNot(HaveOccurred()) + + session := podmanTest.Podman([]string{"ps", "--external", "--no-trunc", "--noheading", "--filter", "command=buildah"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(ExitCleanly()) + + output := session.OutputToStringArray() + Expect(output).To(HaveLen(1)) + }) + + // This test checks ps filtering of external container by container name + It("podman ps filter external by container name", func() { + opts := storage.StoreOptions{ + ImageStore: podmanTest.ImageCacheDir, + RunRoot: podmanTest.RunRoot, + GraphRoot: podmanTest.Root, + } + + storage, err := storage.GetStore(opts) + Expect(err).ShouldNot(HaveOccurred()) + + builder, err := buildah.NewBuilder(context.TODO(), storage, buildah.BuilderOptions{FromImage: ALPINE, Container: "top"}) + Expect(err).ShouldNot(HaveOccurred()) + + err = builder.Save() + Expect(err).ShouldNot(HaveOccurred()) + + session := podmanTest.Podman([]string{"ps", "--external", "--no-trunc", "--noheading", "--filter", "name=top"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(ExitCleanly()) + + output := session.OutputToStringArray() + Expect(output).To(HaveLen(1)) + }) + + // This test checks ps filtering of external container by container id + It("podman ps filter external by container id", func() { + opts := storage.StoreOptions{ + ImageStore: podmanTest.ImageCacheDir, + RunRoot: podmanTest.RunRoot, + GraphRoot: podmanTest.Root, + } + + storage, err := storage.GetStore(opts) + Expect(err).ShouldNot(HaveOccurred()) + + builder, err := buildah.NewBuilder(context.TODO(), storage, buildah.BuilderOptions{FromImage: ALPINE, Container: "top"}) + Expect(err).ShouldNot(HaveOccurred()) + + err = builder.Save() + Expect(err).ShouldNot(HaveOccurred()) + + session := podmanTest.Podman([]string{"ps", "--external", "--no-trunc", "--noheading", "--filter", "id=" + builder.ContainerID}) + session.WaitWithDefaultTimeout() + Expect(session).Should(ExitCleanly()) + + output := session.OutputToStringArray() + Expect(output).To(HaveLen(1)) + }) + + // This test checks ps filtering of external container by container label + It("podman ps filter external by container ancestor", func() { + opts := storage.StoreOptions{ + ImageStore: podmanTest.ImageCacheDir, + RunRoot: podmanTest.RunRoot, + GraphRoot: podmanTest.Root, + } + + storage, err := storage.GetStore(opts) + Expect(err).ShouldNot(HaveOccurred()) + + builder, err := buildah.NewBuilder(context.TODO(), storage, buildah.BuilderOptions{FromImage: ALPINE, Container: "top"}) + Expect(err).ShouldNot(HaveOccurred()) + + err = builder.Save() + Expect(err).ShouldNot(HaveOccurred()) + + session := podmanTest.Podman([]string{"ps", "--external", "--no-trunc", "--noheading", "--filter", "ancestor=" + ALPINE}) + session.WaitWithDefaultTimeout() + Expect(session).Should(ExitCleanly()) + + output := session.OutputToStringArray() + Expect(output).To(HaveLen(1)) + }) + + // This test checks ps filtering of external container created earlier than a given + It("podman ps filter external by container created earlier than a given", func() { + opts := storage.StoreOptions{ + ImageStore: podmanTest.ImageCacheDir, + RunRoot: podmanTest.RunRoot, + GraphRoot: podmanTest.Root, + } + + storage, err := storage.GetStore(opts) + Expect(err).ShouldNot(HaveOccurred()) + + builder, err := buildah.NewBuilder(context.TODO(), storage, buildah.BuilderOptions{FromImage: ALPINE, Container: "early"}) + Expect(err).ShouldNot(HaveOccurred()) + + err = builder.Save() + Expect(err).ShouldNot(HaveOccurred()) + + builder, err = buildah.NewBuilder(context.TODO(), storage, buildah.BuilderOptions{FromImage: ALPINE, Container: "late"}) + Expect(err).ShouldNot(HaveOccurred()) + + err = builder.Save() + Expect(err).ShouldNot(HaveOccurred()) + + session := podmanTest.Podman([]string{"ps", "--external", "--no-trunc", "--noheading", "--filter", "before=late"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(ExitCleanly()) + + output := session.OutputToStringArray() + Expect(output).To(HaveLen(1)) + fmt.Println(output) + Expect(output).Should(ContainElement(ContainSubstring("early"))) + }) + + // This test checks ps filtering of external container created since a given + It("podman ps filter external by container created since a given", func() { + opts := storage.StoreOptions{ + ImageStore: podmanTest.ImageCacheDir, + RunRoot: podmanTest.RunRoot, + GraphRoot: podmanTest.Root, + } + + storage, err := storage.GetStore(opts) + Expect(err).ShouldNot(HaveOccurred()) + + builder, err := buildah.NewBuilder(context.TODO(), storage, buildah.BuilderOptions{FromImage: ALPINE, Container: "early"}) + Expect(err).ShouldNot(HaveOccurred()) + + err = builder.Save() + Expect(err).ShouldNot(HaveOccurred()) + + builder, err = buildah.NewBuilder(context.TODO(), storage, buildah.BuilderOptions{FromImage: ALPINE, Container: "late"}) + Expect(err).ShouldNot(HaveOccurred()) + + err = builder.Save() + Expect(err).ShouldNot(HaveOccurred()) + + session := podmanTest.Podman([]string{"ps", "--external", "--no-trunc", "--noheading", "--filter", "since=early"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(ExitCleanly()) + + output := session.OutputToStringArray() + Expect(output).To(HaveLen(1)) + fmt.Println(output) + Expect(output).Should(ContainElement(ContainSubstring("late"))) + }) })