[WIP/POC] Add navigation properties

src/libraries/Microsoft.PowerFx.Connectors/Internal/Wrappers/ISwaggerSchema.cs

@@ -38,6 +38,12 @@ internal interface ISwaggerSchema : ISwaggerExtensions
         // SalesForce specific
         string RelationshipName { get; }
 
+        string ForeignKey { get; }
+
+        string SourceField { get; }
+
+        string RelationshipType { get; }
+
         // SalesForce specific
         string DataType { get; }
     }

src/libraries/Microsoft.PowerFx.Connectors/Internal/Wrappers/SwaggerJsonSchema.cs

@@ -130,6 +130,12 @@ public IList<IOpenApiAny> Enum
 
         public string RelationshipName => SafeGetString("relationshipName");
 
+        public string ForeignKey => SafeGetString("destinationKey");
+
+        public string SourceField => SafeGetString("sourceField");
+
+        public string RelationshipType => SafeGetString("relationshipType");
+
         public string DataType => SafeGetString("datatype");
 
         private string SafeGetString(string key)

src/libraries/Microsoft.PowerFx.Connectors/Internal/Wrappers/SwaggerSchema.cs

@@ -87,15 +87,18 @@ public IList<IOpenApiAny> Enum
 
         public ISwaggerReference Reference => SwaggerReference.New(_schema?.Reference);
 
-        // SalesForce specific
         public ISet<string> ReferenceTo => null;
 
-        // SalesForce specific
         public string RelationshipName => null;
 
-        // SalesForce specific
         public string DataType => null;
 
+        public string ForeignKey => throw new NotImplementedException();
+
+        public string SourceField => throw new NotImplementedException();
+
+        public string RelationshipType => throw new NotImplementedException();
+
         private static string GetTypeFromExtension(OpenApiSchema schema)
         {
             if (schema == null || schema.Extensions == null)
@@ -104,8 +107,8 @@ private static string GetTypeFromExtension(OpenApiSchema schema)
             }
 
             // OpenAI special extension to indicate the type of the property
-            if (schema.Extensions.TryGetValue("x-oaiTypeLabel", out IOpenApiExtension ext) && 
-                ext is OpenApiString str && 
+            if (schema.Extensions.TryGetValue("x-oaiTypeLabel", out IOpenApiExtension ext) &&
+                ext is OpenApiString str &&
                 !string.IsNullOrEmpty(str.Value))
             {
                 return str.Value;

src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs

@@ -658,8 +658,7 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar
                                 // Here, we have a circular reference and default to a string
                                 return new ConnectorType(schema, openApiParameter, FormulaType.String, hiddenfields.ToRecordType());
                             }
-
-                            //ConnectorType propertyType = new OpenApiParameter() { Name = propLogicalName, Required = schema.Required.Contains(propLogicalName), Schema = kv.Value, Extensions = kv.Value.Extensions }.GetConnectorType(settings.Stack(schemaIdentifier));
+
                             ConnectorType propertyType = new SwaggerParameter(propLogicalName, schema.Required.Contains(propLogicalName), kv.Value, kv.Value.Extensions).GetConnectorType(settings.Stack(schemaIdentifier));
 
                             settings.UnStack();
@@ -684,6 +683,18 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar
                             }
                         }
 
+                        bool HasExternalRef(ConnectorType c) => c.ExternalTables?.Any() == true && !string.IsNullOrEmpty(c.SourceField);
+
+                        // Link navigation properties to source fields
+                        foreach (ConnectorType ct in connectorTypes.Where(c => HasExternalRef(c))
+                                                                   .Union(hiddenConnectorTypes.Where(c => HasExternalRef(c))))
+                        {
+                            ConnectorType sourceField = connectorTypes.FirstOrDefault(c => c.Name == ct.SourceField) ??
+                                                  hiddenConnectorTypes.FirstOrDefault(c => c.Name == ct.SourceField);
+
+                            sourceField?.AddNavigationProperty(ct);                            
+                        }
+
                         return new ConnectorType(schema, openApiParameter, fields.ToRecordType(), hiddenfields.ToRecordType(), connectorTypes.ToArray(), hiddenConnectorTypes.ToArray());
                     }
 

src/libraries/Microsoft.PowerFx.Connectors/Public/CdpDelegationInfo.cs

@@ -2,16 +2,26 @@
 // Licensed under the MIT license.
 
 using System;
+using System.Collections.Generic;
+using Microsoft.PowerFx.Connectors;
 
 namespace Microsoft.PowerFx.Core.Entities
 {
     // Used by ServiceCapabilities.ToDelegationInfo for managing CDP x-ms-capabilities
     internal class CdpDelegationInfo : TableDelegationInfo
     {
+        private readonly ConnectorType _connectorType;
+
+        public CdpDelegationInfo(ConnectorType connectorType)
+            : base()
+        {
+            _connectorType = connectorType;
+        }
+
         public override ColumnCapabilitiesDefinition GetColumnCapability(string fieldName)
         {
             if (ColumnsCapabilities.TryGetValue(fieldName, out ColumnCapabilitiesBase columnCapabilitiesBase))
-            {               
+            {
                 return columnCapabilitiesBase switch
                 {
                     ColumnCapabilities columnCapabilities => columnCapabilities.Definition,
@@ -21,5 +31,10 @@ public override ColumnCapabilitiesDefinition GetColumnCapability(string fieldNam
 
             return null;
         }
+
+        public override IReadOnlyList<INavigationProperty> GetNavigationProperties(string fieldName)
+        {
+            return _connectorType.GetNavigationProperties(fieldName);
+        }
     }
 }

src/libraries/Microsoft.PowerFx.Connectors/Public/CdpTableValue.cs

@@ -18,7 +18,7 @@ public class CdpTableValue : TableValue, IRefreshable, IDelegatableTableValue
     {
         public bool IsDelegable => _tabularService.IsDelegable;
 
-        protected internal readonly CdpService _tabularService;        
+        protected internal readonly CdpService _tabularService;
 
         internal readonly IReadOnlyDictionary<string, Relationship> Relationships;
 
@@ -27,12 +27,12 @@ public class CdpTableValue : TableValue, IRefreshable, IDelegatableTableValue
         internal readonly HttpClient HttpClient;
 
         public RecordType RecordType => _tabularService?.RecordType;
-        
+
         internal CdpTableValue(CdpService tabularService, IReadOnlyDictionary<string, Relationship> relationships)
             : base(IRContext.NotInSource(tabularService.TableType))
         {
             _tabularService = tabularService;
-            Relationships = relationships;                        
+            Relationships = relationships;
             HttpClient = tabularService.HttpClient;
         }
 
@@ -44,10 +44,18 @@ internal CdpTableValue(IRContext irContext)
 
         public override IEnumerable<DValue<RecordValue>> Rows => GetRowsAsync(null, null, CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult();
 
-        public DelegationParameterFeatures SupportedFeatures => DelegationParameterFeatures.Filter |
+#pragma warning disable SA1025 // CodeMustNotContainMultipleWhitespaceInARow
+
+        public DelegationParameterFeatures SupportedFeatures =>
+                DelegationParameterFeatures.Filter |
                 DelegationParameterFeatures.Top |
-                DelegationParameterFeatures.Columns | // $select
-                DelegationParameterFeatures.Sort; // $orderby
+                DelegationParameterFeatures.Columns |      // $select
+                DelegationParameterFeatures.Sort |         // $orderby
+                DelegationParameterFeatures.ApplyGroupBy | // $groupby
+                DelegationParameterFeatures.ApplyJoin |    // $apply
+                DelegationParameterFeatures.Expand;        // $expand
+
+#pragma warning restore SA1025
 
         public async Task<IReadOnlyCollection<DValue<RecordValue>>> GetRowsAsync(IServiceProvider services, DelegationParameters parameters, CancellationToken cancel)
         {
@@ -76,26 +84,35 @@ public Task<FormulaValue> ExecuteQueryAsync(IServiceProvider services, Delegatio
         {
             throw new NotImplementedException();
         }
-    }   
+    }
 
     internal static class ODataParametersExtensions
     {
+#pragma warning disable SA1025 // CodeMustNotContainMultipleWhitespaceInARow
+
         public static ODataParameters ToOdataParameters(this DelegationParameters parameters)
         {
-            DelegationParameterFeatures allowedFeatures = 
-                DelegationParameterFeatures.Filter | 
-                DelegationParameterFeatures.Top | 
-                DelegationParameterFeatures.Columns | // $select
-                DelegationParameterFeatures.Sort;     // $orderby
+            DelegationParameterFeatures allowedFeatures =
+                DelegationParameterFeatures.Filter |
+                DelegationParameterFeatures.Top |
+                DelegationParameterFeatures.Columns |      // $select
+                DelegationParameterFeatures.Sort |         // $orderby
+                DelegationParameterFeatures.ApplyGroupBy | // $groupby
+                DelegationParameterFeatures.ApplyJoin |    // $apply
+                DelegationParameterFeatures.Expand;        // $expand
+#pragma warning restore SA1025
 
             parameters.EnsureOnlyFeatures(allowedFeatures);
 
+            string expand = parameters.GetExpand();
+
             ODataParameters op = new ODataParameters()
             {
                 Filter = parameters.GetOdataFilter(),
                 Top = parameters.Top.GetValueOrDefault(),
-                Select = parameters.GetColumns(),
-                OrderBy = parameters.GetOrderBy()
+                Select = string.IsNullOrEmpty(expand) ? parameters.GetColumns() : null,
+                OrderBy = parameters.GetOrderBy(),
+                Expand = expand
             };
 
             return op;

src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs

@@ -10,6 +10,7 @@
 using Microsoft.OpenApi.Any;
 using Microsoft.OpenApi.Interfaces;
 using Microsoft.PowerFx.Core;
+using Microsoft.PowerFx.Core.Entities;
 using Microsoft.PowerFx.Core.Localization;
 using Microsoft.PowerFx.Core.Utils;
 using Microsoft.PowerFx.Types;
@@ -21,7 +22,7 @@ namespace Microsoft.PowerFx.Connectors
     // FormulaType is used to represent the type of the parameter in the Power Fx expression as used in Power Apps
     // ConnectorType contains more details information coming from the swagger file and extensions
     [DebuggerDisplay("{FormulaType._type}")]
-    public class ConnectorType : SupportsConnectorErrors
+    public class ConnectorType : SupportsConnectorErrors, ISupportsNavigationProperties
     {
         // "name"
         public string Name { get; internal set; }
@@ -119,11 +120,19 @@ public class ConnectorType : SupportsConnectorErrors
         internal ISwaggerSchema Schema { get; private set; } = null;
 
         // Relationships to external tables
-        internal List<string> ExternalTables { get; set; }
+        internal List<string> ExternalTables { get; set; } = null;
 
-        internal string RelationshipName { get; set; }
+        internal string RelationshipName { get; set; } = null;
 
-        internal string ForeignKey { get; set; }
+        internal string ForeignKey { get; set; } = null;
+
+        internal string SourceField { get; set; } = null;
+
+        internal string RelationshipType { get; set; } = null;
+
+        internal IReadOnlyList<ConnectorType> NavigationPropertyReferences => _navigationPropertyReferences;
+
+        private List<ConnectorType> _navigationPropertyReferences = null;
 
         internal ConnectorType(ISwaggerSchema schema, ISwaggerParameter openApiParameter, FormulaType formulaType, ErrorResourceKey warning = default, IEnumerable<KeyValuePair<DName, DName>> list = null, bool isNumber = false)
         {
@@ -150,13 +159,14 @@ internal ConnectorType(ISwaggerSchema schema, ISwaggerParameter openApiParameter
             KeyOrder = schema.GetKeyOrder();
             Permission = schema.GetPermission();
 
-            // We only support one reference for now
-            // SalesForce only
+            // We only support one reference for now            
             if (schema.ReferenceTo != null && schema.ReferenceTo.Count == 1)
             {
                 ExternalTables = new List<string>(schema.ReferenceTo);
                 RelationshipName = schema.RelationshipName;
-                ForeignKey = null; // SalesForce doesn't provide it, defaults to "Id"
+                ForeignKey = schema.ForeignKey; // SalesForce doesn't provide it, defaults to "Id"
+                SourceField = schema.SourceField;
+                RelationshipType = schema.RelationshipType;
             }
 
             Fields = Array.Empty<ConnectorType>();
@@ -312,6 +322,12 @@ internal ConnectorType(ConnectorType connectorType, ConnectorType[] fields, Form
             _warnings = connectorType._warnings;
         }
 
+        internal void AddNavigationProperty(ConnectorType navigationProperty)
+        {
+            _navigationPropertyReferences ??= new List<ConnectorType>();
+            _navigationPropertyReferences.Add(navigationProperty);
+        }
+
         internal DisplayNameProvider DisplayNameProvider
         {
             get
@@ -361,5 +377,31 @@ private Dictionary<string, FormulaValue> GetEnum()
 
             return enumDisplayNames.Zip(enumValues, (dn, ev) => new KeyValuePair<string, FormulaValue>(dn, ev)).ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
         }
+
+        public IReadOnlyList<INavigationProperty> GetNavigationProperties(string fieldName)
+        {
+            ConnectorType field = Fields.FirstOrDefault(ct => ct.Name == fieldName);
+
+            if (field?.NavigationPropertyReferences?.Any() != true)
+            {
+                return null;
+            }
+
+            List<INavigationProperty> navProps = new List<INavigationProperty>();
+
+            foreach (ConnectorType navPropRef in field.NavigationPropertyReferences)
+            {
+                navProps.Add(new NavigationProperty()
+                {
+                    Name = navPropRef.Name,
+                    SourceField = navPropRef.SourceField,
+                    ForeignKey = navPropRef.ForeignKey,
+                    RelationshipName = navPropRef.RelationshipName,
+                    RelationshipType = navPropRef.RelationshipType,
+                });
+            }
+
+            return navProps;
+        }
     }
 }

src/libraries/Microsoft.PowerFx.Connectors/Public/NavigationProperty.cs

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using Microsoft.PowerFx.Core.Entities;
+
+namespace Microsoft.PowerFx.Connectors
+{
+    public class NavigationProperty : INavigationProperty
+    {
+        public string Name { get; init; }
+
+        public string SourceField { get; init; }
+
+        public string ForeignKey { get; init; }
+
+        public string RelationshipName { get; init; }
+
+        public string RelationshipType { get; init; }
+    }
+}

src/libraries/Microsoft.PowerFx.Connectors/Tabular/Capabilities/ServiceCapabilities.cs

@@ -113,7 +113,7 @@ public ServiceCapabilities(SortRestriction sortRestriction, FilterRestriction fi
             Contracts.AssertValue(pagingCapabilities);
 
             SortRestriction = sortRestriction;
-            FilterRestriction = filterRestriction;            
+            FilterRestriction = filterRestriction;
             PagingCapabilities = pagingCapabilities;
             SelectionRestriction = selectionRestriction;
             GroupRestriction = groupRestriction;
@@ -181,15 +181,15 @@ public static TableDelegationInfo ToDelegationInfo(ServiceCapabilities serviceCa
             Dictionary<string, string> columnWithRelationships = connectorType.Fields.Where(f => f.ExternalTables?.Any() == true).Select(f => (f.Name, f.ExternalTables.First())).ToDictionary(tpl => tpl.Name, tpl => tpl.Item2);
             string[] primaryKeyNames = connectorType.Fields.Where(f => f.KeyType == ConnectorKeyType.Primary).OrderBy(f => f.KeyOrder).Select(f => f.Name).ToArray();
 
-            return new CdpDelegationInfo()
+            return new CdpDelegationInfo(connectorType)
             {
                 TableName = tableName,
                 IsReadOnly = isReadOnly,
                 DatasetName = datasetName,
                 SortRestriction = sortRestriction,
                 FilterRestriction = filterRestriction,
                 SelectionRestriction = selectionRestriction,
-                GroupRestriction = groupRestriction,                
+                GroupRestriction = groupRestriction,
                 FilterSupportedFunctions = serviceCapabilities?.FilterSupportedFunctionsEnum,
                 PagingCapabilities = pagingCapabilities,
                 SupportsRecordPermission = serviceCapabilities?.SupportsRecordPermission ?? false,

src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs

@@ -28,6 +28,10 @@ internal class CdpTableResolver : ICdpTableResolver
 
         private readonly bool _doubleEncoding;
 
+#if DEBUG
+        internal Func<string, string, string> UpdateShema = null;
+#endif
+
         public CdpTableResolver(CdpTable tabularTable, HttpClient httpClient, string uriPrefix, bool doubleEncoding, ConnectorLogger logger = null)
         {
             _tabularTable = tabularTable;
@@ -57,6 +61,14 @@ public async Task<ConnectorType> ResolveTableAsync(string tableName, Cancellatio
 
             string text = await CdpServiceBase.GetObject(_httpClient, $"Get table metadata", uri, null, cancellationToken, Logger).ConfigureAwait(false);
 
+#if DEBUG
+            // Used to return an alternate result
+            if (UpdateShema != null)
+            {
+                text = UpdateShema(tableName, text);
+            }
+#endif
+
             if (string.IsNullOrWhiteSpace(text))
             {
                 return null;