PS: Restructure generated files

powershell/misc/README.md

@@ -12,9 +12,9 @@ Run the following commands
 
     ```
     # Clone dotnet/dotnet-api-docs
-    git clone https://github.com/dotnet/dotnet-api-docs
+    git clone https://github.com/dotnet/dotnet-api-docs --depth 1
     # Clone MicrosoftDocs/powershell-docs-sdk-dotnet
-    git clone [email protected]:MicrosoftDocs/powershell-docs-sdk-dotnet.git
+    git clone [email protected]:MicrosoftDocs/powershell-docs-sdk-dotnet.git --depth 1
     # Generate data extensions
     python3 misc/typemodelgen.py dotnet-api-docs/xml/ powershell-docs-sdk-dotnet/dotnet/xml
     ```

powershell/misc/typemodelgen.py

@@ -2,16 +2,25 @@
 from pathlib import Path
 import sys
 import os
+import re
 from collections import defaultdict
 
 
 def fixup(t):
     """Sometimes the docs specify a type that doesn't align with what
     PowerShell reports. This function fixes up those types so that it aligns with PowerShell.
     """
-    if t.startswith("System.ReadOnlySpan<"):
-        return "System.String"
-    return t
+    if t.startswith("system.readonlyspan<"):
+        return "system.string"
+    # A regular expression that matches strings like a.b.c<T, U, W>
+    # and replacee it with a.b.c
+    return re.sub(r"<.*>", "", t)
+
+def skipQualifier(name):
+    """Removes the qualifier from the name."""
+    # A regular expression that matches strings like a.b.c and returns c
+    # and replaces it with c
+    return re.sub(r".*\.", "", name)
 
 
 def isStatic(member):
@@ -22,7 +31,7 @@ def isStatic(member):
     return False
 
 
-def isA(x):
+def isA(member, x):
     """Returns True if member is an `x`."""
     for child in member:
         if child.tag == "MemberType" and child.text == x:
@@ -77,22 +86,29 @@ def generateTypeModels(arg):
         try:
             if not file_path.name.endswith(".xml"):
                 continue
-
+            
             if not file_path.is_file():
                 continue
-
+            
             tree = ET.parse(str(file_path))
             root = tree.getroot()
             if not root.tag == "Type":
                 continue
 
             thisType = root.attrib["FullName"]
 
-            if "`" in file_path.stem or "+" in file_path.stem:
-                continue  # Skip generics (and nested types?) for now
+            parentName = file_path.parent.name
+            # Remove `and + in parentName
+            parentName = parentName.replace("`", "").replace("+", "")
 
-            folderName = file_path.parent.name.replace(".", "")
-            filename = folderName + "/model.yml"
+            # Remove ` in file_path.stem
+            # and + in file_path.stem
+            if thisType == "":
+                print("Error: Empty type name")
+                continue
+
+            folderName = "generated/" + parentName
+            filename = folderName + ".typemodel.yml"
             s = set()
             for elem in root.findall(".//Members/Member"):
                 name = elem.attrib["MemberName"]
@@ -106,10 +122,10 @@ def generateTypeModels(arg):
                 startSelectorMarker = ""
                 endSelectorMarker = ""
                 if isField(elem):
-                    startSelectorMarker = "Field"
+                    startSelectorMarker = "Member"
                     endSelectorMarker = ""
                 if isProperty(elem):
-                    startSelectorMarker = "Property"
+                    startSelectorMarker = "Member"
                     endSelectorMarker = ""
                 if isMethod(elem):
                     startSelectorMarker = "Method"
@@ -134,8 +150,9 @@ def generateTypeModels(arg):
                 returnType = elem.find(".//ReturnValue/ReturnType").text
                 if returnType == "System.Void":
                     continue  # Don't generate type summaries for void methods
+
                 s.add(
-                    f'    - ["{fixup(returnType)}", "{thisType + staticMarker}", "{startSelectorMarker}[{name}]{endSelectorMarker}"]\n'
+                    f'    - ["{fixup(returnType.lower())}", "{fixup(thisType.lower()) + staticMarker}", "{startSelectorMarker}[{skipQualifier(fixup(name.lower()))}]{endSelectorMarker}"]\n'
                 )
 
             summaries[filename].update(s)
@@ -152,7 +169,8 @@ def writeModels():
         if len(s) == 0:
             continue
         os.makedirs(os.path.dirname(filename), exist_ok=True)
-        with open(filename, "x") as file:
+        with open(filename, "w") as file:
+            file.write("# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.\n")
             file.write("extensions:\n")
             file.write("  - addsTo:\n")
             file.write("      pack: microsoft/powershell-all\n")

powershell/ql/lib/qlpack.yml

@@ -16,4 +16,6 @@ dependencies:
 dataExtensions:
   - semmle/code/powershell/frameworks/**/model.yml
   - semmle/code/powershell/frameworks/**/*.model.yml
+  - semmle/code/powershell/frameworks/**/typemodel.yml
+  - semmle/code/powershell/frameworks/**/*.typemodel.yml
 warnOnImplicitThis: true

powershell/ql/lib/semmle/code/powershell/Frameworks.qll

@@ -2,6 +2,6 @@
  * Helper file that imports all framework modeling.
  */
 
-import semmle.code.powershell.frameworks.SystemManagementAutomationRunspaces.Runspaces
-import semmle.code.powershell.frameworks.SystemManagementAutomationPowerShell.PowerShell
-import semmle.code.powershell.frameworks.SystemManagementAutomationEngineIntrinsics.EngineIntrinsics
+import semmle.code.powershell.frameworks.Runspaces
+import semmle.code.powershell.frameworks.PowerShell
+import semmle.code.powershell.frameworks.EngineIntrinsics

powershell/ql/lib/semmle/code/powershell/frameworks/SystemManagementAutomationEngineIntrinsics/EngineIntrinsics.qll → powershell/ql/lib/semmle/code/powershell/frameworks/EngineIntrinsics.qll

@@ -5,8 +5,8 @@ private import semmle.code.powershell.dataflow.internal.DataFlowPublic as DataFl
 module EngineIntrinsics {
   private class EngineIntrinsicsGlobalEntry extends ModelInput::TypeModel {
     override DataFlow::Node getASource(string type) {
-      type = "System.Management.Automation.EngineIntrinsics" and
-      result.asExpr().getExpr().(VarReadAccess).getVariable().matchesName("ExecutionContext")
+      type = "system.management.automation.engineintrinsics" and
+      result.asExpr().getExpr().(VarReadAccess).getVariable().matchesName("executioncontext")
     }
   }
 }

powershell/ql/lib/semmle/code/powershell/frameworks/Microsoft.PowerShell.Utility.model.yml

@@ -0,0 +1,77 @@
+extensions:
+  - addsTo:
+      pack: microsoft/powershell-all
+      extensible: sourceModel
+    data:
+      - ["microsoft.powershell.utility!", "Method[read-host].ReturnValue", "stdin"]
+      - ["microsoft.powershell.utility!", "Method[select-xml].ReturnValue[path]", "file"]
+      - ["microsoft.powershell.utility!", "Method[format-hex].ReturnValue[path]", "file"]
+
+  - addsTo:
+      pack: microsoft/powershell-all
+      extensible: summaryModel
+    data:
+      - ["microsoft.powershell.utility!", "Method[get-unique]", "Argument[-inputobject,pipeline].Element[?]", "ReturnValue.Element[?]", "value"]
+      - ["microsoft.powershell.utility!", "Method[join-string]", "Argument[-inputobject,pipeline].Element[?]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[convertfrom-clixmlreference]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[convertfrom-csv]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[convertfrom-json]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[convertfrom-markdown]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[convertfrom-sddlstring]", "Argument[-sddl,0,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[convertfrom-stringdata]", "Argument[-stringdata,0,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[convertto-clixml]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[convertto-csv]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[convertto-html]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[convertto-json]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[convertto-xml]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[out-string]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[select-object]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[select-string]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[select-xml]", "Argument[-content,-path,-xml]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[sort-object]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[tee-object]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[write-output]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[format-custom]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[format-hex]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[format-list]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[format-table]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[format-wide]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[get-unique]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+      - ["microsoft.powershell.utility!", "Method[join-string]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
+
+  - addsTo:
+      pack: microsoft/powershell-all
+      extensible: typeModel
+    data:
+      - ["system.datetime", "microsoft.powershell.utility!", "Method[get-date].ReturnValue"]
+      - ["system.object", "microsoft.powershell.utility!", "Method[convertfrom-clixmlreference].ReturnValue"]
+      - ["pscustomobject", "microsoft.powershell.utility!", "Method[convertFrom-json].ReturnValue"]
+      - ["system.management.automation.hashtable", "microsoft.powershell.utility!", "Method[convertFrom-json].ReturnValue"]
+      - ["microsoft.powershell.markdownrender.markdownInfo", "microsoft.powershell.utility!", "Method[convertfrom-markdown].ReturnValue"]
+      - ["pscustomobject", "microsoft.powershell.utility!", "Method[convertfrom-sddlstring].ReturnValue"]
+      - ["system.collections.hashtable", "microsoft.powershell.utility!", "Method[convertfrom-stringdata].ReturnValue"]
+      - ["system.string", "microsoft.powershell.utility!", "Method[convertto-clixml].ReturnValue"]
+      - ["system.string", "microsoft.powershell.utility!", "Method[convertto-csv].ReturnValue"]
+      - ["system.string[]", "microsoft.powershell.utility!", "Method[convertto-csv].ReturnValue"]
+      - ["system.string", "microsoft.powershell.utility!", "Method[convertto-html].ReturnValue"]
+      - ["system.string[]", "microsoft.powershell.utility!", "Method[convertto-html].ReturnValue"]
+      - ["system.string", "microsoft.powershell.utility!", "Method[convertto-json].ReturnValue"]
+      - ["system.string[]", "microsoft.powershell.utility!", "Method[convertto-json].ReturnValue"]
+      - ["system.string", "microsoft.powershell.utility!", "Method[convertto-xml].ReturnValue"]
+      - ["system.string[]", "microsoft.powershell.utility!", "Method[convertto-xml].ReturnValue"]
+      - ["system.string", "microsoft.powershell.utility!", "Method[out-string].ReturnValue"]
+      - ["pscustomobject", "microsoft.powershell.utility!", "Method[select-object].ReturnValue"]
+      - ["microsoft.powerShell.commands.matchinfo", "microsoft.powershell.utility!", "Method[select-string].ReturnValue"]
+      - ["system.boolean", "microsoft.powershell.utility!", "Method[select-string].ReturnValue"]
+      - ["system.string", "microsoft.powershell.utility!", "Method[select-string].ReturnValue"]
+      - ["microsoft.powerShell.commands.selectxmlinfo", "microsoft.powershell.utility!", "Method[select-xml].ReturnValue"]
+      - ["pscustomobject", "microsoft.powershell.utility!", "Method[sort-object].ReturnValue"]
+      - ["pscustomobject", "microsoft.powershell.utility!", "Method[tee-object].ReturnValue"]
+      - ["pscustomobject", "microsoft.powershell.utility!", "Method[write-output].ReturnValue"]
+      - ["microsoft.powershell.commands.internal.format", "microsoft.powershell.utility!", "Method[format-custom].ReturnValue"]
+      - ["microsoft.powershell.commands.bytecollection", "microsoft.powershell.utility!", "Method[format-hex].ReturnValue"]
+      - ["microsoft.powershell.commands.internal.format", "microsoft.powershell.utility!", "Method[format-list].ReturnValue"]
+      - ["microsoft.powershell.commands.internal.format", "microsoft.powershell.utility!", "Method[format-table].ReturnValue"]
+      - ["microsoft.powershell.commands.internal.format", "microsoft.powershell.utility!", "Method[format-wide].ReturnValue"]
+      - ["pscustomobject", "microsoft.powershell.utility!", "Method[get-unique].ReturnValue"]
+      - ["system.string", "microsoft.powershell.utility!", "Method[join-string].ReturnValue"]

powershell/ql/lib/semmle/code/powershell/frameworks/Microsoft.Win32.model.yml

@@ -0,0 +1,9 @@
+extensions:
+  - addsTo:
+      pack: microsoft/powershell-all
+      extensible: sourceModel
+    data:
+      - ["microsoft.win32.registry!", "Method[getvalue].ReturnValue", "windows-registry"]
+      - ["microsoft.win32.registrykey", "Method[getvalue].ReturnValue", "windows-registry"]
+      - ["microsoft.win32.registrykey", "Method[getvaluenames].ReturnValue", "windows-registry"]
+      - ["microsoft.win32.registrykey", "Method[getsubkeynames].ReturnValue", "windows-registry"]