Skip to content

Commit 70ec1d6

Browse files
devel@edk2.groups.ioshuishouqiaozhi
authored andcommitted
BaseTools: Optimize GenerateByteArrayValue and CollectPlatformGuids APIs
(cherry picked from commit 8f316e9) During the Incremental build GenerateByteArrayValue used to generate the ByteArrayValue even when there is no change in the PCD/VPDs. which is time consuming API based on the number of PCD/VPDs and SKU IDs. The optimization is that GenerateByteArrayValue is used to store the StructuredPcdsData in a JSON file for each of the arch. and during the Incremental build this API will check, if there is any change in the Structured PCD/VPDs then rest of the flow remains the same. if there is no change then it will return the provious build data. Flow: during the 1st build StructuredPcdsData.json is not exists, StructuredPcdsData will be dumped to json file. and it will copy the output.txt as well. Note: as the output.txt are different for different Arch, so it will be stored in the Arch folder. During the Incremental build check if there is any change in Structured PCD/VPD. if there is a change in Structured VPD/PCD then recreate the StructuredPcdsData.json, and rest of the flow remains same. if there is no change in VPD/PCD read the output.txt and return the data Unit Test: Test1: Modified the Structured Pcds default from DEC file. current flow is executing. Test2: Override the default value of the PCD from DEC file. current flow is executing. Test3: Modified/Override the PCD from DSC file. current flow executing Test4: Modified/Override the FDF from DSC file. current flow executing Test5: update the default value from Command Line.current flow executing Test6: Build without change in PCD in DSC, FDF, DEC and Command Line the proposed changes will be executing, and the return data remains the same with and without the changes. Test7: Build with and without modified the include headers of Structured PCDs. if there is any change in those Structured PCD header then current flow will be executed. With these changes it's helping to save around ~2.5min to ~3.5min of Incremental build time in my build environment. Sample PR: tianocore/edk2-basetools#113 Cc: Yuwei Chen <[email protected]> Cc: Rebecca Cran <[email protected]> Cc: Liming Gao <[email protected]> Cc: Bob Feng <[email protected]> Cc: Amy Chan <[email protected]> Cc: Sai Chaganty <[email protected]> Cc: Digant H Solanki <[email protected]> Signed-off-by: Ashraf Ali S <[email protected]> Reviewed-by: Yuwei Chen <[email protected]> (cherry picked from commit 8f316e9) Change-Id: I2a76d8e48e326cd2ee0a05903461d2b6a03f908f
1 parent 50aebfa commit 70ec1d6

File tree

2 files changed

+139
-58
lines changed

2 files changed

+139
-58
lines changed

BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py

+6-10
Original file line numberDiff line numberDiff line change
@@ -160,22 +160,18 @@ def ValidateBuildTarget(self):
160160

161161
def CollectPlatformGuids(self):
162162
oriInfList = []
163-
oriPkgSet = set()
164-
PlatformPkg = set()
163+
pkgSet = set()
165164
for Arch in self.ArchList:
166165
Platform = self.BuildDatabase[self.MetaFile, Arch, self.BuildTarget, self.ToolChain]
167166
oriInfList = Platform.Modules
168167
for ModuleFile in oriInfList:
169168
ModuleData = self.BuildDatabase[ModuleFile, Platform._Arch, Platform._Target, Platform._Toolchain]
170-
oriPkgSet.update(ModuleData.Packages)
171-
for Pkg in oriPkgSet:
172-
Guids = Pkg.Guids
173-
GlobalData.gGuidDict.update(Guids)
169+
pkgSet.update(ModuleData.Packages)
174170
if Platform.Packages:
175-
PlatformPkg.update(Platform.Packages)
176-
for Pkg in PlatformPkg:
177-
Guids = Pkg.Guids
178-
GlobalData.gGuidDict.update(Guids)
171+
pkgSet.update(Platform.Packages)
172+
for Pkg in pkgSet:
173+
Guids = Pkg.Guids
174+
GlobalData.gGuidDict.update(Guids)
179175

180176
@cached_property
181177
def FdfProfile(self):

BaseTools/Source/Python/Workspace/DscBuildData.py

+133-48
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
from Common.Misc import SaveFileOnChange
3838
from Workspace.BuildClassObject import PlatformBuildClassObject, StructurePcd, PcdClassObject, ModuleBuildClassObject
3939
from collections import OrderedDict, defaultdict
40+
import json
41+
import shutil
4042

4143
def _IsFieldValueAnArray (Value):
4244
Value = Value.strip()
@@ -56,6 +58,7 @@ def _IsFieldValueAnArray (Value):
5658

5759
PcdValueInitName = 'PcdValueInit'
5860
PcdValueCommonName = 'PcdValueCommon'
61+
StructuredPcdsDataName = 'StructuredPcdsData.json'
5962

6063
PcdMainCHeader = '''
6164
/**
@@ -2750,13 +2753,130 @@ def ParseCCFlags(self, ccflag):
27502753
ccflags.add(item)
27512754
i +=1
27522755
return ccflags
2756+
2757+
def GetStructurePcdSet (self, OutputValueFile):
2758+
if not os.path.isfile(OutputValueFile):
2759+
EdkLogger.error("GetStructurePcdSet", FILE_NOT_FOUND, "Output.txt doesn't exist", ExtraData=OutputValueFile)
2760+
return []
2761+
File = open (OutputValueFile, 'r')
2762+
FileBuffer = File.readlines()
2763+
File.close()
2764+
2765+
#start update structure pcd final value
2766+
StructurePcdSet = []
2767+
for Pcd in FileBuffer:
2768+
PcdValue = Pcd.split ('|')
2769+
PcdInfo = PcdValue[0].split ('.')
2770+
StructurePcdSet.append((PcdInfo[0], PcdInfo[1], PcdInfo[2], PcdInfo[3], PcdValue[2].strip()))
2771+
return StructurePcdSet
2772+
2773+
def GetBuildOptionsValueList(self):
2774+
CC_FLAGS = LinuxCFLAGS
2775+
if sys.platform == "win32":
2776+
CC_FLAGS = WindowsCFLAGS
2777+
BuildOptions = OrderedDict()
2778+
for Options in self.BuildOptions:
2779+
if Options[2] != EDKII_NAME:
2780+
continue
2781+
Family = Options[0]
2782+
if Family and Family != self.ToolChainFamily:
2783+
continue
2784+
Target, Tag, Arch, Tool, Attr = Options[1].split("_")
2785+
if Tool != 'CC':
2786+
continue
2787+
if Attr != "FLAGS":
2788+
continue
2789+
if Target == TAB_STAR or Target == self._Target:
2790+
if Tag == TAB_STAR or Tag == self._Toolchain:
2791+
if 'COMMON' not in BuildOptions:
2792+
BuildOptions['COMMON'] = set()
2793+
if Arch == TAB_STAR:
2794+
BuildOptions['COMMON']|= self.ParseCCFlags(self.BuildOptions[Options])
2795+
if Arch in self.SupArchList:
2796+
if Arch not in BuildOptions:
2797+
BuildOptions[Arch] = set()
2798+
BuildOptions[Arch] |= self.ParseCCFlags(self.BuildOptions[Options])
2799+
2800+
if BuildOptions:
2801+
ArchBuildOptions = {arch:flags for arch,flags in BuildOptions.items() if arch != 'COMMON'}
2802+
if len(ArchBuildOptions.keys()) == 1:
2803+
BuildOptions['COMMON'] |= (list(ArchBuildOptions.values())[0])
2804+
elif len(ArchBuildOptions.keys()) > 1:
2805+
CommonBuildOptions = reduce(lambda x,y: x&y, ArchBuildOptions.values())
2806+
BuildOptions['COMMON'] |= CommonBuildOptions
2807+
ValueList = [item for item in BuildOptions['COMMON'] if item.startswith((r"/U","-U"))]
2808+
ValueList.extend([item for item in BuildOptions['COMMON'] if item.startswith((r"/D", "-D"))])
2809+
CC_FLAGS += " ".join(ValueList)
2810+
return CC_FLAGS
2811+
2812+
27532813
def GenerateByteArrayValue (self, StructuredPcds):
27542814
#
27552815
# Generate/Compile/Run C application to determine if there are any flexible array members
27562816
#
27572817
if not StructuredPcds:
27582818
return
27592819

2820+
StructuredPcdsData = {}
2821+
StoredStructuredPcdObjectPaths = {}
2822+
SkipPcdValueInit = False
2823+
2824+
CC_FLAGS = self.GetBuildOptionsValueList()
2825+
2826+
for PcdName in StructuredPcds:
2827+
Pcd = StructuredPcds[PcdName]
2828+
TokenSpaceGuidCName = Pcd.TokenSpaceGuidCName
2829+
TokenCName = Pcd.TokenCName
2830+
2831+
# Create a key using TokenSpaceGuidCName and TokenCName
2832+
StructuredPcdsData[f"{TokenSpaceGuidCName}_{TokenCName}"] = {
2833+
"DefaultValueFromDec": Pcd.DefaultValueFromDec,
2834+
"DefaultValues": Pcd.DefaultValues,
2835+
"PcdFieldValueFromComm": Pcd.PcdFieldValueFromComm,
2836+
"PcdFieldValueFromFdf": Pcd.PcdFieldValueFromFdf,
2837+
"DefaultFromDSC": Pcd.DefaultFromDSC,
2838+
"PcdFiledValueFromDscComponent": Pcd.PcdFiledValueFromDscComponent
2839+
}
2840+
2841+
# Store the CC Flags
2842+
StructuredPcdsData["CC_FLAGS"] = CC_FLAGS
2843+
#
2844+
# If the output path doesn't exists then create it
2845+
#
2846+
if not os.path.exists(self.OutputPath):
2847+
os.makedirs(self.OutputPath)
2848+
2849+
StructuredPcdsDataPath = os.path.join(self.OutputPath, self._Arch, StructuredPcdsDataName)
2850+
PcdRecordOutputValueFile = os.path.join(self.OutputPath, self._Arch, 'Output.txt')
2851+
2852+
if not os.path.exists(os.path.dirname(StructuredPcdsDataPath)):
2853+
os.makedirs(os.path.dirname(StructuredPcdsDataPath))
2854+
#
2855+
# Check if the StructuredPcdsData.json exists or not
2856+
# if exits then it might be a incremental build then check if the StructuredPcdsData has been changed or not.
2857+
# if changed then proceed further, if not changed then return the stored data from earlier build
2858+
#
2859+
if os.path.isfile(StructuredPcdsDataPath):
2860+
with open(StructuredPcdsDataPath, 'r') as file:
2861+
StoredStructuredPcdsData = json.load(file)
2862+
# OBJECTS will have the modified time, which needs to be checked later
2863+
StoredStructuredPcdObjectPaths = StoredStructuredPcdsData.pop("OBJECTS", {})
2864+
2865+
if StructuredPcdsData == StoredStructuredPcdsData:
2866+
SkipPcdValueInit = True
2867+
for filename, file_mtime in StoredStructuredPcdObjectPaths.items():
2868+
f_mtime = os.path.getmtime(filename)
2869+
#
2870+
# check if the include_file are modified or not,
2871+
# if modified then generate the PcdValueInit
2872+
#
2873+
if f_mtime != file_mtime:
2874+
SkipPcdValueInit = False
2875+
break
2876+
2877+
if SkipPcdValueInit:
2878+
return self.GetStructurePcdSet(PcdRecordOutputValueFile)
2879+
27602880
InitByteValue = ""
27612881
CApp = PcdMainCHeader
27622882

@@ -2832,8 +2952,6 @@ def GenerateByteArrayValue (self, StructuredPcds):
28322952

28332953
CApp = CApp + PcdMainCEntry + '\n'
28342954

2835-
if not os.path.exists(self.OutputPath):
2836-
os.makedirs(self.OutputPath)
28372955
CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName)
28382956
SaveFileOnChange(CAppBaseFileName + '.c', CApp, False)
28392957

@@ -2890,42 +3008,6 @@ def GenerateByteArrayValue (self, StructuredPcds):
28903008
IncSearchList.append(inc)
28913009
MakeApp = MakeApp + '\n'
28923010

2893-
CC_FLAGS = LinuxCFLAGS
2894-
if sys.platform == "win32":
2895-
CC_FLAGS = WindowsCFLAGS
2896-
BuildOptions = OrderedDict()
2897-
for Options in self.BuildOptions:
2898-
if Options[2] != EDKII_NAME:
2899-
continue
2900-
Family = Options[0]
2901-
if Family and Family != self.ToolChainFamily:
2902-
continue
2903-
Target, Tag, Arch, Tool, Attr = Options[1].split("_")
2904-
if Tool != 'CC':
2905-
continue
2906-
if Attr != "FLAGS":
2907-
continue
2908-
if Target == TAB_STAR or Target == self._Target:
2909-
if Tag == TAB_STAR or Tag == self._Toolchain:
2910-
if 'COMMON' not in BuildOptions:
2911-
BuildOptions['COMMON'] = set()
2912-
if Arch == TAB_STAR:
2913-
BuildOptions['COMMON']|= self.ParseCCFlags(self.BuildOptions[Options])
2914-
if Arch in self.SupArchList:
2915-
if Arch not in BuildOptions:
2916-
BuildOptions[Arch] = set()
2917-
BuildOptions[Arch] |= self.ParseCCFlags(self.BuildOptions[Options])
2918-
2919-
if BuildOptions:
2920-
ArchBuildOptions = {arch:flags for arch,flags in BuildOptions.items() if arch != 'COMMON'}
2921-
if len(ArchBuildOptions.keys()) == 1:
2922-
BuildOptions['COMMON'] |= (list(ArchBuildOptions.values())[0])
2923-
elif len(ArchBuildOptions.keys()) > 1:
2924-
CommonBuildOptions = reduce(lambda x,y: x&y, ArchBuildOptions.values())
2925-
BuildOptions['COMMON'] |= CommonBuildOptions
2926-
ValueList = [item for item in BuildOptions['COMMON'] if item.startswith((r"/U","-U"))]
2927-
ValueList.extend([item for item in BuildOptions['COMMON'] if item.startswith((r"/D", "-D"))])
2928-
CC_FLAGS += " ".join(ValueList)
29293011
MakeApp += CC_FLAGS
29303012

29313013
if sys.platform == "win32":
@@ -2946,7 +3028,9 @@ def GenerateByteArrayValue (self, StructuredPcds):
29463028
SearchPathList.append(os.path.normpath(mws.join(GlobalData.gGlobalDefines["EDK_TOOLS_PATH"], "BaseTools/Source/C/Common")))
29473029
SearchPathList.extend(str(item) for item in IncSearchList)
29483030
IncFileList = GetDependencyList(IncludeFileFullPaths, SearchPathList)
3031+
StructuredPcdsData["OBJECTS"] = {}
29493032
for include_file in IncFileList:
3033+
StructuredPcdsData["OBJECTS"][include_file] = os.path.getmtime(include_file)
29503034
MakeApp += "$(OBJECTS) : %s\n" % include_file
29513035
if sys.platform == "win32":
29523036
PcdValueCommonPath = os.path.normpath(mws.join(GlobalData.gGlobalDefines["EDK_TOOLS_PATH"], "Source\C\Common\PcdValueCommon.c"))
@@ -3042,17 +3126,18 @@ def GenerateByteArrayValue (self, StructuredPcds):
30423126
if returncode != 0:
30433127
EdkLogger.warn('Build', COMMAND_FAILURE, 'Can not collect output from command: %s\n%s\n%s\n' % (Command, StdOut, StdErr))
30443128

3045-
#start update structure pcd final value
3046-
File = open (OutputValueFile, 'r')
3047-
FileBuffer = File.readlines()
3048-
File.close()
3129+
#
3130+
# In 1st build create the StructuredPcdsData.json
3131+
# update the record as PCD Input has been changed if its incremental build
3132+
#
3133+
with open(StructuredPcdsDataPath, 'w') as file:
3134+
json.dump(StructuredPcdsData, file, indent=2)
30493135

3050-
StructurePcdSet = []
3051-
for Pcd in FileBuffer:
3052-
PcdValue = Pcd.split ('|')
3053-
PcdInfo = PcdValue[0].split ('.')
3054-
StructurePcdSet.append((PcdInfo[0], PcdInfo[1], PcdInfo[2], PcdInfo[3], PcdValue[2].strip()))
3055-
return StructurePcdSet
3136+
# Copy update output file for each Arch
3137+
shutil.copyfile(OutputValueFile, PcdRecordOutputValueFile)
3138+
3139+
#start update structure pcd final value
3140+
return self.GetStructurePcdSet(OutputValueFile)
30563141

30573142
@staticmethod
30583143
def NeedUpdateOutput(OutputFile, ValueCFile, StructureInput):

0 commit comments

Comments
 (0)