Improve build time in 1st build and incremental build

edk2basetools/AutoGen/PlatformAutoGen.py

@@ -707,6 +707,8 @@ def SortDynamicPcd(self):
         self._DynamicPcdList.extend(list(OtherPcdArray))
         self._DynamicPcdList.sort()
         allskuset = [(SkuName, Sku.SkuId) for pcd in self._DynamicPcdList for (SkuName, Sku) in pcd.SkuInfoList.items()]
+        # Remove duplicate sets in the list
+        allskuset = list(set(allskuset))
         for pcd in self._DynamicPcdList:
             if len(pcd.SkuInfoList) == 1:
                 for (SkuName, SkuId) in allskuset:

edk2basetools/AutoGen/WorkspaceAutoGen.py

@@ -160,22 +160,18 @@ def ValidateBuildTarget(self):
 
     def CollectPlatformGuids(self):
         oriInfList = []
-        oriPkgSet = set()
-        PlatformPkg = set()
+        pkgSet = set()
         for Arch in self.ArchList:
             Platform = self.BuildDatabase[self.MetaFile, Arch, self.BuildTarget, self.ToolChain]
             oriInfList = Platform.Modules
             for ModuleFile in oriInfList:
                 ModuleData = self.BuildDatabase[ModuleFile, Platform._Arch, Platform._Target, Platform._Toolchain]
-                oriPkgSet.update(ModuleData.Packages)
-                for Pkg in oriPkgSet:
-                    Guids = Pkg.Guids
-                    GlobalData.gGuidDict.update(Guids)
+                pkgSet.update(ModuleData.Packages)
             if Platform.Packages:
-                PlatformPkg.update(Platform.Packages)
-                for Pkg in PlatformPkg:
-                    Guids = Pkg.Guids
-                    GlobalData.gGuidDict.update(Guids)
+                pkgSet.update(Platform.Packages)
+        for Pkg in pkgSet:
+            Guids = Pkg.Guids
+            GlobalData.gGuidDict.update(Guids)
 
     @cached_property
     def FdfProfile(self):

edk2basetools/Workspace/DscBuildData.py

@@ -37,6 +37,8 @@
 from edk2basetools.Common.Misc import SaveFileOnChange
 from edk2basetools.Workspace.BuildClassObject import PlatformBuildClassObject, StructurePcd, PcdClassObject, ModuleBuildClassObject
 from collections import OrderedDict, defaultdict
+import json
+import shutil
 
 def _IsFieldValueAnArray (Value):
     Value = Value.strip()
@@ -56,6 +58,7 @@ def _IsFieldValueAnArray (Value):
 
 PcdValueInitName = 'PcdValueInit'
 PcdValueCommonName = 'PcdValueCommon'
+PcdRecordListName = 'PcdRecordList.json'
 
 PcdMainCHeader = '''
 /**
@@ -1599,7 +1602,7 @@ def UpdateStructuredPcds(self, TypeList, AllPcds):
         S_pcd_set = DscBuildData.OverrideByComm(S_pcd_set)
 
         # Create a tool to caculate structure pcd value
-        Str_Pcd_Values = self.GenerateByteArrayValue(S_pcd_set)
+        Str_Pcd_Values = self.GenerateByteArrayValue(S_pcd_set, RecordList)
 
         if Str_Pcd_Values:
             for (skuname, StoreName, PcdGuid, PcdName, PcdValue) in Str_Pcd_Values:
@@ -2750,12 +2753,66 @@ def ParseCCFlags(self, ccflag):
                 ccflags.add(item)
             i +=1
         return ccflags
-    def GenerateByteArrayValue (self, StructuredPcds):
+
+    def GetStructurePcdSet (self, OutputValueFile):
+        if not os.path.isfile(OutputValueFile):
+            EdkLogger.error("GetStructurePcdSet", FILE_NOT_FOUND, ExtraData=OutputValueFile)
+            return []
+        File = open (OutputValueFile, 'r')
+        FileBuffer = File.readlines()
+        File.close()
+
+        #start update structure pcd final value
+        StructurePcdSet = []
+        for Pcd in FileBuffer:
+            PcdValue = Pcd.split ('|')
+            PcdInfo = PcdValue[0].split ('.')
+            StructurePcdSet.append((PcdInfo[0], PcdInfo[1], PcdInfo[2], PcdInfo[3], PcdValue[2].strip()))
+        return StructurePcdSet
+
+    def GenerateByteArrayValue (self, StructuredPcds, PcdRecordList):
         #
         # Generate/Compile/Run C application to determine if there are any flexible array members
         #
         if not StructuredPcds:
             return
+
+        #
+        # If the output path doesn't exists then create it
+        #
+        if not os.path.exists(self.OutputPath):
+            os.makedirs(self.OutputPath)
+
+        PcdRecordListPath = os.path.join(self.OutputPath, self._Arch, PcdRecordListName)
+        PcdRecordOutputValueFile = os.path.join(self.OutputPath, self._Arch, 'Output.txt')
+
+        if not os.path.exists(os.path.dirname(PcdRecordListPath)):
+            os.makedirs(os.path.dirname(PcdRecordListPath))
+        #
+        # Check if the PcdRecordList.json exists or not
+        # if exits then it might be a incremental build then check if the PcdRecord list has been changed or not.
+        # if changed then proceed further, if not changed then return the stored data from earlier build
+        #
+        if os.path.isfile(PcdRecordListPath):
+            with open(PcdRecordListPath, 'r') as file:
+                file_content_str = file.read()
+                if file_content_str:
+                    # Use json.loads to convert the string back to a list
+                    file_content = json.loads(file_content_str)
+                    # Check if all PcdRecordList in record_set are present in file_content
+                    # and if there are no extra PcdRecordList in file_content
+                    if set(map(tuple, PcdRecordList)) == set(map(tuple, file_content)):
+                        return self.GetStructurePcdSet(PcdRecordOutputValueFile)
+                    else:
+                        # update the record as PCD Input has been changed in incremental build
+                        with open(PcdRecordListPath, 'w') as file:
+                            json.dump(PcdRecordList, file)
+        else:
+            #
+            # 1st build, create the PcdRecordList.json
+            #
+            with open(PcdRecordListPath, 'w') as file:
+                json.dump(PcdRecordList, file)
 
         InitByteValue = ""
         CApp = PcdMainCHeader
@@ -2832,8 +2889,6 @@ def GenerateByteArrayValue (self, StructuredPcds):
 
         CApp = CApp + PcdMainCEntry + '\n'
 
-        if not os.path.exists(self.OutputPath):
-            os.makedirs(self.OutputPath)
         CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName)
         SaveFileOnChange(CAppBaseFileName + '.c', CApp, False)
 
@@ -3042,17 +3097,10 @@ def GenerateByteArrayValue (self, StructuredPcds):
             if returncode != 0:
                 EdkLogger.warn('Build', COMMAND_FAILURE, 'Can not collect output from command: %s\n%s\n%s\n' % (Command, StdOut, StdErr))
 
+        # Copy update output file for each Arch
+        shutil.copyfile(OutputValueFile, PcdRecordOutputValueFile)
         #start update structure pcd final value
-        File = open (OutputValueFile, 'r')
-        FileBuffer = File.readlines()
-        File.close()
-
-        StructurePcdSet = []
-        for Pcd in FileBuffer:
-            PcdValue = Pcd.split ('|')
-            PcdInfo = PcdValue[0].split ('.')
-            StructurePcdSet.append((PcdInfo[0], PcdInfo[1], PcdInfo[2], PcdInfo[3], PcdValue[2].strip()))
-        return StructurePcdSet
+        return self.GetStructurePcdSet(OutputValueFile)
 
     @staticmethod
     def NeedUpdateOutput(OutputFile, ValueCFile, StructureInput):