From 96ed5db3c2876d253f93565cb9f856927ed3e571 Mon Sep 17 00:00:00 2001 From: Jason Albert Date: Tue, 18 Oct 2016 10:30:00 -0500 Subject: Initial support for ktvpfile support - The logic is working, but the code is messy now and needs to be refactored - Fixed inconsistent spacing in some cases (3 vs 4) - Checkpoint commit to save working logic before refactor --- createVpd.py | 391 ++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 227 insertions(+), 164 deletions(-) diff --git a/createVpd.py b/createVpd.py index ec81180..fb3b6be 100755 --- a/createVpd.py +++ b/createVpd.py @@ -48,15 +48,15 @@ import os # By default, element tree doesn't preserve comments # http://stackoverflow.com/questions/4474754/how-to-keep-comments-while-parsing-xml-using-python-elementtree class PCParser(ET.XMLTreeBuilder): - def __init__(self): - ET.XMLTreeBuilder.__init__(self) - # assumes ElementTree 1.2.X - self._parser.CommentHandler = self.handle_comment - - def handle_comment(self, data): - self._target.start(ET.Comment, {}) - self._target.data(data) - self._target.end(ET.Comment) + def __init__(self): + ET.XMLTreeBuilder.__init__(self) + # assumes ElementTree 1.2.X + self._parser.CommentHandler = self.handle_comment + + def handle_comment(self, data): + self._target.start(ET.Comment, {}) + self._target.data(data) + self._target.end(ET.Comment) class RecordInfo: """Stores the info about each vpd record""" @@ -82,17 +82,17 @@ class RecordInfo: # Find file in a given path or paths # searchPath comes from the --inpath option def findFile(filename, searchPath): - found = False - paths = searchPath.split(os.path.pathsep) - for path in paths: - #print("Trying %s" % (os.path.join(path,filename))) - if os.path.exists(os.path.join(path, filename)): - found = 1 - break - if found: - return os.path.abspath(os.path.join(path, filename)) - else: - return None + found = False + paths = searchPath.split(os.path.pathsep) + for path in paths: + #print("Trying %s" % (os.path.join(path,filename))) + if os.path.exists(os.path.join(path, filename)): + found = 1 + break + if found: + return os.path.abspath(os.path.join(path, filename)) + else: + return None # Function to write out the resultant tvpd xml file def writeTvpd(manifest, outputFile): @@ -100,37 +100,166 @@ def writeTvpd(manifest, outputFile): tree.write(outputFile, encoding="utf-8", xml_declaration=True) return None -# Parses a tvpd file using ET. That will generate errors for bad xml formatting -# The valid XML is then checked to make sure the required tags are found at each level -def parseTvpd(tvpdFile, topLevel): +def parseRtvpd(): + return 0 +def parseKtvpd(): + return 0 + +def processKeyword(keyword, recordName): + errorsFound = 0 + + # Define the expected tags at this level + keywordTags = {"kwdesc" : 0, "kwformat" : 0, "kwlen" : 0, "kwdata" : 0, "ktvpdfile" : 0} + + # Make sure the keyword has a name attrib, save for later use + keywordName = keyword.attrib.get("name") + if (keywordName == None): + out.error(" tag in record %s is missing the name attribute" % (recordName)) + errorsFound += 1 + keywordName = "INVALID" # Set the invalid name so the code below can use it without issue + + # Loop thru the tags defined for this keyword + for keywordEntries in keyword: + # Comments aren't basestring tags + if not isinstance(keywordEntries.tag, basestring): + continue + + # See if this is a tag we even expect + if keywordEntries.tag not in keywordTags: + out.error("Unsupported tag <%s> found while parsing the level for keyword %s in record %s" % (keywordEntries.tag, keywordName, recordName)) + errorsFound += 1 + # We continue here because we don't want to parse down this hierarcy path when we don't know what it is + continue + # It was a supported tag + else: + keywordTags[keywordEntries.tag] += 1 + + # We've checked for unknown keyword tags, now make sure we have the right number of each + # This is a simple one, we can only have 1 of each + keywordTagCount = 1 + if (keywordTags["ktvpdfile"] != 0): + if (keywordTags["ktvpdfile"] > 1): + out.error("The tag is only allowed to be used once for keyword %s in record %s" % (keywordName, recordName)) + errorsFound += 1 + # Only had one ktvpfile, so set our expected to the rest to 0 + keywordTagCount = 0 + + # Depending upon the state of ktvpdfile, check to ensure we have only 1 of each tag or 0 of each tag + for tag in ["kwdesc", "kwformat", "kwlen", "kwdata"]: + if (keywordTags[tag] != keywordTagCount): + out.error("The tag <%s> was expected to have a count of %d, but was found with a count of %d for keyword %s in record %s" % (tag, keywordTagCount, keywordTags[tag], keywordName, recordName)) + errorsFound += 1 + + # All done + return errorsFound + +def processRecord(record): + errorsFound = 0 + + # Define the expected tags at this level + recordTags = {"rdesc" : 0, "keyword" : 0, "rtvpdfile" : 0, "rbinfile" : 0} + + # Make sure the record has a name attrib, save for later use + recordName = record.attrib.get("name") + if (recordName == None): + out.error("A tag is missing the name attribute") + errorsFound += 1 + recordName = "INVALID" # Set the invalid name so the code below can use it without issue + + # Loop thru the tags defined for this record + for recordEntries in record: + + # Comments aren't basestring tags + if not isinstance(recordEntries.tag, basestring): + continue + + # See if this is a tag we even expect + if recordEntries.tag not in recordTags: + out.error("Unsupported tag <%s> found while parsing the level for record %s" % (recordEntries.tag, recordName)) + errorsFound += 1 + # We continue here because we don't want to parse down this hierarcy path when we don't know what it is + continue + + # It was a supported tag + else: + recordTags[recordEntries.tag] += 1 + + # Do the keyword level checks + if (recordEntries.tag == "keyword"): + + # Call the dedicated function to process keyword tags + errorsFound += processKeyword(recordEntries, recordName) + + # We've checked for unknown record tags, now make sure we've got the right number, they don't conflict, etc.. + recordTagTotal = bool(recordTags["keyword"]) + bool(recordTags["rbinfile"]) + bool(recordTags["rtvpdfile"]) + # keyword, rbinfile and rtvpdfile are mutually exclusive. Make sure we have only one + if (recordTagTotal > 1): + out.error("For record %s, more than one tag of type keyword, rbinfile or rtvpdfile was given!" % (recordName)) + out.error("Use of only 1 at a time is supported for a given record!") + errorsFound += 1 + # We checked if we had more than 1, let's make sure we have at least 1 + if (recordTagTotal < 1): + out.error("For record %s, 0 tags of type keyword, rbinfile or rtvpdfile were given!" % (recordName)) + out.error("1 tag of the 3 must be in use for the record to be valid!") + errorsFound += 1 + # Make sure the rdesc is available + if (recordTags["keyword"] and recordTags["rdesc"] != 1): + out.error("The tag was expected to have a count of 1, but was found with a count of %d for record %s" % (recordTags["rdesc"], recordName)) + errorsFound += 1 + + # All done + return errorsFound + +# Parses a vpd xml file using ET. That will generate errors for bad xml formatting +def parseVpdXml(vpdXmlFile): # Accumulate errors and return the total at the end # This allows the user to see all mistakes at once instead of iteratively running errorsFound = 0 + # Get the full path to the file given + fullPathFile = findFile(vpdXmlFile, clInputPath) + if (fullPathFile == None): + out.error("The xml file %s could not be found! Please check your -m or -i cmdline options for typos" % (vpdXmlFile)) + exit(1) + # Let the user know what file we are reading # We could make this optional with a new function param in the future - out.msg("Parsing tvpd %s" % tvpdFile) + out.msg("Parsing file %s" % fullPathFile) # Read in the file # If there are tag mismatch errors or other general gross format problems, it will get caught here # Once we return from this function, then we'll check to make sure only supported tags were given, etc.. # Invoke the extended PCParser, which will handle preserving comments in the output file parser = PCParser() - tvpdRoot = ET.parse(tvpdFile, parser=parser).getroot() + vpdTree = ET.parse(fullPathFile, parser=parser).getroot() # Print the top level tags from the parsing if (clDebug): out.debug("Top level tag/attrib found") - for child in tvpdRoot: + for child in vpdTree: out.debug("%s %s" % (child.tag, child.attrib)) + ###### + # All done, vary our return based upon the errorsFound + if (errorsFound): + return (errorsFound, None) + else: + return(0, vpdTree) + + +# Parse the XML to make sure the required tags are found +def parseVpd(vpdTree): + # Accumulate errors and return the total at the end + # This allows the user to see all mistakes at once instead of iteratively running + errorsFound = 0 + # Do some basic error checking of what we've read in # Make sure the root starts with the vpd tag # If it doesn't, it's not worth syntax checking any further # This is the only time we'll just bail instead of accumulating - if (tvpdRoot.tag != "vpd"): + if (vpdTree.tag != "vpd"): out.error("%s does not start with a tag. No further checking will be done until fixed!" % tvpdFile) - return(1, None) + exit(1) # We at least have a proper top level vpd tag, so loop thru the rest of the levels and check for any unknown tags # This will also be a good place to check for the required tags @@ -139,7 +268,7 @@ def parseTvpd(tvpdFile, topLevel): vpdTags = {"name" : 0, "size" : 0, "VD" : 0, "record" : 0} # Go thru the tags at this level - for vpd in tvpdRoot: + for vpd in vpdTree: # Comments aren't basestring tags if not isinstance(vpd.tag, basestring): continue @@ -152,100 +281,17 @@ def parseTvpd(tvpdFile, topLevel): continue # It was a supported tag else: - vpdTags[vpd.tag]+=1 + vpdTags[vpd.tag] += 1 # Do the record level checks if (vpd.tag == "record"): - # Define the expected tags at this level - recordTags = {"rdesc" : 0, "keyword" : 0, "rtvpdfile" : 0, "rbinfile" : 0} - - # Make sure the record has a name attrib, save for later use - recordName = vpd.attrib.get("name") - if (recordName == None): - out.error("A tag is missing the name attribute") - errorsFound += 1 - recordName = "INVALID" # Set the invalid name so the code below can use it without issue - - # Loop thru the tags defined for this record - for record in vpd: - # Comments aren't basestring tags - if not isinstance(record.tag, basestring): - continue - - # See if this is a tag we even expect - if record.tag not in recordTags: - out.error("Unsupported tag <%s> found while parsing the level for record %s" % (record.tag, recordName)) - errorsFound += 1 - # We continue here because we don't want to parse down this hierarcy path when we don't know what it is - continue - - # It was a supported tag - else: - recordTags[record.tag]+=1 - - # Do the keyword level checks - if (record.tag == "keyword"): - # Define the expected tags at this level - keywordTags = {"kwdesc" : 0, "kwformat" : 0, "kwlen" : 0, "kwdata" : 0} - - # Make sure the keyword has a name attrib, save for later use - keywordName = record.attrib.get("name") - if (keywordName == None): - out.error(" tag in record %s is missing the name attribute" % (recordName)) - errorsFound += 1 - keywordName = "INVALID" # Set the invalid name so the code below can use it without issue - - # Loop thru the tags defined for this keyword - for keyword in record: - # Comments aren't basestring tags - if not isinstance(keyword.tag, basestring): - continue - - # See if this is a tag we even expect - if keyword.tag not in keywordTags: - out.error("Unsupported tag <%s> found while parsing the level for keyword %s in record %s" % (keyword.tag, keywordName, recordName)) - errorsFound += 1 - # We continue here because we don't want to parse down this hierarcy path when we don't know what it is - continue - # It was a supported tag - else: - keywordTags[keyword.tag] += 1 - - # We've checked for unknown keyword tags, now make sure we have the right number of each - # This is a simple one, we can only have 1 of each - for tag in keywordTags: - if (keywordTags[tag] != 1): - out.error("The tag <%s> was expected to have a count of 1, but was found with a count of %d for keyword %s in record %s" % (tag, keywordTags[tag], keywordName, recordName)) - errorsFound += 1 - - # We've checked for unknown record tags, now make sure we've got the right number, they don't conflict, etc.. - recordTagTotal = bool(recordTags["keyword"]) + bool(recordTags["rbinfile"]) + bool(recordTags["rtvpdfile"]) - # keyword, rbinfile and rtvpdfile are mutually exclusive. Make sure we have only one - if (recordTagTotal > 1): - out.error("For record %s, more than one tag of type keyword, rbinfile or rtvpdfile was given!" % (recordName)) - out.error("Use of only 1 at a time is supported for a given record!") - errorsFound += 1 - # We checked if we had more than 1, let's make sure we have at least 1 - if (recordTagTotal < 1): - out.error("For record %s, 0 tags of type keyword, rbinfile or rtvpdfile were given!" % (recordName)) - out.error("1 tag of the 3 must be in use for the record to be valid!") - errorsFound += 1 - # Make sure the rdesc is available - if (recordTags["keyword"] and recordTags["rdesc"] != 1): - out.error("The tag was expected to have a count of 1, but was found with a count of %d for record %s" % (recordTags["rdesc"], recordName)) - errorsFound += 1 - + # Call the dedicated function to process a + errorsFound += processRecord(vpd) + # Do some checking of what we found at the vpd level - # Top level is the manifest passed in on the command line - # When false, it's just a record description and doesn't require the high level descriptors - if (topLevel == True): - comparer = 1 - else: - comparer = 0 - # Don't go thru all of them, "record" has special handling below for tag in ["name", "size", "VD"]: - if (vpdTags[tag] != comparer): - out.error("The tag <%s> was expected to have a count of %d, but was found with a count of %d" % (tag, comparer, vpdTags[tag])) + if (vpdTags[tag] != 1): + out.error("The tag <%s> was expected to have a count of 1, but was found with a count of %d" % (tag, vpdTags[tag])) errorsFound += 1 # Make sure at least one record tag was found @@ -253,20 +299,10 @@ def parseTvpd(tvpdFile, topLevel): out.error("At least one must be defined for the file to be valid!") errorsFound += 1 - # If this is an included tvpd, it can only have 1 record in it - # This check is just by convention. If a compelling case to change it was provided, it could be done - if (topLevel == False): - if (vpdTags["record"] > 1): - out.error("More than 1 record entry found in %s. Only 1 record is allowed!" % (tvpdFile)) - errorsFound += 1 - ###### - # All done, vary our return based upon the errorsFound - if (errorsFound): - return (errorsFound, None) - else: - return(0, tvpdRoot) - + # All done + return errorsFound + # Function to write properly packed/encoded data to the vpdFile def writeDataToVPD(vpdFile, data, offset = None): rc = 0 @@ -426,30 +462,30 @@ clBinaryKeywords = args.binary_keywords # We are going to do this in 3 stages # 1 - Read in the manifest and any other referenced files. This will create a complete XML description of the VPD # We will also check to make sure that all required tags are given and no extra tags exist -# 2 - Parse thru the tvpd description and make sure the data with in the tags is valid. These are checks like data not greater than length, etc.. +# 2 - Parse thru the now complete vpd tree and make sure the data within the tags is valid. These are checks like data not greater than length, etc.. # 3 - With the XML and contents verified correct, loop thru it again and write out the VPD data +# # Note: Looping thru the XML twice between stage 1 and 2 makes it easier to surface multiple errors to the user at once. # If we were trying to both validate the xml and data at once, it would be harder to continue and gather multiple errors like we do now ################################################ # Work with the manifest out.setIndent(0) -out.msg("==== Stage 1: Parsing tvpd XML") +out.msg("==== Stage 1: Parsing VPD XML files") out.setIndent(2) errorsFound = 0 -# Get the full path to the file given -manifestfile = findFile(clManifestFile, clInputPath) -if (manifestfile == None): - out.error("The manifest file %s could not be found! Please check your -m or -i cmdline options for typos" % (clManifestFile)) - exit(1) - -# Read in the manifest -(rc, manifest) = parseTvpd(manifestfile, True) +# Read in the top level manifest file and create the vpdTree +(rc, manifest) = parseVpdXml(clManifestFile) if (rc): - out.error("Problem reading in the manifest! - %s" % manifestfile) + out.error("Problem reading in the manifest! - %s" % clManifestFile) exit(rc) +# Now check the top level VPD syntax +rc = parseVpd(manifest) +if (rc): + exit(rc) + # Stash away some variables for use later vpdName = manifest.find("name").text # If the user passed in the special name of FILENAME, we'll use in the input file name, minus the extension, as the output @@ -491,43 +527,70 @@ for record in manifest.iter("record"): # See if a rtvpdfile was given and if so, load it in rtvpdfile = record.find("rtvpdfile") if (rtvpdfile != None): - # Get the full path to the file given - fileName = findFile(rtvpdfile.text, clInputPath) - if (fileName == None): - out.error("The rtvpdfile %s could not be found! Please check your tvpd or input path" % (rtvpdfile.text)) - errorsFound += 1 - break - # Read in the rtvpdfile since it exists - (rc, recordTvpd) = parseTvpd(fileName, False) + # Read in the rtvpdfile + (rc, recordTvpd) = parseVpdXml(rtvpdfile.text) if (rc): - out.error("Error occurred reading in %s" % fileName) + out.error("Error occurred reading in %s" % rtvpdfile.text) errorsFound += 1 break - # Merge the new record into the main manifest - # ET doesn't have a replace function. You can do an extend/remove, but that changes the order of the file - # The goal is to preserve record & keyword order, so that method doesn't work - # The below code will insert the rtvpdfile record in the list above the current matching record definition - # Then remove the original record definition, preserving order - # Since the referenced file also starts with tag, you need to get one level down and find the start of the record element, hence the find # We know this will contain a record because the parse validates that. No need to check for find errors. - subRecord = recordTvpd.find("record") + replRecord = recordTvpd.find("record") # -------- # Make sure the record found in rtvpdfile is the same as the record in the manifiest # We have to do this error check here because the recordName doesn't exist in parseTvpd - subRecordName = subRecord.attrib.get("name") - if (subRecordName != recordName): - out.error("The record (%s) found in %s doesn't match the record name in the manifest (%s)" % (subRecordName, rtvpd.text, recordName)) + if (replRecord.attrib.get("name") != recordName): + out.error("The record (%s) found in %s doesn't match the record name in the manifest (%s)" % (replRecord.attrib.get("name"), rtvpdfile.text, recordName)) errorsFound += 1 break - # Everything looks good, insert/remove - manifest.insert(list(manifest).index(record), subRecord) - manifest.remove(record) + else: + # It's not a rtvpdfile record, so hold onto it for replacement below + replRecord = record + + # Look for ktvpdfile lines + for keyword in replRecord.iter("keyword"): + keywordName = keyword.attrib.get("name") + + # See if a ktvpdfile was given and if so, load it in + ktvpdfile = keyword.find("ktvpdfile") + if (ktvpdfile != None): + # Read in the ktvpdfile + (rc, replKeyword) = parseVpdXml(ktvpdfile.text) + if (rc): + out.error("Error occurred reading in %s" % ktvpdfile.text) + errorsFound += 1 + break + # -------- + # Make sure the keyword found in ktvpdfile is the same as the keyword in the manifiest + # We have to do this error check here because the keywordName doesn't exist in parseTvpd + if (replKeyword.attrib.get("name") != keywordName): + out.error("The keyword (%s) found in %s doesn't match the keyword name in the manifest (%s)" % (replKeyword.attrib.get("name"), ktvpdfile.text, keywordName)) + errorsFound += 1 + break + + # Merge the new record into the main manifest + # ET doesn't have a replace function. You can do an extend/remove, but that changes the order of the file + # The goal is to preserve record & keyword order, so that method doesn't work + # The below code will insert the rtvpdfile record in the list above the current matching record definition + # Then remove the original record definition, preserving order + replRecord.insert(list(record).index(keyword), replKeyword) + replRecord.remove(keyword) + + + # Merge the new record into the main manifest + # ET doesn't have a replace function. You can do an extend/remove, but that changes the order of the file + # The goal is to preserve record & keyword order, so that method doesn't work + # The below code will insert the rtvpdfile record in the list above the current matching record definition + # Then remove the original record definition, preserving order + manifest.insert(list(manifest).index(record), replRecord) + manifest.remove(record) + + # All done with error checks, bailout if we hit something if (errorsFound): out.msg("") -- cgit v1.2.1