Example Python Script to Convert Scan Files to FlexConnect Framework

Overview

This article will show you how to take data from a custom (or currently unsupported) tool and format the data into a format that you can upload into Nucleus. This article contains an example python script you can use to write data into a CSV file for upload.

Example Using WhiteSource XML File

We will be using a WhiteSource XML file, but you can do this with any files you wish to parse.

#!/usr/bin/python3.7
__author__ = "Nucleus  Security"
__license__ = "Apache Free License"
__version__ = "0.0"

import xml.etree.ElementTree as ET
import csv
import argparse
import sys

def customParser(inputPath, outputPath):

    # Create the csv file for writing
    with open(outputPath, 'w', newline='') as csvfile:

        csvwriter = csv.writer(csvfile, delimiter=',')

        csvwriter.writerow(['nucleus_import_version', 'host_name', 'scan_type', 'scan_tool', 'finding_type', 'finding_cve', 'finding_number', 'finding_name', 'finding_severity', 'finding_description', 'finding_solution', 'finding_output', 'finding_path', 'finding_result'])

        # Used to verify paths and differentiate between two of the same tag
        path = []
 
        # Try to parse the data. 
        try:

            # Loop to stream data from the xml file into a csv
            # Looks through the xml tags and takes action based on tag name
            for event, elem in ET.iterparse(inputPath, events=("start", "end")):

                # Used to check the paths later
                if event == 'start':

                    # Used to verify where in the hierarchy you are when finding certain tags in a stream
                    # Need to have a store because we are streaming the data instead of finding all
                    # Necessary in case of really big xml files for mem use
                    path.append(elem.tag)

                # Where we start building parsing for each item we want
                # Based on the end tags in the xml doc
                elif event == 'end':

                    csv_line = []
 
                   # iterate through the tags in the xml until you get to a severity tag
                   if elem.tag == 'severity':

                       severity = elem.text.strip()


                   # Check for the affected packge from xml file
                   if elem.tag == 'library':

                       library = elem.text.strip()


                   # Check the vuln description and the top fix
                   if elem.tag == 'description':

                       # Check the path to see where we are in the file
                       if 'topFix' not in path:

                           # Strip is for cleaning the newlines off the output before writing into the csv file
                           vulnDescription = elem.text.strip()

                       else:

                           # Strip is for cleaning the newlines off the output before writing into the csv file
                           solutionDescription = elem.text.strip()

                   # Get both the assets affected and the name of the vuln
                   # Also used to get the affected CVEs
                   if elem.tag == 'name':

                      # Get vuln first based on path and assign to variable so can add to assets
                      if 'name' and 'occurrences' not in path:

                          vulnName = elem.text.strip()
 
                      # Get the assets affected and add vuln name to asset
                      elif 'project' not in path:

                          host_name = elem.text.strip()

 
                      # All that is left is the finding path
                      else:
 
                          finding_path = elem.text.strip()

                         # Write the csv line for every finding path
                         csv_line = ['1', host_name, "Application", "WhiteSource", "Vuln", vulnName, vulnName+host_name, vulnName+": "+library, severity, vulnDescription, solutionDescription, library, finding_path, 'FAILED']

                     # Check to see if the csv line is blank and don't write to file unless 
                     # data is present in the line
                     if csv_line != []:

                         csvwriter.writerow(csv_line)

                         # For testing the output of each line being written into the csv file
                         # print("Wrote line into csv file! ", csv_line)

                     else:

                         pass
 
                     # Reset the path for the next piece of the streamed xml file
                     path.pop()

        # Catch the errors in parsing the output
        except Exception as e:

            print("Error, probably bad xml document. Check that you are trying to parse the correct doc type")

            print("Error was the following:", e)

# Make this script able to handle inputfile and outputfile selections
def get_args():
    parser = argparse.ArgumentParser(description="For parsing whitesource files to be uploaded into Nucleus")

    # List arguments. Should only include input file and output file
    parser.add_argument('-i', dest='inputFile', help="Path to whitesource xml file to parse", required=True)
    parser.add_argument('-o', dest='outputFile', help="Path to csv file output", required=True)

    # Define the arguments globally for ease of use
    global args

    args = parser.parse_args()

    return args


if __name__ == "__main__":

    # Get the arguments
    arguments = get_args()

    # Get the input file to parse
    inputPath = arguments.inputFile

    # Get the output file to save to
    outputPath = arguments.outputFile

    # Start the parsing and csv writing
    customParser(inputPath, outputPath)

Parse WhiteSource Files into Nucleus Schema

1. Copy the above python script into a file called whitesource_parse.py.

2. Get a WhiteSource XML file from your WhiteSource console. An example XML file is attached below.

3. Make sure you have python 3 installed on your computer.

4. Run the following command in a terminal/command prompt

python whitesource_parse.py -i whitesource_input_file.xml -o whitesource_nucleus_output.csv

The script should give you a CSV file that you can then upload to Nucleus for managing your WhiteSource vulnerabilities in Nucleus alongside your other scan types. This process can be repeated for all the scan types that you wish to manage in Nucleus.

If you have any questions, please contact us through the support center.