Initial Commit

.gitignore

+dist/
+build/
+ncmltransformdemo.egg-info/
+.DS_Store
+.idea
+__pycache__

ncmltransformdemo/metadatatransformer.py

+"""
+This class investigates CF NetCDF files and creates granular metadata
+from them in the form of XML.
+
+- Justin Lynn Reid
+"""
+
+import os
+
+
+class MetadataTransformer:
+
+    def __init__(self, netcdf_path, out_path):
+
+        if not os.path.exists(netcdf_path):
+            raise IOError("Input NetCDF doesn't exist")
+        elif not os.path.exists(out_path):
+            raise IOError("Output Path doesn't exist")
+        else:
+            self.in_path = netcdf_path
+            self.nc_dump_path = ""
+            self.nc_classic_path = ""
+            self.in_file = os.path.basename(netcdf_path)
+            self._out_path = out_path
+            self.out_file = ""
+
+    def dump_nc_file(self):
+        """
+        Dump a CF NetCDF's header contents to a text file.
+        :return:
+        """
+        self.nc_dump_path = os.path.join(
+            self._out_path,
+            self.in_file.replace(".nc", ".txt")
+        )
+
+        cmd = " ".join(["ncdump", "-h", self.in_path, ">", self.nc_dump_path])
+        os.system(cmd)
+
+        if not os.path.exists(self.nc_dump_path):
+            raise IOError("Output path {0} doesn't exist".format(self.nc_dump_path))
+
+    def get_classic_nc(self):
+
+        """
+        Take a text file that contains a NetCDF header and transform it to a 'Classic'
+        NetCDF so that NCML can be generated from the file.
+        :return:
+        """
+
+        self.nc_classic_path = os.path.join(
+            self._out_path,
+            self.nc_dump_path.replace(".txt", ".nc")
+        )
+
+        cmd = " ".join(["ncgen", "-k", "classic", "-o", self.nc_classic_path, self.nc_dump_path])
+        os.system(cmd)
+
+        if not os.path.exists(self.nc_classic_path):
+            raise IOError("Classic NC file {0} wasn't created!".format(self.nc_classic_path))
+
+    def get_ncml(self):
+        """
+        Dump NCML from a 'classic' NetCDF file.
+        :return:
+        """
+        self.out_file = os.path.join(
+            self._out_path,
+            self.in_file.replace(".nc", ".xml")
+        )
+
+        if not os.path.exists(self.nc_classic_path):
+            raise IOError("No 'Classic' NetCDF transformed with this transformer.")
+
+        cmd = " ".join(["ncdump", "-x", self.nc_classic_path, ">", self.out_file])
+        os.system(cmd)
+
+        if not os.path.exists(self.out_file):
+            raise IOError("Final Output NCML {0} doesn't exist".format(self.out_file))
+
+    def transform_final_xml(self, xsl_path):
+        """
+        Transform the XML file associated with this transformer using a given
+        XSL file.
+
+        :param xsl_path:
+        :return:
+        """
+        final_xml = os.path.join(
+            self._out_path,
+            self.out_file.replace(".xml", "_transformed.xml")
+        )
+
+        if not os.path.exists(self.out_file):
+            raise IOError("No valid NCML associated with this transformer. Can't wrangle!")
+
+        cmd = " ".join(["xsltproc", xsl_path, self.out_file, ">", final_xml])
+        os.system(cmd)
+
+        if not os.path.exists(final_xml):
+            raise IOError("Final Transformed {0} doesn't exist".format(final_xml))
+
+        return final_xml

ncmltransformdemo/metadatatransformertests.py

+"""
+This class contains all the unit tests for the GetNCMetdata Module
+
+- Justin Lynn Reid
+"""
+
+import os
+import shutil
+import unittest
+
+from ncmltransformdemo.metadatatransformer import MetadataTransformer
+
+
+class GetNCMetadataTests(unittest.TestCase):
+
+    def setUp(self):
+        """
+        Create Test Fixtures
+        :return:
+        """
+
+        # Set test data directory and file path names
+        self.home_path = os.path.dirname(__file__)
+        self.test_data_path = os.path.join(self.home_path, "testdata")
+        self.test_file = os.path.join(
+            self.test_data_path,
+            "Polar-APP-X_v01r01_Nhem_0400_d20140109_c20151222.nc"
+        )
+        self.test_out_path = os.path.expanduser("~/test_xml")
+        self.test_xsl = os.path.join(self.test_data_path, "ncml2iso_modified_from_UnidataDD2MI_demo.xsl")
+
+        # Create test data directory
+        if not os.path.exists(self.test_out_path):
+            os.makedirs(self.test_out_path)
+
+        # Create instance of MetadataTransformer to be used in each test
+        self.transformer = MetadataTransformer(self.test_file, self.test_out_path)
+
+    def tearDown(self):
+        """
+        Delete directory where test data was created/manipulated.
+        :return:
+        """
+        shutil.rmtree(self.test_out_path)
+
+    def testGetNCHeader(self):
+        """
+        Test whether NetCDF header can be successfully dumped.
+        :return:
+        """
+        self.transformer.dump_nc_file()
+        self.assertTrue(self.transformer.nc_dump_path)
+
+    def testGetClassicNC(self):
+        """
+        Test whether dumped NC header can be converted into a 'Classic'
+        NetCDF.
+        :return:
+        """
+        self.transformer.dump_nc_file()
+        self.transformer.get_classic_nc()
+        self.assertTrue(os.path.exists(self.transformer.nc_classic_path))
+
+    def testGetNCML(self):
+        """
+        Test whether NCML can be created from a 'Classic' Netcdf.
+        :return:
+        """
+        self.transformer.dump_nc_file()
+        self.transformer.get_classic_nc()
+        self.transformer.get_ncml()
+        self.assertTrue(os.path.exists(self.transformer._out_path))
+
+    def testGetTransformedXML(self):
+        """
+        Test if NCML output can be wrangled via a given XSL file.
+        :return:
+        """
+        self.transformer.dump_nc_file()
+        self.transformer.get_classic_nc()
+        self.transformer.get_ncml()
+        trans_xml = self.transformer.transform_final_xml(self.test_xsl)
+        self.assertTrue(os.path.exists(trans_xml))
+
+
+if __name__ == "__main__":
+    unittest.main()

ncmltransformdemo/ncmltransform.py

+"""
+This tool uses the code from the MetadataTransformer class to
+transform CF NetCDF NCML/XML metadata into another format as
+defined by a given XSL stylesheet.
+
+- Justin Lynn Reid
+"""
+
+import argparse
+import os
+
+from ncmltransformdemo.metadatatransformer import MetadataTransformer
+
+
+class NcmlTransform:
+
+    @staticmethod
+    def create_arg_parser():
+        """
+        Get command line arguments
+        :return: ArgParser instances containing command line arguments
+        """
+        parser = argparse.ArgumentParser(description="A Utility for creating NCML and wrangling it using XSL.")
+        parser.add_argument("--in_netcdf", dest="in_netcdf", help="Input Netcdf")
+        parser.add_argument("--xsl_file", dest="xsl_file", help="Input XSL")
+        parser.add_argument(
+            "--out_dir", dest="out_dir",
+            default=os.path.expanduser("~"), help="Output dir"
+        )
+
+        return parser.parse_args()
+
+    @staticmethod
+    def main():
+        """
+        Main method
+        :return: Null
+        """
+
+        # Get command line arguments
+        args = NcmlTransform.create_arg_parser()
+
+        # Make instance of transformer and perform xml transformation
+        meta_transformer = MetadataTransformer(
+            os.path.expanduser(args.in_netcdf),
+            os.path.expanduser(args.out_dir)
+        )
+        meta_transformer.dump_nc_file()
+        meta_transformer.get_classic_nc()
+        meta_transformer.get_ncml()
+        trans_xml = meta_transformer.transform_final_xml(os.path.expanduser(args.xsl_file))
+
+        # Delete all created side files on filesystem
+        os.remove(meta_transformer.nc_classic_path)
+        os.remove(meta_transformer.nc_dump_path)
+        os.remove(meta_transformer.out_file)
+
+        print("XML file {0} has been successfully created!".format(trans_xml))
+
+if __name__ == "__main__":
+    NcmlTransform.main()

setup.py

+"""
+This module contains the configuration for this package.
+
+- Justin Lynn Reid
+"""
+
+from setuptools import setup, find_packages
+
+
+setup(
+    name="ncmltransformdemo",
+    version=1.0,
+    packages=find_packages(),
+    scripts=["ncmltransformdemo/ncmltransform.py"],
+    test_suite='nose.collector',
+    tests_require='nose'
+)