hylite.io

  1"""
  2Import and export hyperspectral data. For hyperspectral images this is mostly done using GDAL,
  3while for point clouds and hyperspectral libraries a variety of different methods are included.
  4"""
  5from .headers import *
  6from .images import *
  7from .clouds import *
  8from .libraries import *
  9from .pmaps import *
 10from .cameras import saveCameraTXT, loadCameraTXT
 11from pathlib import Path
 12
 13from hylite import HyImage, HyCloud, HyLibrary, HyCollection, HyScene, HyData
 14from hylite.project import PMap, Camera, Pushbroom
 15from hylite.analyse.mwl import MWL
 16from distutils.dir_util import copy_tree
 17import os
 18
 19# check if gdal is installed
 20try:
 21    from osgeo import gdal
 22    usegdal = True
 23except ModuleNotFoundError:
 24    usegdal = False
 25
 26def save(path, data, **kwds):
 27    """
 28    A generic function for saving HyData instances such as HyImage, HyLibrary and HyCloud. The appropriate file format
 29    will be chosen automatically.
 30
 31    Args:
 32        path (str): the path to save the file too.
 33        data (HyData or ndarray): the data to save. This must be an instance of HyImage, HyLibrary or HyCloud.
 34        **kwds: Keywords can include:
 35
 36             - vmin = the data value that = 0 when saving RGB images.
 37             - vmax = the data value that = 255 when saving RGB images. Must be > vmin.
 38    """
 39
 40    if isinstance(data, HyImage):
 41
 42        # special case - save ternary image to png or jpg or bmp
 43        ext = os.path.splitext(path)[1].lower()
 44        if 'jpg' in ext or 'bmp' in ext or 'png' in ext or 'pdf' in ext:
 45            if data.band_count() == 1 or data.band_count() == 3 or data.band_count == 4:
 46                rgb = np.transpose( data.data, (1,0,2) )
 47                if not ((data.is_int() and np.max(rgb) <= 255)): # handle normalisation
 48                    vmin = kwds.get("vmin", np.nanpercentile(rgb, 1 ) )
 49                    vmax = kwds.get("vmax", np.nanpercentile(rgb, 99) )
 50                    rgb = (rgb - vmin) / (vmax-vmin)
 51                    rgb = (np.clip(rgb, 0, 1) * 255).astype(np.uint8) # convert to 8 bit image
 52                #from matplotlib.pyplot import imsave
 53                # imsave( path, rgb )
 54                from skimage import io as skio
 55                skio.imsave( path, rgb ) # save the image
 56                return
 57        elif ((data.band_count() == 1) or (data.band_count() == 3) or (data.band_count() == 4)) and (data.data.dtype == np.uint8):
 58            # save 1, 3 and 4 band uint8 arrays as png files
 59            # from matplotlib.pyplot import imsave
 60            # imsave( os.path.splitext(path)[0]+".png", data.data)  # save the image
 61            #from skimage import io as skio
 62            #skio.imsave(os.path.splitext(path)[0]+".png", np.transpose( data.data, (1,0,2) ))  # save the image
 63            from PIL import Image
 64            if (data.band_count() == 1):
 65                img = Image.fromarray(data.data[..., 0].T)  # single-band PNG
 66            else:
 67                img = Image.fromarray(np.transpose(data.data, (1, 0, 2)))  # ternary PNG
 68
 69            img.save(os.path.splitext(path)[0]+".png", "PNG")
 70            save( os.path.splitext(path)[0] + ".hdr", data.header ) # save header
 71            return
 72        else: # save hyperspectral image
 73            if usegdal:
 74                from osgeo import gdal  # is gdal installed?
 75                save_func = saveWithGDAL
 76            else:  # no gdal
 77                #save_func = saveWithSPy
 78                save_func = saveWithNumpy
 79            if 'lib' in ext: # special case - we are actually saving a HyLibrary (as an image)
 80                ext = 'lib'
 81            else:
 82                ext = 'dat'
 83    elif isinstance(data, HyHeader):
 84        save_func = saveHeader
 85        ext = 'hdr'
 86    elif isinstance(data, HyCloud):
 87        save_func = saveCloudPLY
 88        ext = 'ply'
 89    elif isinstance(data, HyLibrary):
 90        save_func = saveLibraryLIB
 91        ext = 'lib'
 92    elif isinstance(data, PMap ):
 93        save_func = savePMap
 94        ext = 'npz'
 95    elif isinstance(data, Camera ):
 96        save_func = saveCameraTXT
 97        ext = 'cam'
 98    elif isinstance(data, Pushbroom):
 99        save_func = saveCameraTXT
100        ext = 'brm'
101    elif isinstance(data, HyCollection):
102        save_func = _saveCollection
103        ext = data.ext[1:]
104        outdir = str( Path(data.root) / os.path.splitext(data.name)[0])
105        os.makedirs( os.path.splitext(path)[0] +"."+ ext, exist_ok=True ) # make output directory (even if empty)
106        if os.path.splitext(path)[0] != outdir:
107            if os.path.exists( outdir+"."+ext): # if it exists...
108                #if sys.version_info[1] >= 8: # python 3.8 or greater
109                #    shutil.copytree( outdir+"."+ext, os.path.splitext(path)[0]+"."+ext, dirs_exist_ok=True)
110                #else:
111                #    shutil.copytree( outdir+"."+ext, os.path.splitext(path)[0]+"."+ext ) # will fail if directory already exists unfortunately.
112                copy_tree(outdir+"."+ext, os.path.splitext(path)[0]+"."+ext)
113
114
115    elif isinstance(data, np.ndarray) or isinstance(data, list):
116        save_func = np.save
117        ext = 'npy'
118    elif isinstance( data, dict ):
119        # try serialising dicts as json files
120        def save_json( path, data ):
121            import json
122            with open(path,'w') as f:
123                json.dump( data, f, **kwds)
124        save_func = save_json
125        ext = 'json'
126    else:
127        assert False, "Error - data type %s is unsupported by hylite.io.save." % type(data)
128
129    # check path file extension
130    if 'hdr' in os.path.splitext(path)[1]: # auto strip .hdr extensions if provided
131        path = os.path.splitext(path)[0]
132    if ext not in os.path.splitext(path)[1]: # add type-specific extension if needed
133        path += '.%s'%ext
134
135    # save!
136    os.makedirs( os.path.dirname(path), exist_ok=True)  # make output directory
137    save_func( path, data )
138
139def load(path):
140    """
141    A generic function for loading hyperspectral images, point clouds and libraries. The appropriate load function
142    will be chosen based on the file extension.
143
144    Args:
145        path (str): the path of the file to load.
146
147    Returns:
148        The loaded data.
149    """
150
151    assert os.path.exists( path ), "Error: file %s does not exist." % path
152
153    # load file formats with no associated header
154    if 'npz' in os.path.splitext( path )[1].lower():
155        return loadPMap(path)
156    elif 'npy' in os.path.splitext( path )[1].lower():
157        return np.load( path ) # load numpy
158    elif 'json' in os.path.splitext( path )[1].lower():
159        import json
160        out = {}
161        with open(path,'r') as f:
162            out = json.load( f )
163        return out
164
165    # file (should/could) have header - look for it
166    header, data = matchHeader( path )
167    assert os.path.exists(str(data)), "Error - data file %s does not exist." % path
168    ext = os.path.splitext(data)[1].lower()
169    if ext == '':
170        assert os.path.isfile(data), "Error - %s is a directory not a file." % data
171
172    # load other file types
173    if 'ply' in ext: # point or hypercloud
174        out = loadCloudPLY(path) # load dataset
175    elif 'las' in ext: # point or hypercloud
176        out =  loadCloudLAS(path)
177    elif 'csv' in ext: # (flat) spectral library
178        out = loadLibraryCSV(path)
179    elif 'txt' in ext: # (flat) spectral library
180        out = loadLibraryTXT(path)
181    elif 'sed' in ext: # (flat) spectral library
182        out = loadLibrarySED(path)
183    elif 'tsg' in ext: # (flat) spectral library
184        out = loadLibraryTSG(path)
185    elif 'hyc' in ext or 'hys' in ext or 'mwl' in ext: # load hylite collection, hyscene or mwl map
186        out = _loadCollection(path)
187    elif 'cam' in ext or 'brm' in ext: # load pushbroom and normal cameras
188        out = loadCameraTXT(path)
189    else: # image
190        # load conventional images with PIL
191        if 'png' in ext or 'jpg' in ext or 'bmp' in ext:
192            # load image with matplotlib
193            #from matplotlib.pyplot import imread
194            #im = imread(path)
195            from skimage import io as skio
196            im = skio.imread(data)
197            if len(im.shape) == 2:
198                im = im[:,:,None] # add last dimension if greyscale image is loaded
199            out = HyImage(np.transpose(im, (1, 0, 2)))
200            if header is not None:
201                out.header = loadHeader(header)
202        else:
203            if usegdal:
204                from osgeo import gdal # is gdal installed?
205                out = loadWithGDAL(path)
206            else: # no gdal
207                #out = loadWithSPy(path)
208                out = loadWithNumpy(path)
209        # special case - loading spectral library; convert image to HyData
210        if 'lib' in ext:
211            out = HyLibrary(out.data, header=out.header)
212    return out  # return dataset
213
214##############################################
215## save and load data collections
216##############################################
217# save collection
218def _saveCollection(path, collection):
219    # generate file paths
220    dirmap = collection.get_file_dictionary(root=os.path.dirname(path),
221                                            name=os.path.splitext(os.path.basename(path))[0])
222    # print(dirmap)
223    for p, o in dirmap.items():
224        os.makedirs(os.path.dirname(p), exist_ok=True) # make output directory if needed
225        save(p, o)  # save each path and item [ n.b. this includes the header file! :-) ]
226
227def _loadCollection(path):
228    # load header and find directory path
229    header, directory = matchHeader(path)
230
231    # parse name and root
232    root = os.path.dirname(directory)
233    name = os.path.basename(os.path.splitext(directory)[0])
234
235    if 'hyc' in os.path.splitext(directory)[1]:
236        C = HyCollection(name, root, header=loadHeader(header))
237    elif 'hys' in os.path.splitext(directory)[1]:
238        C = HyScene(name, root, header=loadHeader(header))
239    elif 'mwl' in os.path.splitext(directory)[1]:
240        C = MWL(name, root, header=loadHeader(header))
241    else:
242        # print(header, directory )
243        assert False, "Error - %s is an invalid collection." % directory
244    return C