diff --git a/gutils/__init__.py b/gutils/__init__.py index 2a4e157..6be5b87 100644 --- a/gutils/__init__.py +++ b/gutils/__init__.py @@ -12,7 +12,12 @@ import numpy as np import pandas as pd -from scipy.signal import boxcar, convolve +try: + # SciPy 1.13.0 DepricationWarning + from scipy.signal.windows import boxcar + from scipy.signal import convolve +except ImportError: + from scipy.signal import boxcar, convolve from pocean.meta import MetaInterface from pocean.utils import ( diff --git a/gutils/nc.py b/gutils/nc.py index 711a5fd..327232e 100644 --- a/gutils/nc.py +++ b/gutils/nc.py @@ -396,10 +396,13 @@ def create_netcdf(attrs, data, output_path, mode, profile_id_type=ProfileIdTypes if not extras_df.empty: + # Skip extra flag + skip_extra = False + # Write the extras dimension to a new profile file profile_extras = extras_df.loc[extras_df.profile == pi].copy() if profile_extras.empty: - continue + skip_extra = True # Standardize the columns of the "extras" from the matched profile profile_extras.loc[:, 't'] = profile_extras.index @@ -420,9 +423,18 @@ def create_netcdf(attrs, data, output_path, mode, profile_id_type=ProfileIdTypes profile_extras.loc[:, c] = profile_extras[c].astype(profile[c].dtype) """ + # If there is less than 3 time coordinates with an echogram, do not + # write out the profile. + if 'echogram_sv' in profile_extras.columns: + if 't' not in profile_extras.columns: + skip_extra = True + if len(pd.unique(profile_extras['t'])) < 3: + skip_extra = True + try: - cr = create_profile_netcdf(attrs, profile_extras, output_path, mode + '_extra', profile_id_type) - written_files.append(cr) + if not skip_extra: + cr = create_profile_netcdf(attrs, profile_extras, output_path, mode + '_extra', profile_id_type) + written_files.append(cr) except BaseException: L.exception('Error creating extra netCDF profile {}. Skipping.'.format(pi)) continue diff --git a/gutils/slocum/__init__.py b/gutils/slocum/__init__.py index 2ca9061..36756bb 100644 --- a/gutils/slocum/__init__.py +++ b/gutils/slocum/__init__.py @@ -672,6 +672,9 @@ def slocum_binary_sorter(x): # of a reader. self.extra_kwargs = self.attrs.pop('extra_kwargs', {}) + if self.extra_kwargs.get('enable_debug', False): + L.setLevel(logging.DEBUG) + def __del__(self): # Remove tmpdir shutil.rmtree(self.tmpdir, ignore_errors=True) @@ -868,7 +871,7 @@ def convert(self): # upstream expects *.dat files, so produce # a *.pq file and link with to *.dat file try: - os.symlink(pqFile, datFile) + os.symlink(os.path.basename(pqFile), datFile) except FileExistsError: L.error(f"Symlink already exists for {datFile}") else: @@ -888,7 +891,7 @@ def convert(self): echograms_attrs = self.extra_kwargs.get('echograms', {}) enable_ascii = echograms_attrs.get('enable_ascii', False) enable_image = echograms_attrs.get('enable_image', False) - enable_debug = echograms_attrs.get('enable_debug', False) + enable_debug = self.extra_kwargs.get('enable_debug', False) if enable_debug: pargs.remove('-q') diff --git a/gutils/slocum/bin/convertDbds.sh b/gutils/slocum/bin/convertDbds.sh index 6afb330..dd17eb0 100755 --- a/gutils/slocum/bin/convertDbds.sh +++ b/gutils/slocum/bin/convertDbds.sh @@ -425,7 +425,7 @@ do # dbdSource must have the ascii header line dbd_label: to be a valid *bd # file - is_dbd=$(grep 'dbd_label:' $dbdSource); + is_dbd=$(grep -a 'dbd_label:' $dbdSource); if [ -z "$is_dbd" ] then echo "Invalid flight source file: $dbdSource" >&2; @@ -471,7 +471,7 @@ do # dbdSource must have the ascii header line dbd_label: to be a valid *bd # file - is_dbd=$(grep 'dbd_label:' $sciSource); + is_dbd=$(grep -a 'dbd_label:' $sciSource); if [ -z "$is_dbd" ] then echo "Invalid science source file: $sciSource" >&2; diff --git a/gutils/slocum/echotools/teledyne.py b/gutils/slocum/echotools/teledyne.py index b35589c..aa02567 100644 --- a/gutils/slocum/echotools/teledyne.py +++ b/gutils/slocum/echotools/teledyne.py @@ -8,6 +8,7 @@ import sys import struct import subprocess +import warnings import numpy as np from matplotlib import pyplot as plt import matplotlib as mpl @@ -23,6 +24,21 @@ logging.getLogger("dbdreader").setLevel(logging.WARNING) logging.getLogger("matplotlib").setLevel(logging.WARNING) +# Default +if os.environ.get('PYTHONWARNINGS', None): + pass +else: + warnings.simplefilter("default") + +# Development +#warnings.resetwarnings() +#warnings.filterwarnings("error") + +# netcdf4 (obscure error) +warnings.filterwarnings("ignore", message='numpy.*size changed.*') +# TODO: Plotting breaks down when there are multiple profiles +warnings.filterwarnings("ignore", message='.*coordinates to pcolormesh.*') + class Glider: ''' @@ -293,7 +309,7 @@ def createFileInventory(self, fileList, cache_dir): plabel = f"PATH{ct:04d}" self.data['inventory_paths'][abspath] = plabel - dbdFp = dbdreader.DBD(infile, cacheDir=cache_dir) + dbdFp = dbdreader.DBD(infile, cacheDir=cache_dir, skip_initial_line=False) dbdData = dbdFp.get(*dbdFp.parameterNames, return_nans=True) cacheFile = f"{dbdFp.cacheID}.cac" @@ -365,8 +381,8 @@ def filterFiles(self, start_time=None, end_time=None): # Convert 0000-00-00 00:00:00 to nan mask = ds['End'] == '0000-00-00 00:00:00' ds['End'][mask] = np.nan - ds['Start_dt'] = ds['Start'].astype('datetime64', errors='ignore') - ds['End_dt'] = ds['End'].astype('datetime64', errors='ignore') + ds['Start_dt'] = ds['Start'].astype('datetime64[ns]', errors='ignore') + ds['End_dt'] = ds['End'].astype('datetime64[ns]', errors='ignore') ds_start_time = pd.to_datetime(start_time).to_datetime64() ds_end_time = pd.to_datetime(end_time).to_datetime64() @@ -420,9 +436,9 @@ def filterFiles(self, start_time=None, end_time=None): # and pull through files that match the base file for # completeness. for gkey in groupList.keys(): - match = "%s\\." % (os.path.basename(gkey)) - ss1 = ds.loc[ds['File'].str.contains(match)] - ss2 = df.loc[df['File'].str.contains(match)] + match = "%s." % (os.path.basename(gkey)) + ss1 = ds.loc[ds['File'].str.contains(match, regex=False)] + ss2 = df.loc[df['File'].str.contains(match, regex=False)] df = pd.concat([df, ss1, ss2]).drop_duplicates() inv.data['inventory'] = df fileList = inv.getFullFilenamesFromFileInventory() @@ -639,6 +655,7 @@ def loadFileInventory(self, fname): df = pd.concat([df, pd.Series(rec).to_frame().T], ignore_index=True) self.data['inventory'] = df + fn.close() return @@ -662,7 +679,8 @@ def loadMetadata(self): # Attempt to read echotools.json configuration file try: echotoolsFile = os.path.join(self.args['deploymentDir'], 'echotools.json') - testLoad = json.load(open(echotoolsFile)) + with open(echotoolsFile) as fp: + testLoad = json.load(fp) self.echotools = testLoad except Exception: print("WARNING: Unable to parse json echotools file: %s" % (echotoolsFile)) @@ -671,7 +689,8 @@ def loadMetadata(self): # Attempt to read deployment.json try: deploymentFile = os.path.join(self.args['deploymentDir'], 'deployment.json') - testLoad = json.load(open(deploymentFile)) + with open(deploymentFile, "r") as fp: + testLoad = json.load(fp) self.deployment = testLoad except Exception: print("ERROR: Unable to parse json deployment file: %s" % (deploymentFile)) @@ -680,7 +699,8 @@ def loadMetadata(self): # Attempt to read instruments.json try: instrumentsFile = os.path.join(self.args['deploymentDir'], 'instruments.json') - testLoad = json.load(open(instrumentsFile)) + with open(instrumentsFile, "r") as fp: + testLoad = json.load(fp) self.instruments = testLoad except Exception as err: print(f"ERROR: Unable to parse json instruments file: {instrumentsFile} {err=}") @@ -699,7 +719,8 @@ def loadMetadata(self): # Attempt to read