Bash script interfaces for generate_pstokes and noise from autocorrel…

…ations (#314)

* script interfaces.

* add scripts to installer.

* fix pstokes unittest.

* option ton include autocorrelations.

* updates.

* add tests.

* add python hash bangs.

* fix pstokes bug.

* use tildes.

* fix silent error when we have a single uvdata and dont specify dset pairs.

* add support for stokes beams.

* init primary_beam_stokes.

* import copy.

* remove bug in xants and adjust to handle autos.

* handle existing errors.

* correctly average stats arrays.

* make store_window optional in script interface.

* white space.

* add option to exclude autocorrs.

* allow the fft.

* kargs typo.

* fix exclude cross argument propagation.

* restore grouping.

* restore utils.

* restore pspecbeam.py

* restore include autocorrs / crosscorrs.

* restore pspec_run.

* get rid of auto/cross corrs.

* get rid of errant print statement.

* add bootstrap stats propagation..

* modify arg formats.

* restore parser.

* get rid of nosetest in new tests.

* add coverage for bootstrap error bar propagation.

* get rid of propagateing existing stats to bootstrap averages.

* get rid of lingering error_fields arg.

* fix grouping.

* add inplace arg.

* add arg for inplace.

* fix typo.

* fix comment edit errors and add backwards compatibility for tuple args.

* fix backwards compatibility test.

* fix tuple tuples.

* inplace -> keep_vispols

* fix spherical averaging screw up.

* remove white space.

* restore grouping line break.

* remove duplicate comment.

* roll back modifications to test_grouping.py

* add docstrings for tuple tuple args.

hera_pspec/pstokes.py

@@ -5,6 +5,7 @@
 import pyuvdata
 import copy
 from collections import OrderedDict as odict
+import argparse
 
 from . import version
 
@@ -26,7 +27,7 @@ def miriad2pyuvdata(dset, antenna_nums=None, bls=None, polarizations=None,
     Parameters
     ----------
     dset : str
-        Miriad file to convert to UVData object containing visibilities and 
+        Miriad file to convert to UVData object containing visibilities and
         corresponding metadata
 
     antenna_nums: integer list
@@ -57,7 +58,7 @@ def miriad2pyuvdata(dset, antenna_nums=None, bls=None, polarizations=None,
     """
     uvd = pyuvdata.UVData()
     uvd.read_miriad(dset, antenna_nums=antenna_nums, bls=bls,
-                    polarizations=polarizations, ant_str=ant_str, 
+                    polarizations=polarizations, ant_str=ant_str,
                     time_range=time_range)
     return uvd
 
@@ -206,7 +207,7 @@ def construct_pstokes(dset1, dset2, pstokes='pI', run_check=True, antenna_nums=N
         Ex: ['xx', 'yy', ...]
 
     time_range: float list
-        len-2 list containing min and max range of times (Julian Date) to 
+        len-2 list containing min and max range of times (Julian Date) to
         read-in. Ex: [2458115.20, 2458115.40]
 
     history : str
@@ -221,15 +222,15 @@ def construct_pstokes(dset1, dset2, pstokes='pI', run_check=True, antenna_nums=N
         assert isinstance(dset1, (str, np.str)), \
             "dset1 must be fed as a string or UVData object"
         uvd1 = miriad2pyuvdata(dset1, antenna_nums=antenna_nums, bls=bls,
-                               polarizations=polarizations, ant_str=ant_str, 
+                               polarizations=polarizations, ant_str=ant_str,
                                time_range=time_range)
     else:
         uvd1 = dset1
     if isinstance(dset2, pyuvdata.UVData) == False:
         assert isinstance(dset2, (str, np.str)), \
             "dset2 must be fed as a string or UVData object"
         uvd2 = miriad2pyuvdata(dset2, antenna_nums=antenna_nums, bls=bls,
-                               polarizations=polarizations, ant_str=ant_str, 
+                               polarizations=polarizations, ant_str=ant_str,
                                time_range=time_range)
     else:
         uvd2 = dset2
@@ -261,14 +262,14 @@ def construct_pstokes(dset1, dset2, pstokes='pI', run_check=True, antenna_nums=N
     if np.array_equal(bls1, bls2) == False:
         raise ValueError("dset1 and dset2 must have the same baselines")
 
-    # makes the Npol length==1 so that the UVData carries data for the 
+    # makes the Npol length==1 so that the UVData carries data for the
     # required polarization only
     st_keys = list(pol_weights[pstokes].keys())
     req_pol1 = st_keys[0]
     req_pol2 = st_keys[1]
 
-    # check polarizations of UVData objects are consistent with the required 
-    # polarization to form the desired pseudo Stokes visibilities. If multiple 
+    # check polarizations of UVData objects are consistent with the required
+    # polarization to form the desired pseudo Stokes visibilities. If multiple
     # exist, downselect on polarization.
     assert req_pol1 in uvd1.polarization_array, \
         "Polarization {} not found in dset1 object".format(req_pol1)
@@ -281,7 +282,7 @@ def construct_pstokes(dset1, dset2, pstokes='pI', run_check=True, antenna_nums=N
         uvd2 = uvd2.select(polarizations=req_pol2, inplace=False)
 
     # combining visibilities to form the desired Stokes visibilties
-    uvdS = _combine_pol(uvd1=uvd1, uvd2=uvd2, pol1=req_pol1, pol2=req_pol2, 
+    uvdS = _combine_pol(uvd1=uvd1, uvd2=uvd2, pol1=req_pol1, pol2=req_pol2,
                         pstokes=pstokes)
     uvdS.history += history
 
@@ -334,7 +335,29 @@ def filter_dset_on_stokes_pol(dsets, pstokes):
     assert desired_pols[0] in pols and desired_pols[1] in pols, \
         "necessary input pols {} and {} not found in dsets".format(*desired_pols)
 
-    inp_dsets = [dsets[pols.index(desired_pols[0])], 
+    inp_dsets = [dsets[pols.index(desired_pols[0])],
                  dsets[pols.index(desired_pols[1])]]
 
     return inp_dsets
+
+def generate_pstokes_argparser():
+    """
+    Get argparser to generate pstokes from linpol files.
+
+    Args:
+        N/A
+    Returns:
+        a: argparser object with arguments used in generate_pstokes_run.py
+    """
+    a = argparse.ArgumentParser(description="argument parser for computing "
+                                            "pstokes from linpol files.")
+    a.add_argument("inputdata", type=str, help="Filename of UVData object with"
+                                               "linearly polarized data to add pstokes to.")
+    a.add_argument("--pstokes", type=str, help="list of pStokes you wish to calculate. Default is ['pI']",
+                   nargs="+", default="pI")
+    a.add_argument("--outputdata", type=str, help="Filename to write out data. Output includes original linear pols."
+                                                   "if no outputdata is provided, will use inputdata, appending"
+                                                   "pstokes to original linear pols.")
+    a.add_argument("--clobber", action="store_true", default=False, help="Overwrite outputdata or original linpol only file.")
+    a.add_argument("--keep_vispols", action="store_true", default=False, help="If inplace, keep the original linear polarizations in the input file. Default is False.")
+    return a

hera_pspec/tests/test_grouping.py

@@ -10,7 +10,6 @@
 from hera_cal import redcal
 import copy
 
-
 class Test_grouping(unittest.TestCase):
 
     def setUp(self):
@@ -228,7 +227,6 @@ def test_bootstrap_average_blpairs(self):
         uvp1, wgts = grouping.bootstrap_average_blpairs([self.uvp,],
                                                         blpair_groups,
                                                         time_avg=False)
-
         uvp2, wgts = grouping.bootstrap_average_blpairs([self.uvp,],
                                                         blpair_groups,
                                                         time_avg=True)
@@ -248,6 +246,7 @@ def test_bootstrap_average_blpairs(self):
         # Reduce UVPSpec to only 3 blpairs and set them all to the same values
         _blpairs = list(np.unique(self.uvp.blpair_array)[:3])
         uvp3 = self.uvp.select(spws=0, inplace=False, blpairs=_blpairs)
+
         Nt = uvp3.Ntimes
         uvp3.data_array[0][Nt:2*Nt] = uvp3.data_array[0][:Nt]
         uvp3.data_array[0][2*Nt:] = uvp3.data_array[0][:Nt]

hera_pspec/tests/test_pspecdata.py

@@ -2187,9 +2187,18 @@ def test_window_funcs():
 
 
 def test_get_argparser():
+    args = pspecdata.get_pspec_run_argparser()
+    a = args.parse_args([['foo'], 'bar', '--dset_pairs', '0~0,1~1', '--pol_pairs', 'xx~xx,yy~yy',
+                         '--spw_ranges', '300~400, 600~800', '--blpairs', '24~25~24~25, 37~38~37~38'])
+    assert a.pol_pairs == [('xx', 'xx'), ('yy', 'yy')]
+    assert a.dset_pairs == [(0, 0), (1, 1)]
+    assert a.spw_ranges == [(300, 400), (600, 800)]
+    assert a.blpairs == [((24, 25), (24, 25)), ((37, 38), (37, 38))]
+
+def test_get_argparser_backwards_compatibility():
     args = pspecdata.get_pspec_run_argparser()
     a = args.parse_args([['foo'], 'bar', '--dset_pairs', '0 0, 1 1', '--pol_pairs', 'xx xx, yy yy',
-                         '--spw_ranges', '300 400, 600 800', '--blpairs', '24 25 24 25, 37 38 37 38'])
+                        '--spw_ranges', '300 400, 600 800', '--blpairs', '24 25 24 25, 37 38 37 38'])
     assert a.pol_pairs == [('xx', 'xx'), ('yy', 'yy')]
     assert a.dset_pairs == [(0, 0), (1, 1)]
     assert a.spw_ranges == [(300, 400), (600, 800)]

hera_pspec/tests/test_pstokes.py

@@ -2,7 +2,7 @@
 import pytest
 import os, sys
 from hera_pspec.data import DATA_PATH
-from .. import pstokes 
+from .. import pstokes
 import pyuvdata
 import pyuvdata.utils as uvutils
 import copy
@@ -111,6 +111,14 @@ def test_filter_dset_on_stokes_pol(self):
         out2 = pstokes.filter_dset_on_stokes_pol(dsets, 'pI')
         assert out == out2
 
+    def test_generate_pstokes_argparser(self):
+        # test argparser for noise error bars.
+        ap = pstokes.generate_pstokes_argparser()
+        args=ap.parse_args(["input.uvh5", "--pstokes", "pI", "pQ", "--clobber"])
+        assert args.inputdata == "input.uvh5"
+        assert args.outputdata is None
+        assert args.clobber
+
 
 if __name__ == "__main__":
     unittest.main()

hera_pspec/tests/test_utils.py

@@ -342,6 +342,16 @@ def test_get_blvec_reds():
      red_bl_tag) = utils.get_blvec_reds(uvp, bl_error_tol=1.0, match_bl_lens=True)
     assert len(red_bl_grp) == 1
 
+def test_uvp_noise_error_arser():
+    # test argparser for noise error bars.
+    ap = utils.uvp_noise_error_parser()
+    args=ap.parse_args(["container.hdf5", "autos.uvh5", "beam.uvbeam", "--groups", "dset0_dset1"])
+    assert args.pspec_container == "container.hdf5"
+    assert args.auto_file == "autos.uvh5"
+    assert args.beam == "beam.uvbeam"
+    assert args.groups == ["dset0_dset1"]
+    assert args.spectra is None
+
 def test_job_monitor():
     # open empty files
     datafiles = ["./{}".format(i) for i in ['a', 'b', 'c', 'd']]