allow 'nice' transfers

bin/desi_tucson_transfer_catchup.sh

@@ -27,7 +27,8 @@ log_root=${HOME}/Documents/Logfiles
 for d in engineering/focalplane engineering/focalplane/hwtables \
     spectro/data \
     spectro/redux/daily spectro/redux/daily/exposures spectro/redux/daily/preproc spectro/redux/daily/tiles \
-    spectro/nightwatch/kpno spectro/staging/lost+found; do
+    spectro/nightwatch/kpno spectro/staging/lost+found \
+    software/AnyConnect software/CiscoSecureClient; do
     case ${d} in
         engineering/focalplane) priority='nice'; exclude='--exclude archive --exclude hwtables --exclude *.ipynb --exclude .ipynb_checkpoints' ;;
         engineering/focalplane/hwtables) priority='nice'; exclude='--include *.csv --exclude *' ;;

py/desitransfer/test/test_tucson.py

@@ -169,11 +169,14 @@ def test_running_read_exit(self, mock_log, mock_popen, mock_exists, mock_remove)
 
     @patch('subprocess.Popen')
     @patch('desitransfer.tucson.log')
-    def test_get_proc(self, mock_log, mock_popen):
+    @patch('desitransfer.tucson.priority')
+    def test_get_proc(self, mock_priority, mock_log, mock_popen):
         """Test the function for generating external procedures.
         """
         directories = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i']
         exclude = set(['d', 'g'])
+        mock_priority.__contains__ = lambda self, x: x == 'e' or x == 'f'
+        # mock_priority.__iter__.return_value = ('e', 'f')
         options = MagicMock()
         options.test = False
         options.log = self.temp_dir
@@ -186,6 +189,7 @@ def test_get_proc(self, mock_log, mock_popen):
         self.assertEqual(LOG_B, os.path.join(self.temp_dir, 'desi_tucson_transfer_b.log'))
         proc, LOG_C, d = _get_proc(directories, exclude, '/src', '/dst', options)
         self.assertEqual(d, 'c')
+        options.test = False
         proc, LOG_E, d = _get_proc(directories, exclude, '/src', '/dst', options)
         self.assertEqual(d, 'e')
         proc, LOG_F, d = _get_proc(directories, exclude, '/src', '/dst', options)
@@ -196,3 +200,13 @@ def test_get_proc(self, mock_log, mock_popen):
         self.assertEqual(d, 'i')
         proc, LOG_J, d = _get_proc(directories, exclude, '/src', '/dst', options)
         self.assertIsNone(proc)
+        mock_log.info.assert_has_calls([call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/a/ /dst/a/'),
+                                        call("Directory '%s' will be transferred with os.nice(%d)", 'a', 5),
+                                        call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/e/ /dst/e/'),
+                                        call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/f/ /dst/f/'),
+                                        call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/h/ /dst/h/'),
+                                        call("Directory '%s' will be transferred with os.nice(%d)", 'h', 5),
+                                        call('/usr/bin/rsync --archive --checksum --verbose --delete --delete-after --no-motd --password-file /Users/benjamin.weaver/.desi /src/i/ /dst/i/'),
+                                        call("Directory '%s' will be transferred with os.nice(%d)", 'i', 5)])
+        mock_log.warning.assert_has_calls([call('%s skipped at user request.', 'd'),
+                                           call('%s skipped at user request.', 'g')])

py/desitransfer/tucson.py

@@ -55,6 +55,7 @@
            'spectro/redux/daily/preproc',
            'spectro/redux/daily/tiles',
            'engineering/focalplane',
+           'engineering/focalplane/hwtables',
            'software/AnyConnect',
            'software/CiscoSecureClient']
 
@@ -78,6 +79,13 @@
                                 "--include", "gaiadr2", "--include", "subpriority", "--exclude", "*"]}
 
 
+priority = ('spectro/data',
+            'spectro/redux/daily',
+            'spectro/redux/daily/exposures',
+            'spectro/redux/daily/preproc',
+            'spectro/redux/daily/tiles')
+
+
 def _configure_log(debug):
     """Re-configure the default logger returned by ``desiutil.log``.
 
@@ -169,7 +177,7 @@ def _rsync(src, dst, d, checksum=False):
     return cmd
 
 
-def _get_proc(directories, exclude, src, dst, options):
+def _get_proc(directories, exclude, src, dst, options, nice=5):
     """Prepare the next download directory for processing.
 
     Parameters
@@ -184,13 +192,23 @@ def _get_proc(directories, exclude, src, dst, options):
         Root destination directory.
     options : :class:`argparse.Namespace`
         The parsed command-line options.
+    nice : :class:`int`, optional.
+        Lower-priority transfers will be run with this value passed to :func:`os.nice`,
+        default 5.
 
     Returns
     -------
     :class:`tuple`
         A tuple containing information about the process.
     """
     global log
+
+    def preexec_nice():  # pragma: no cover
+        os.nice(nice)
+
+    def preexec_pass():  # pragma: no cover
+        pass
+
     try:
         d = directories.pop(0)
         while d in exclude:
@@ -204,7 +222,12 @@ def _get_proc(directories, exclude, src, dst, options):
         else:
             log.info(' '.join(command))
             LOG = open(log_file, 'ab')
-            return (sub.Popen(command, stdout=LOG, stderr=sub.STDOUT), LOG, d)
+            if d in priority:
+                preexec_fn = preexec_pass
+            else:
+                log.info("Directory '%s' will be transferred with os.nice(%d)", d, nice)
+                preexec_fn = preexec_nice
+            return (sub.Popen(command, preexec_fn=preexec_fn, stdout=LOG, stderr=sub.STDOUT), LOG, d)
     except IndexError:
         return (None, None, None)