Merge pull request #19 from SiLab-Bonn/enhancements

Tuning and Online-Monitor

.travis.yml

@@ -20,10 +20,18 @@ install:
   - export PATH="$HOME/miniconda/bin:$PATH"
   - conda update --yes conda
   - conda info -a
-  - conda create --yes -n test-environment python="$TRAVIS_PYTHON_VERSION" bitarray numpy pytables pyyaml tqdm nose
+  - conda create --yes -n test-environment python="$TRAVIS_PYTHON_VERSION" bitarray matplotlib numba numpy pyqt pytables pyyaml qtpy tqdm nose
   - source activate test-environment
+  - pip install xvfbwrapper  # fake x server for Qt gui tests
+  # Install basil
+  - pip install 'basil-daq>=3.0.0,<4.0.0'
+  # Install pybar_mimosa26_interpreter
+  - pip install git+https://github.com/SiLab-Bonn/pyBAR_mimosa26_interpreter@master
+  # Install online_monitor
+  - pip install git+https://github.com/SiLab-Bonn/online_monitor@development
   - pip install -e .
   - conda list
+  - pip list
 
 script:
   - nosetests

README.md

@@ -38,12 +38,22 @@ Python 2.7 or Python 3 or higher must be used. There are many ways to install Py
 
 Install additional required packages:
 ```bash
-conda install bitarray numpy pytables pyyaml tqdm
+conda install bitarray matplotlib numba numpy pyqt pytables pyyaml qtpy tqdm
 ```
 
-Install [Basil](https://github.com/SiLab-Bonn/basil) (>=2.4.12,<3.0.0):
+Install [Basil](https://github.com/SiLab-Bonn/basil) (>=3.0.0,<4.0.0):
 ```bash
-pip install 'basil_daq>=2.4.12,<3.0.0'
+pip install 'basil_daq>=3.0.0,<4.0.0'
+```
+
+Install [Mimosa26 Interpreter](https://github.com/SiLab-Bonn/pyBAR_mimosa26_interpreter):
+```bash
+pip install git+https://github.com/SiLab-Bonn/pyBAR_mimosa26_interpreter@master
+```
+
+Install [Online Monitor](https://github.com/SiLab-Bonn/online_monitor):
+```bash
+pip install 'online_monitor>=0.4.2,<0.5'
 ```
 
 Finally, install pymosa via:

pymosa/m26.py

@@ -23,6 +23,7 @@
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
+FORMAT = '%(asctime)s [%(name)-17s] - %(levelname)-7s %(message)s'
 
 
 class m26(object):
@@ -111,7 +112,7 @@ def check_jtag(irs, IR):
                 self.dut['JTAG'].scan_ir([BitLogic(IR[ir])] * 6)
                 ret[ir] = self.dut['JTAG'].scan_dr([self.dut[ir][:]])[0]
             # check registers
-            for k, v in ret.iteritems():
+            for k, v in ret.items():
                 if k == "CTRL_8b10b_REG1_ALL":
                     pass
                 elif k == "BSR_ALL":
@@ -205,7 +206,7 @@ def scan(self):
             while not self.stop_scan:
                 sleep(1.0)
                 if not got_data:
-                    if self.m26_readout.data_words_per_second() > 0:
+                    if self.m26_readout.data_words_per_second()[0] > 0:
                         got_data = True
                         logging.info('Taking data...')
                         if self.max_triggers:
@@ -261,6 +262,7 @@ def start(self):
         # set up logger
         self.fh = logging.FileHandler(self.run_filename + '.log')
         self.fh.setLevel(logging.DEBUG)
+        self.fh.setFormatter(logging.Formatter(FORMAT))
         self.logger = logging.getLogger()
         self.logger.addHandler(self.fh)
 
@@ -396,7 +398,7 @@ def main():
     # Open Mimosa26 std. configuration
     pymosa_path = os.path.dirname(pymosa.__file__)
     with open(os.path.join(pymosa_path, 'm26_configuration.yaml'), 'r') as f:
-        config = yaml.load(f)
+        config = yaml.safe_load(f)
 
     # Set config from arguments
     if args.filename is not None:

pymosa/m26_configuration.yaml

@@ -6,7 +6,7 @@ m26_jtag_configuration : True  # Send Mimosa26 configuration via JTAG, default:
 no_data_timeout : 30  # No data timeout after which the scan will be aborted, in seconds; if 0, the timeout is disabled
 scan_timeout : 0  # Timeout after which the scan will be stopped, in seconds; if 0, the timeout is disabled; use Ctrl-C to stop run
 max_triggers : 0  # Maximum number of triggers; if 0, there is no limit on the number of triggers; use Ctrl-C to stop run
-#send_data : 'tcp://127.0.0.1:4500'  # TCP address to which the telescope data is send; to allow incoming connections on all interfaces use 0.0.0.0
+send_data : 'tcp://127.0.0.1:8500'  # TCP address to which the telescope data is send; to allow incoming connections on all interfaces use 0.0.0.0
 #output_folder: telescope_data  # Name of the subfolder which will be created in order to store the telescope data
 #filename: run_1  # Filename of the telescope data file
 

pymosa/m26_raw_data.py

@@ -277,7 +277,7 @@ def save_conf():
 
         scan_param_table = h5_file.create_table(configuration_group, name=configuation_name, description=NameValue, title=configuation_name)
         row_scan_param = scan_param_table.row
-        for key, value in dict.iteritems(configuration):
+        for key, value in dict.items(configuration):
             row_scan_param['name'] = key
             row_scan_param['value'] = str(value)
             row_scan_param.append()

pymosa/m26_readout.py

@@ -1,7 +1,6 @@
 import logging
 import datetime
 from time import sleep, time, mktime
-from itertools import izip
 from threading import Thread, Event, Lock, Condition
 from collections import deque, Iterable
 import sys
@@ -12,6 +11,12 @@
 
 data_iterable = ("data", "timestamp_start", "timestamp_stop", "error")
 
+# Python 2/3 compability
+try:
+    basestring  # noqa
+except NameError:
+    basestring = str  # noqa
+
 
 def get_float_time():
     '''returns time as double precision floats - Time64 in pytables - mapping to and from python datetime's
@@ -206,7 +211,7 @@ def wait_for_thread_timeout(thread, fifo, timeout):
                     thread.join(timeout=timeout)
                     if thread.is_alive():
                         raise StopTimeout('Stopping %s readout thread timed out after %0.1fs' % (fifo, timeout))
-                except StopTimeout, e:
+                except StopTimeout as e:
                     self.force_stop[fifo].set()
                     if self.errback:
                         self.errback(sys.exc_info())
@@ -329,7 +334,7 @@ def worker(self, fifo):
                 if data_tuple is None:  # if None then exit
                     break
                 else:
-                    for index, (filter_func, converter_func, fifo_select) in enumerate(izip(self.filter_func, self.converter_func, self.fifo_select)):
+                    for index, (filter_func, converter_func, fifo_select) in enumerate(zip(self.filter_func, self.converter_func, self.fifo_select)):
                         if fifo_select is None or fifo_select == fifo:
                             # filter and do the conversion
                             converted_data_tuple = convert_data_iterable((data_tuple,), filter_func=filter_func, converter_func=converter_func)[0]
@@ -359,7 +364,7 @@ def writer(self, index, no_data_timeout=None):
         while True:
             try:
                 if no_data_timeout:
-                    for m26_id, time_last_data_m26 in time_last_data.iteritems():
+                    for m26_id, time_last_data_m26 in time_last_data.items():
                         if time_last_data_m26 + no_data_timeout < time():
                             raise NoDataTimeout('Received no data for %0.1f second(s) from Mimosa26 plane with ID %d' % (no_data_timeout, m26_id))
                     if time_last_data_all + no_data_timeout < time():

pymosa/noise_occupancy_scan.py

@@ -0,0 +1,130 @@
+import logging
+import os
+from time import sleep
+
+import yaml
+import numpy as np
+from tqdm import tqdm
+from matplotlib.backends.backend_pdf import PdfPages
+
+from pymosa.m26 import m26
+from pymosa import online as oa
+from pymosa.m26_raw_data import open_raw_data_file, send_meta_data
+from pymosa import plotting as plotting
+
+
+class NoiseOccScan(m26):
+    '''Noise Occupancy Scan
+
+    This script measures the noise occupancy.
+    '''
+
+    def print_log_status(self):
+        m26_rx_names = [rx.name for rx in self.dut.get_modules('m26_rx')]
+        logging.info('Mimosa26 RX channel:     %s', " | ".join([name.rjust(3) for name in m26_rx_names]))
+        for i, region in enumerate(['A', 'B', 'C', 'D']):
+            logging.info('Fake hit rate %s:         %s', region, " | ".join([format(count, '.1e').rjust(max(3, len(m26_rx_names[index]))) for index, count in enumerate(self.fake_hit_rate_meas[:len(m26_rx_names), i])]))
+        logging.info('Noise occupancy:         %s', " | ".join([repr(count).rjust(max(3, len(m26_rx_names[index]))) for index, count in enumerate(self.hit_occ_map[:, :, :len(m26_rx_names)].sum(axis=(0, 1)))]))
+
+    def take_data(self, update_rate=1):
+        with self.readout():
+            logging.info('Taking data...')
+            self.pbar = tqdm(total=self.scan_timeout, ncols=80)
+            for _ in range(int(self.scan_timeout / update_rate)):
+                sleep(update_rate)
+                try:
+                    self.pbar.update(update_rate)
+                except ValueError:
+                        pass
+
+            self.pbar.close()
+
+        # Get hit occupancy for every plane using fast online analysis
+        hit_occ_map = self.hist_occ.get()
+
+        return hit_occ_map
+
+    def init(self, init_conf=None, configure_m26=True):
+        # set name of scan
+        init_conf["run_id"] = 'noise_occupancy_scan'
+
+        super(NoiseOccScan, self).init(init_conf=init_conf, configure_m26=configure_m26)
+        self.hist_occ = oa.OccupancyHistogramming()
+
+        self.scan_timeout = self.telescope_conf.get('scan_timeout', 5)  # time for which noise occupancy is measured in seconds
+
+    def open_file(self):
+        self.raw_data_file = open_raw_data_file(filename=self.run_filename,
+                                                mode='w',
+                                                title=os.path.basename(self.run_filename),
+                                                socket_address=self.send_data)
+        if self.raw_data_file.socket:
+            # send reset to indicate a new scan for the online monitor
+            send_meta_data(self.raw_data_file.socket, None, name='Reset')
+
+    def handle_data(self, data, new_file=False, flush=True):
+        '''Handling of raw data and meta data during readout.
+        '''
+        for data_tuple in data:
+            if data_tuple is None:
+                continue
+            self.raw_data_file.append(data_iterable=data_tuple, scan_parameters=None, new_file=new_file, flush=flush)
+            # Add every raw data chunk to online analysis
+            for data in data_tuple:
+                self.hist_occ.add(raw_data=data[0])
+
+    def scan(self):
+        # Define columns which belong to regions A, B, C, D
+        m26_regions = [(0, 288), (288, 576), (576, 864), (864, 1151)]
+        self.fake_hit_rate_meas = np.zeros(shape=(6, 4))
+
+        # Take data and get noise occupancy
+        self.hit_occ_map = self.take_data()
+        for plane in range(6):
+            for region in range(4):
+                self.fake_hit_rate_meas[plane, region] = self.hit_occ_map[m26_regions[region][0]:m26_regions[region][1], :, plane].sum() / 576. / 288. / self.scan_timeout / 1e6 * 115.2
+
+        self.hist_occ.stop.set()  # stop analysis process
+
+        # Log status (fake hit rate, noise occupoancy, threshold setting)
+        self.print_log_status()
+
+    def analyze(self):
+        output_file = self.run_filename + '_interpreted.pdf'
+        logging.info('Plotting results into {0}'.format(output_file))
+        with PdfPages(output_file) as output_pdf:
+            for plane in range(6):
+                plotting.plot_occupancy(hist=np.ma.masked_where(self.hit_occ_map[:, :, plane] == 0, self.hit_occ_map[:, :, plane]).T,
+                                        title='Occupancy for plane %i' % plane,
+                                        output_pdf=output_pdf)
+
+
+if __name__ == "__main__":
+    try:
+        from pymosa import __version__ as pymosa_version
+    except ImportError:
+        try:
+            with open(os.path.join(os.path.split(os.path.split(os.path.abspath(__file__))[0])[0], 'VERSION')) as version_file:
+                pymosa_version = version_file.read().strip()
+        except IOError:
+            raise
+            pymosa_version = "(local)"
+
+    import argparse
+    parser = argparse.ArgumentParser(description='Noise occupancy scan for pymosa %s\nExample: python noise_occupancy_scan.py' % pymosa_version, formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument('-t', '--scan_timeout', type=int, metavar='<scan timeout>', action='store', help='Scan timeout, time in which noise occupancy is integrated, in seconds')
+    args = parser.parse_args()
+
+    with open('./m26_configuration.yaml', 'r') as f:
+        config = yaml.safe_load(f)
+
+    if args.scan_timeout is not None:
+        config["scan_timeout"] = range(args.scan_timeout)
+
+    noise_occ_scan = NoiseOccScan()  # None: use default hardware configuration
+    # Initialize telescope hardware and set up parameters
+    noise_occ_scan.init(init_conf=config)
+    # Start telescope readout
+    noise_occ_scan.start()
+    # Close the resources
+    noise_occ_scan.close()