diff --git a/.github/workflows/phantoms.yaml b/.github/workflows/phantoms.yaml index 5e2222d0..053b7e5f 100644 --- a/.github/workflows/phantoms.yaml +++ b/.github/workflows/phantoms.yaml @@ -4,6 +4,7 @@ on: push: branches: - main + - switch-phantoms-to-s3 pull_request: branches: - main @@ -66,6 +67,18 @@ jobs: brew install dcm2niix fi + - name: Debug + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} + timeout-minutes: 15 + with: + limit-access-to-actor: true + + - name: Check dcm2niix is installed and on path + if: runner.os != 'Windows' + run: + dcm2niix -h + - name: Install dcm2niix windows if: runner.os == 'Windows' run: | @@ -84,7 +97,6 @@ jobs: run: | cd pypet2bids pip install . - pip install gdown - name: Install Poetry Build Package run: | @@ -99,9 +111,9 @@ jobs: - name: Install BIDS Validator run: npm install -g bids-validator - - name: Collect Phantoms from Google Drive + - name: Collect Phantoms if: ${{ steps.cache-phantoms.outputs.cache-hit != 'true' }} && ${{ !env.ACT }} - run: gdown ${{ secrets.ZIPPED_OPEN_NEURO_PET_PHANTOMS_URL }} -O PHANTOMS.zip + run: wget -O PHANTOMS.zip https://openneuropet.s3.amazonaws.com/US-sourced-OpenNeuroPET-Phantoms.zip - name: Decompress phantoms windows if: steps.cache-phantoms.outputs.cache-hit != 'true' && !env.ACT && runner.os == 'Windows' diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index b5f04270..4db84581 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -4,6 +4,7 @@ on: pull_request: branches: - main + - switch-phantoms-to-s3 workflow_call: workflow_dispatch: inputs: @@ -18,8 +19,7 @@ jobs: runs-on: ${{ matrix.os }} env: - ECAT_TEST_FOLDER: "cimbi36" - REAL_TEST_ECAT_PATH: cimbi36/Gris_102_19_2skan-2019.04.30.13.04.41_em_3d.v + REAL_ECAT_TEST_PATH: ${{ github.workspace }}/OpenNeuroPET-Phantoms/sourcedata/SiemensHRRT-JHU/Hoffman.v SMALLER_ECAT_PATH: ${{ github.workspace }}/ecat_validation/ECAT7_multiframe.v.gz TEST_ECAT_PATH: ${{ github.workspace }}/ecat_validation/ECAT7_multiframe.v OUTPUT_NIFTI_PATH: ${{ github.workspace}}/pypet2bids/tests/ECAT7_multiframe.nii @@ -36,15 +36,15 @@ jobs: - name: Checkout Repo uses: actions/checkout@v2 - - name: Cache ECAT - id: cache-ecat + - name: Cache Phantoms + id: cache-phantoms uses: actions/cache@v2 with: - path: cimbi36 - key: ${{ runner.os }}-ecats + path: US-sourced-OpenNeuroPET-Phantoms.zip + key: ${{ runner.os }}-phantoms - name: Set up python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} @@ -52,25 +52,33 @@ jobs: if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: | cd pypet2bids - pip3 install . + python -m pip install --upgrade pip + pip install poetry + poetry install --with dev - - name: Install gdown and collect Ecat from Google Drive - if: steps.cache-ecat.outputs.cache-hit != 'true' - run: "python3 -m pip install gdown && gdown ${{ secrets.CIMBI_ECAT_ON_GOOGLE_DRIVE }} -O ecat_test" + - name: Collect ECAT and other phantoms + if: steps.cache-phantoms.outputs.cache-hit != 'true' + run: "wget https://openneuropet.s3.amazonaws.com/US-sourced-OpenNeuroPET-Phantoms.zip" - - name: Decompress dataset - if: steps.cache-ecat.outputs.cache-hit != 'true' - run: "tar xvzf ecat_test && rm ecat_test" + - name: Decompress ECAT and other phantoms + run: "unzip US-sourced-OpenNeuroPET-Phantoms.zip" + + - name: Debug + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} + timeout-minutes: 15 + with: + limit-access-to-actor: true - name: Test CLI --help run: | cd pypet2bids/ - python3 -m pypet2bids.ecat_cli --help + poetry run python3 -m pypet2bids.ecat_cli --help - name: Test CLI Ecat Dump run: | cd pypet2bids/ - python3 -m pypet2bids.ecat_cli ../${{ env.REAL_TEST_ECAT_PATH }} --dump + poetry run python3 -m pypet2bids.ecat_cli ${{ env.REAL_ECAT_TEST_PATH }} --dump # the larger real data file uses too much ram for the github runner, we use the small file for # heavy io operations instead @@ -79,9 +87,9 @@ jobs: gzip -d ${{ env.SMALLER_ECAT_PATH }} - name: Test ecatread - run: "cd pypet2bids/ && python3 -m tests.test_ecatread" + run: "cd pypet2bids/ && poetry run python3 -m tests.test_ecatread" - name: Run All Other Python Tests w/ Pytest run: | cd pypet2bids - pytest -k 'not write_pixel_data' -k 'not test_convert_pmod_to_blood' tests/ + poetry run pytest -k 'not write_pixel_data' -k 'not test_convert_pmod_to_blood' tests/ diff --git a/.github/workflows/readthedocs.yaml b/.github/workflows/readthedocs.yaml new file mode 100644 index 00000000..a6e61ba4 --- /dev/null +++ b/.github/workflows/readthedocs.yaml @@ -0,0 +1,23 @@ +name: readthedocs/actions +on: + pull_request_target: + types: [opened, synchronize, reopened] + workflow_dispatch: + inputs: + debug_enabled: + type: boolean + description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' + required: false + default: false + +permissions: + pull-requests: write + +jobs: + pull-request-links: + runs-on: ubuntu-latest + steps: + - name: Preview RTD + uses: readthedocs/actions/preview@v1 + with: + project-slug: "pet2bids" diff --git a/.gitignore b/.gitignore index a6cefea2..f145782a 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,13 @@ D:\\BIDS* # test data *OpenNeuroPET-Phantoms* **PHANTOMS.zip +ecat_validation/ECAT7_multiframe.v +variables.env +ecat_testing/matlab/ +ecat_testing/steps/ +ecat_testing/**/*.nii* +*osem* +*fbp* + +matlab/SiemensHRRTparameters.txt + diff --git a/.readthedocs.yaml b/.readthedocs.yaml index a51fe36e..1e52d0f7 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -14,27 +14,22 @@ submodules: # Set the version of Python and other tools you might need build: - os: ubuntu-20.04 + os: ubuntu-22.04 tools: - python: "3.9" + python: "3.11" jobs: post_create_environment: + - cp -r metadata pypet2bids/pypet2bids/ # install poetry - # https://python-poetry.org/docs/#osx--linux--bashonwindows-install-instructions - - curl -sSL https://install.python-poetry.org | python3 - - # Tell poetry to not use a virtual environment - - $HOME/.local/bin/poetry config virtualenvs.create false + - pip install poetry + - poetry config virtualenvs.create false # just use poetry to export a requirements.txt as that worked much better than the previous attempts - - cd pypet2bids && $HOME/.local/bin/poetry export --with dev --without-hashes -o requirements.txt + - cd pypet2bids && poetry lock && poetry export --without-hashes --with dev --format=requirements.txt > requirements.txt # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/conf.py -# If using Sphinx, optionally build your docs in additional formats such as PDF -# formats: -# - pdf - # Optionally declare the Python requirements required to build your docs python: install: diff --git a/CITATION.cff b/CITATION.cff index 0f29efc0..f1bfda08 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -31,11 +31,17 @@ authors: - family-names: "Ganz-Benjaminsen" given-names: "Melanie" orcid: "https://orcid.org/0000-0002-9120-8098" +- family-names: "Bilgel" + given-names: "Murat" + orcid: "https://orcid.org/0000-0001-5042-7422" +- family-names: "Eierud" + given-names: "Cyrus" + orcid: "https://orcid.org/0000-0002-9942-676X" - family-names: "Pernet" given-names: "Cyril" orcid: "https://orcid.org/0000-0003-4010-4632" title: "PET2BIDS" -version: 1.0.0 +version: 1.3.20240502 url: "https://github.com/openneuropet/PET2BIDS" date-released: 2022-10-01 license: MIT diff --git a/JOSS_metadata/persons_info.txt b/JOSS_metadata/persons_info.txt index eeb76abb..9045d569 100644 --- a/JOSS_metadata/persons_info.txt +++ b/JOSS_metadata/persons_info.txt @@ -21,3 +21,5 @@ 0000-0002-9120-8098 Melanie Ganz-Benjaminse Department of Computer Science, University of Copenhagen, Copenhagen, Denmark 0000-0003-4010-4632 Cyril Pernet Neurobiology Research Unit, Rigshospitalet, Copenhagen, Denmark + +0000-0001-5042-7422 Murat Bilgel National Institute on Aging Intramural Research Program, Baltimore, MD, USA diff --git a/Makefile b/Makefile index ef1b26d9..67a1470d 100644 --- a/Makefile +++ b/Makefile @@ -43,3 +43,47 @@ installpackage: testphantoms: @scripts/testphantoms + +html: + @cd docs && make html + +installdependencies: + @cd pypet2bids; \ + python -m pip install --upgrade pip; \ + pip install poetry; \ + poetry install --with dev + +collectphantoms: +ifeq (, $(wildcard ./PHANTOMS.zip)) + @wget -O PHANTOMS.zip https://openneuropet.s3.amazonaws.com/US-sourced-OpenNeuroPET-Phantoms.zip +else + @echo "PHANTOMS.zip already exists" +endif + +decompressphantoms: + @unzip -o PHANTOMS.zip + +testecatcli: + @cd pypet2bids; \ + poetry run python -m pypet2bids.ecat_cli --help; \ + poetry run python -m pypet2bids.ecat_cli ../OpenNeuroPET-Phantoms/sourcedata/SiemensHRRT-JHU/Hoffman.v --dump + +testecatread: + @cd pypet2bids; \ + export TEST_ECAT_PATH="../OpenNeuroPET-Phantoms/sourcedata/SiemensHRRT-JHU/Hoffman.v"; \ + export READ_ECAT_SAVE_AS_MATLAB="$$PWD/tests/ECAT7_multiframe.mat"; \ + export NIBABEL_READ_ECAT_SAVE_AS_MATLAB="$$PWD/tests/ECAT7_multiframe.nibabel.mat"; \ + poetry run python3 -m tests.test_ecatread + +testotherpython: + cd pypet2bids; \ + export TEST_DICOM_IMAGE_FOLDER="../OpenNeuroPET-Phantoms/sourcedata/SiemensBiographPETMR-NIMH/AC_TOF"; \ + poetry run pytest --ignore=tests/test_write_ecat.py tests/ -vvv + +pythongithubworkflow: installdependencies collectphantoms decompressphantoms testecatread testecatcli testotherpython + @echo finished running python tests + +black: + @for file in `find pypet2bids/ -name "*.py"`; do \ + black $$file; \ + done diff --git a/README.md b/README.md index d9a6ffa7..68f5e7c4 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ # PET2BIDS is a code library to convert source Brain PET data to BIDS -## (more doc @https://pet2bids.readthedocs.io/en/latest/index.html) - [![python](https://github.com/openneuropet/PET2BIDS/actions/workflows/python.yaml/badge.svg)](https://github.com/openneuropet/PET2BIDS/actions/workflows/python.yaml) [![Matlab PET2BIDS Tests](https://github.com/openneuropet/PET2BIDS/actions/workflows/matlab.yaml/badge.svg)](https://github.com/openneuropet/PET2BIDS/actions/workflows/matlab.yaml) [![Documentation Status](https://readthedocs.org/projects/pet2bids/badge/?version=latest)](https://pet2bids.readthedocs.io/en/latest/?badge=latest) -[![phantoms](https://github.com/openneuropet/PET2BIDS/actions/workflows/phantoms.yaml/badge.svg?event=push)](https://github.com/openneuropet/PET2BIDS/actions/workflows/phantoms.yaml) +[![phantoms](https://github.com/openneuropet/PET2BIDS/actions/workflows/phantoms.yaml/badge.svg)](https://github.com/openneuropet/PET2BIDS/actions/workflows/phantoms.yaml) -This repository is hosting tools to curate PET brain data using the [Brain Imaging Data Structure Specification](https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/09-positron-emission-tomography.html). The work to create these tools is funded by [Novo Nordisk fonden](https://novonordiskfonden.dk/en/) (NNF20OC0063277) and the [BRAIN initiative](https://braininitiative.nih.gov/) (MH002977-01). +This repository is hosting tools to curate PET brain data using the [Brain Imaging Data Structure Specification](https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/09-positron-emission-tomography.html). +The work to create these tools is funded by [Novo Nordisk Foundation](https://novonordiskfonden.dk/en/) (NNF20OC0063277) and the +[BRAIN initiative](https://braininitiative.nih.gov/) (MH002977-01). -For DICOM conversion, we rely on [dcm2niix](https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage), +For DICOM image conversion, we rely on [dcm2niix](https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage), collaborating with Prof. Chris Rorden without whom we could not convert your data! For more information on dcm2niix and nifti please see [The first step for neuroimaging data analysis: DICOM to NIfTI conversion](https://www.ncbi.nlm.nih.gov/pubmed/26945974) paper. @@ -22,7 +22,8 @@ For **more detailed** (and most likely helpful) documentation visit the Read the ## Installation -Simply download the repository - follow the specific Matlab or Python explanations. Matlab and Python codes provide the same functionalities. +Simply download the repository - follow the specific Matlab or Python explanations. Matlab and Python codes provide the +same functionalities. ### matlab @@ -34,19 +35,74 @@ Simply download the repository - follow the specific Matlab or Python explanatio ### pypet2bids -Use pip: +Use pip to install this library directly from PyPI: [![asciicast](https://asciinema.org/a/TZJg5BglDMFM2fEEX9dSpnJEy.svg)](https://asciinema.org/a/TZJg5BglDMFM2fEEX9dSpnJEy) -For advance users clone this repository and run from the python source under the `PET2BIDS/pypet2bids` folder. If you -wish to build and install via pip locally we recommend you do so using [poetry](https://python-poetry.org/) build or -using the make commands below. +If you wish to install directly from this repository see the instructions below to either build +a packaged version of `pypet2bids` or how to run the code from source. + +
+Build Package Locally and Install with PIP + +We use [poetry](https://python-poetry.org/) to build this package, no other build methods are supported, +further we encourage the use of [GNU make](https://www.gnu.org/software/make/) and a bash-like shell to simplify the +build process. + +After installing poetry, you can build and install this package to your local version of Python with the following +commands (keep in mind the commands below are executed in a bash-like shell): + +```bash +cd PET2BIDS +cp -R metadata/ pypet2bids/pypet2bids/metadata +cp pypet2bids/pyproject.toml pypet2bids/pypet2bids/pyproject.toml +cd pypet2bids && poetry lock && poetry build +pip install dist/pypet2bids-X.X.X-py3-none-any.whl +``` + +Why is all the above required? Well, because this is a monorepo and we just have to work around that sometimes. + + +[!NOTE] +Make and the additional scripts contained in the `scripts/` directory are for the convenience of +non-windows users. + +If you have GNU make installed and are using a bash or something bash-like in you your terminal of choice, run the +following: ```bash cd PET2BIDS make installpoetry buildpackage installpackage ``` +
+ +
+Run Directly From Source + +Lastly, if one wishes run pypet2bids directly from the source code in this repository or to help contribute to the python portion of this project or any of the documentation they can do so via the following options: + +```bash +cd PET2BIDS/pypet2bids +poetry install +``` + +Or they can install the dependencies only using pip: + +```bash +cd PET2BIDS/pypet2bids +pip install . +``` + +After either poetry or pip installation of dependencies modules can be executed as follows: + +```bash +cd PET2BIDS/pypet2bids +python dcm2niix4pet.py --help +``` + +
+ **Note:** *We recommend using dcm2niix v1.0.20220720 or newer; we rely on metadata included in these later releases. It's best to collect releases from the [rorden lab/dcm2niix/releases](https://github.com/rordenlab/dcm2niix/releases) page. We have @@ -64,7 +120,7 @@ A small collection of json files for our metadata information. ### user metadata -No matter the way you prefer inputting metadata (passing all arguments, using txt or env file, using spreadsheets), you are always right! DICOM values will be ignored - BUT they are checked and the code tells you if there is inconsistency between your inputs and what DICOM says. +No matter the way you prefer inputting metadata (passing all arguments, using txt or env file, using spreadsheets), you are always right! DICOM values will be ignored - BUT they are checked and the code tells you if there is inconsistency between your inputs and what the DICOM says. ### ecat_validation diff --git a/contributors.md b/contributors.md index 3c2da6d7..5ec83807 100644 --- a/contributors.md +++ b/contributors.md @@ -3,7 +3,7 @@ The following individuals have contributed to the PET2BIDS project (in alphabetical order). If you contributed and some icons needs to be added or your name is not listed, please add it. -Murat Bilgel 💻 🐛 💡 +Murat Bilgel 💻 🐛 💡 Anthony Galassi 💻 📖 💬 🎨 💡 ⚠️ Melanie Ganz-Benjaminsen 🔍 💬 🤔 📋 Gabriel Gonzalez-Escamilla 💻 ⚠️ 🐛 👀 diff --git a/docs/conf.py b/docs/conf.py index b1cb057f..608a46b4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,19 +31,19 @@ # get absolute path to python project files -python_project_path = pathlib.Path(os.path.abspath('../pypet2bids')) +python_project_path = pathlib.Path(os.path.abspath("../pypet2bids")) matlab_project_path = os.path.join(python_project_path.parent) sys.path.insert(0, str(python_project_path)) sys.path.insert(0, matlab_project_path) # -- Project information ----------------------------------------------------- -project = 'PET2BIDS' -copyright = '2022, OpenNeuroPET' -author = 'OpenNeuroPET' +project = "PET2BIDS" +copyright = "2022, OpenNeuroPET" +author = "OpenNeuroPET" # The full version, including alpha/beta/rc tags -release = '0.0.11' +release = "0.0.11" # -- General configuration --------------------------------------------------- @@ -54,21 +54,21 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.duration', - 'sphinx.ext.doctest', - 'sphinx.ext.autodoc', - 'sphinx_rtd_theme', - 'sphinxcontrib.matlab', + "sphinx.ext.duration", + "sphinx.ext.doctest", + "sphinx.ext.autodoc", + "sphinx_rtd_theme", + "sphinxcontrib.matlab", ] -#autodoc_mock_imports = ['pypet2bids'] +# autodoc_mock_imports = ['pypet2bids'] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'tests'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "tests"] # -- Options for HTML output ------------------------------------------------- @@ -77,7 +77,7 @@ # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/index.rst b/docs/index.rst index e822c8b4..4e2415d8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,7 +11,6 @@ Welcome to PET2BIDS's documentation! installation usage modules - matlab spreadsheets :maxdepth: 2 :caption: Contents: diff --git a/docs/installation.rst b/docs/installation.rst index 0e3a4e50..6c2d54e4 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -1,4 +1,4 @@ -.. _installation +.. _installation: Installation ============ @@ -6,7 +6,8 @@ Installation Matlab ------ -In short, add the contents of the `matlab` folder to your matlab path with `addpath`: +Clone the repo at https://github.com/openneuropet/PET2BIDS.git and the contents of the `matlab` folder to your matlab +path with `addpath`: .. code-block:: @@ -23,7 +24,7 @@ Python ------ The python version of PET2BIDS (from herein referenced by it's library name *pypet2bids*) can be installed -via pip for Python versions >3.7.1,<3.10 +via pip for Python versions >3.7.1,=<3.11 .. code-block:: @@ -35,11 +36,15 @@ via pip for Python versions >3.7.1,<3.10 +If you wish to contribute, are unable to install from PyPi, or simply wish to run pypet2bids from source, continue +reading the `Additional Install Notes`_ section below. + Additional Install Notes -======================== +------------------------ -Matlab ------- +**Matlab** + +------------------------------------------------------------------------------------------------------------------------ **Dependencies** @@ -72,16 +77,47 @@ all arguments in although this is also possible). You can find templates of such ------------------------------------------------------------------------------------------------------------------------ -Python ------- +**Python** -pypet2bids can be run from source by cloning the source code at our Github_. +If you are unable to install this library from PyPi you can clone this repository to build and install the package +as distributed on PyPi yourself with poetry. + +We use `poetry `_ to build this package, no other build methods are supported, +further we encourage the use of `GNU make `_ and a bash-like shell to simplify the +build process. + +After installing poetry, you can build and install this package to your local version of Python with the following +commands (keep in mind the commands below are executed in a bash-like shell): + +.. code-block:: + + cd PET2BIDS + cp -R metadata/ pypet2bids/pypet2bids/metadata + cp pypet2bids/pyproject.toml pypet2bids/pypet2bids/pyproject.toml + cd pypet2bids && poetry lock && poetry build + pip install dist/pypet2bids-X.X.X-py3-none-any.whl + +.. note:: + + Make and the additional scripts contained in the `scripts/` directory are for the convenience of + non-windows users. + +If you have GNU make installed and are using a bash or something bash-like in you your terminal of choice, run the +following: + +.. code-block:: + + cd PET2BIDS + make installpoetry buildpackage installpackage .. _Github: https://github.com/openneuropet/PET2BIDS + +pypet2bids can be run from source by cloning the source code at our Github_. + .. code-block:: - git clone git@github.com:openneuropet/PET2BIDS.git + git clone https://github.com/openneuropet/PET2BIDS and then installing it's dependencies via pip: @@ -90,13 +126,20 @@ and then installing it's dependencies via pip: cd PET2BIDS/pypet2bids pip install . -Or with `Poetry `_: +or installing them with `Poetry `_: .. code-block:: cd PET2BIDS/pypet2bids poetry install +After either poetry or pip installation of dependencies modules can be executed as follows: + +.. code-block:: + + cd PET2BIDS/pypet2bids + python dcm2niix4pet.py --help + **Windows Only** It's important that python be on your windows path; when installing Python be sure to select **Add Python 3.XXX** @@ -133,5 +176,3 @@ Or using the *dcm2niix4pet* tool itself to set up the configuration: .. code-block:: dcm2niix4pet --set-dcm2niix-path \path\to\dcm2niix.exe - ------------------------------------------------------------------------------------------------------------------------- diff --git a/docs/matlab.rst b/docs/matlab.rst index fe5cb054..42d6786d 100644 --- a/docs/matlab.rst +++ b/docs/matlab.rst @@ -9,7 +9,7 @@ matlab .. mat:autofunction:: dcm2niix4pet -.. mat:autofunction:: updatjsonpetfile +.. mat:autofunction:: updatejsonpetfile .. mat:autofunction:: ecat2nii diff --git a/docs/modules.rst b/docs/modules.rst index 10bca7d0..b1b588c8 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -1,7 +1,8 @@ -pypet2bids -========== +Code +==== .. toctree:: :maxdepth: 4 pypet2bids + matlab \ No newline at end of file diff --git a/docs/pypet2bids.rst b/docs/pypet2bids.rst index 483e465a..be5de10b 100644 --- a/docs/pypet2bids.rst +++ b/docs/pypet2bids.rst @@ -1,10 +1,7 @@ .. _pypet2bids-package: -pypet2bids package -================== - -Submodules ----------- +pypet2bids +========== pypet2bids.ecat\_cli module --------------------------- @@ -79,6 +76,14 @@ pypet2bids.multiple\_spreadsheets module :undoc-members: :show-inheritance: +pypet2bids.update_pet_json module +--------------------------------- + +.. automodule:: pypet2bids.update_json_pet_file + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/spreadsheets.rst b/docs/spreadsheets.rst index 76cabf0d..ac985b5d 100644 --- a/docs/spreadsheets.rst +++ b/docs/spreadsheets.rst @@ -83,7 +83,7 @@ An example of a bids compliant auto-sampled tsv can be seen `here `_ These types of recordings are stored along side manually sampled blood data within the BIDS tree. In order to -differentiate between auto-sampled and manually sampled files the `recording <>'_ entity is used. e.g. this file would +differentiate between auto-sampled and manually sampled files the *recording* entity is used. e.g. this file would be saved as: .. code-block:: diff --git a/docs/usage.rst b/docs/usage.rst index 3d86f434..7728a723 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -10,7 +10,7 @@ Matlab BIDS requires nifti files and json. While json can be writen be hand, this is more convenient to populate them as one reads data. One issue is that some information is not encoded in ecat/dicom headers and thus needs to be created -overwise. +otherwise. @@ -27,7 +27,7 @@ The simplest way to convert DICOM files is to call `dcm2niix4pet.m `_ which wraps around dcm2niix. Assuming dcm2niix is present in your environment, Matlab will call it to convert your data to nifti and json - and the wrapper function will additionally edit the json file. Arguments in are the dcm folder(s) in, the metadata -as a structure (using the get_pet_metadata.m function for instance) and possibly options as per dcm2nixx. +as a structure (using the get_pet_metadata.m function for instance) and possibly options as per dcm2niix. *Note for windows user*: edit the dcm2niix4pet.m line 51 to indicate where is the .exe function located @@ -303,3 +303,90 @@ Then point the optional `metadatapath` flag at the spreadsheet location: dcm2niix4pet /folder/containing/PET/dicoms/ --destination /folder/containing/PET/nifti_jsons --metadatapath /file/PET_metadata.xlsx + +Development and Testing +----------------------- + +PET2BIDS makes use of both unit and integration tests to ensure that the code is working as expected. Phantom, +synthetic imaging, and blood test data can be retrieved directly from either this repository or via the url contained +within the **collectphantoms** make command in the top level Makefile. Tests are written with relative paths to the test +data contained in several directories within this repository: + +1) metadata/ +2) ecat_validation/ +3) tests/ +4) OpenNeuroPET-Phantoms/ (this last directory needs be downloaded separately with the make commands `collectphantoms` and `unzipphantoms`) + + +The unit tests for Python can be found in the `tests` folder and are run using the `pytest` library. Python tests are +best run with `poetry run`: + +.. code-block:: + + poetry run pytest test/.py + + +The matlab unit tests can be found in the `matlab/unit_tests` folder and are run in Matlab after adding this repository +and its contents to you Matlab path. + +.. code-block:: + + addpath('path/to/PET2BIDS/matlab') + addpath('path/to/PET2BIDS/matlab/unit_tests') + +Integration tests are run using the `make testphantoms` command. However, running the Matlab integration tests requires +the user add the script located in `OpenNeuroPET-Phantoms/code/matlab_conversions.m` to their Matlab path. + +To emulate the CI/CD pipeline, the following commands can be run locally in the following order: + +.. code-block:: + + make testphantoms + make testpython + +Documentation for Read the Docs can be built and previewed locally using the following command: + +.. code-block:: + + make html + +The output of the build can be found at docs/_build/html/ and previewed by opening any of the html files contained +therein with a web browser. Chances are good if you're able to build with no errors locally than any changes you make +to the documentation rst files will be built successfully on Read the Docs, but not guaranteed. + + +Working with existing NiFTi and JSON files +------------------------------------------ + +PET2BIDS is primarily designed to work with DICOM and ECAT files, but it can also be used to add or update sidecar json +metadata after conversion to NiFTi (whether by PET2BIDS or some other tool). This can be done using the +`updatejsonpetfile.m` function in Matlab or the `updatepetjson` command line tool in Python. Additionally, if one has +dicom or ecat files available to them they can use `updatepetjsonfromdicom` or `updatepetjsonfromecat` respectively +instead of `updatepetjson` or `updatejsonpetfile.m`. + + +*Disclaimer: +Again, we strongly encourage users to use PET2BIDS or dcm2niix to convert their data from dicom or ecat into nifti. +We have observed that nifti's not produced by dcm2niix are often missing dicom metadata such as frame timing or image +orientation. Additionally some non-dcm2niix nifti's contain extra singleton dimensions or extended but undefined main +image headers.* + + +Examples of using the command line tools can be seen below: + + +.. code-block:: + + updatepetjson /path/to/nifti.nii /path/to/json.json --kwargs TimeZero=12:12:12 + updatepetjsonfromdicom /path/to/dicom.dcm /path/to/json.json --kwargs TimeZero=12:12:12 + updatepetjsonfromecat /path/to/ecat.v /path/to/json.json --kwargs TimeZero=12:12:12 + + +.. code-block:: + + # run matlab to get this output + >> jsonfilename = fullfile(pwd,'DBS_Gris_13_FullCT_DBS_Az_2mm_PRR_AC_Images_20151109090448_48.json'); + >> metadata = get_SiemensBiograph_metadata('TimeZero','ScanStart','tracer','AZ10416936','Radionuclide','C11', + ... 'ModeOfAdministration','bolus','Radioactivity', 605.3220,'InjectedMass', 1.5934,'MolarActivity', 107.66); + >> dcminfo = dicominfo('DBSGRIS13.PT.PETMR_NRU.48.13.2015.11.11.14.03.16.226.61519201.dcm'); + >> status = updatejsonpetfile(jsonfilename,metadata,dcminfo); \ No newline at end of file diff --git a/ecat_testing/ints/README.md b/ecat_testing/ints/README.md new file mode 100644 index 00000000..acdf0891 --- /dev/null +++ b/ecat_testing/ints/README.md @@ -0,0 +1,60 @@ +# matlab_numpy_ints + +This is a very simple example of testing matlab vs python reading and writing of ecat files scaling up to a more +complex set of testing that reads real ecat data and writes it to niftis. + +## Dead Simple Testing / Proofing + +Then simplest test cases that use "imaging" data made of small 2 x 1 arrays of integer data can found in: + +`ints.m` and `ints.py` + +## Outline of Testing on Real Data (e.g. steps required to read and write ecat into Nifti) + +A part of this debugging process is to capture the state of inputs and outputs through each iteration of the python +and matlab code. These outputs are cataloged and saved in this directory (`ecat_testing/ints/`) within subdirectory +`steps/`. The naming, order, and description of each step are laid out in the table below for easy reference. At each +of these steps the outputs are saved in the `steps/` directory with the filestem (and the python or matlab code used to +make them) as described. These temporary files are written if and only if the environment variable `ECAT_SAVE_STEPS=1` +is set. + +| Step | Description | Matlab | Python | FileStem | +|------|-------------------------------------------------------|---------------|----------------|------------------------------| +| 1 | Read main header | `read_ECAT7.m` | `read_ecat.py` | `1_read_mh_ecat*` | +| 2 | Read subheaders | `read_EACT7.m` | `read_ecat.py` | `2_read_sh_ecat*` | +| 3 | Determine file/data type | `read_ECAT7.m` | `read_ecat.py` | `3_determine_data_type*` | +| 4 | Read image data | `read_ECAT7.m` | `read_ecat.py` | `4_read_img_ecat*` | +| 5 | scale if calibrated | `read_ECAT7.m` | `read_ecat.py` | `5_scale_img_ecat*` | +| 6 | Pass Data to ecat2nii | `ecat2nii.m` | `ecat2nii.py` | `6_ecat2nii*` | +| 7 | Flip ECAT data into Nifti space | `ecat2nii.m` | `ecat2nii.py` | `7_flip_ecat2nii*` | +| 8 | Rescale to 16 bits | `ecat2nii.m` | `ecat2nii.py` | `8_rescale_to_16_ecat2nii*` | +| 9 | Calibration Units Scaling | `ecat2nii.m` | `ecat2nii.py` | `9_scal_cal_units_ecat2nii*` | +| 10 | Save to Nifti | `ecat2nii.m` | `ecat2nii.py` | `10_save_nii_ecat2nii*` | +| 11 | Check to see if values are recorded as they should be | `ecat2nii.m` | `ecat2nii.py` | `11_read_saved_nii*` | + + +1. Read main header: this is the first step in reading the ecat file, the main header contains information about the + file, the subheaders, and the image data. These will be saved as jsons for comparison. +2. Read subheaders: the subheaders contain information about the image data, these will be saved as jsons for comparison + as well. +3. Determine file/data type: this step is to determine the type of data in the image data, this will be saved a json or + a text file with a single line specifying the datatype and endianness, but probably a json. +4. Read image data: this is the final step in reading the ecat file, the image data is read in and will be examined at + 3 different time points (if available). E.g. the first frame, the middle frame, and the final frame. Only a single 2D + slice will be saved from each of the time points, and it too will be taken from the "middle" of its 3D volume. We're + only attempting to compare whether python and matlab have done a decent job of reading the data as recorded in. +5. Repeat step 4 but scale the data if it should be scaled +6. Save objects for comparison (as best as one can) before they are passed to the ecat2nii function. This will include + the mainheader, subheaders, and image data. +7. Return the transformed data to the nifti space from ecat. This follows the 3 flip dimension steps performed across + the 3D image data. This output will use the same frames as step 4 and 5. +8. Rescale the data to 16 bits: this should only occur if the data is 12bit as is sometimes the case with ecat data. As + a note to self, attempting these steps in Python will start to lead to wildly different values when compared to + matlab. It's most likely not necessary to do this step as the data is handled in numpy, but this writer won't promise + to eat his hat if it turns out to be necessary. +9. Calibration Units: (Search for 'calibration_units == 1' to locate this in code) Here we can potentially alter the + data again by scaling, rounding, and converting from int to float. As in steps 4 and 5 we will save the first, + middle, and last frames of the data as 2D slices in the middle of their respective 3D volumes for comparison. +10. Save to nifti: the data is saved to a nifti file and the output is saved for comparison as .nii files named in line + with the FileStem column above. +11. Additional Steps: TBD diff --git a/ecat_testing/ints/check_ecat_read.py b/ecat_testing/ints/check_ecat_read.py new file mode 100644 index 00000000..bceb9924 --- /dev/null +++ b/ecat_testing/ints/check_ecat_read.py @@ -0,0 +1,14 @@ +import numpy +import nibabel +import dotenv +import os + + + +# load the environment variables +dotenv.load_dotenv('variables.env') + +# get the path to the input ecat file from the environment variables +ecat_file = os.getenv('WELL_BEHAVED_ECAT_FILE') +wustl_fbp_ecat=os.getenv('WUSTL_FBP_ECAT') +wustl_osem_ecat=os.getenv('WUSTL_OSEM_ECAT') \ No newline at end of file diff --git a/ecat_testing/ints/ints.m b/ecat_testing/ints/ints.m new file mode 100644 index 00000000..0f15b1f9 --- /dev/null +++ b/ecat_testing/ints/ints.m @@ -0,0 +1,55 @@ +env = loadenv('variables.env'); +ecat_file=env('WELL_BEHAVED_ECAT_FILE'); +wustl_fbp_ecat=env('WUSTL_FBP_ECAT'); +wustl_osem_ecat=env('WUSTL_OSEM_ECAT'); + +% convert the following ints to bytes and write them to a file +ints_to_write = [-32768, 32767] + + +% use the same format as the python version of this code +destination_file = 'matlab_sample_bytes.bytes' + +% write all the ints as bytes to the file +fid = fopen(destination_file, 'w'); +disp("writing ints_to_write to sample file"); +disp(ints_to_write); +fwrite(fid, ints_to_write, 'int16', 'b'); +%for i = 1:length(ints_to_write) +% fwrite(fid, typecast(ints_to_write(i), 'int16'), 'int16', 'b'); +%end +fclose(fid); + +% read out the bytes as written to the sample file +fid = fopen('matlab_sample_bytes.bytes', 'r'); +bytes = fread(fid); +disp("bytes written from matlab in matlab_sample_bytes.byte and read with fread (no arguments)"); +disp(bytes); +fclose(fid); + +% read in bytes from python, at least I have a good idea of what that is written as +fid = fopen('python_sample_bytes.bytes', 'r'); +python_bytes = fread(fid); +disp("bytes written from python in python_sample_bytes.byte and read with fread (no arguments)"); +disp(python_bytes); + +various_data_types = {'int16=>int16', 'int16', 'uint16'}; + +disp("now we open the matlab file and read the bytes using multiple arguments for fread"); + +% read in the bytes as int16 + +calibration_factor = 0.7203709074867248; + +for t = 1:length(various_data_types) + % oddly enough varying the second argument to fread doesn't seem to change the output + fid = fopen(destination_file, 'r', 'ieee-be'); + various_bytes = fread(fid, various_data_types{t}); + disp(['datatype used for reading in bytes: ', various_data_types{t}]); + disp(various_bytes); + disp(various_data_types{t}); + fclose(fid); + + % scale the data to see what happens + scaled_bytes = calibration_factor * various_bytes +end diff --git a/ecat_testing/ints/ints.py b/ecat_testing/ints/ints.py new file mode 100644 index 00000000..15b419fa --- /dev/null +++ b/ecat_testing/ints/ints.py @@ -0,0 +1,97 @@ +import numpy +import nibabel +import dotenv +import os + +# load the environment variables +dotenv.load_dotenv('variables.env') + +# get the path to the input ecat file from the environment variables +ecat_file = os.getenv('WELL_BEHAVED_ECAT_FILE') +wustl_fbp_ecat=os.getenv('WUSTL_FBP_ECAT') +wustl_osem_ecat=os.getenv('WUSTL_OSEM_ECAT') + + +def create_sample_bytes(destination_file='python_sample_bytes.bytes'): + """ + Create a file with two 16-bit integers, one positive and one negative. + + :param destination_file: _description_, defaults to 'sample_bytes.bytes' + :type destination_file: str, optional + """ + bytes_to_write = [-32768, 32767] + with open(destination_file, 'wb') as file: + byte_order = 'big' + signed = True + for b in bytes_to_write: + print(f"Writing {b} to {destination_file}, byte order: {byte_order}, signed: {signed}") + file.write(b.to_bytes(2, byteorder=byte_order, signed=True)) + + +def read_bytes(file_name='python_sample_bytes.bytes'): + """ + Open a byte file and read in the bytes without any extra fluff. + + :param file_name: the file to read bytes from, defaults to 'sample_bytes.bytes' + :type file_name: str, optional + :return: the bytes read from the file + :rtype: bytes + """ + with open(file_name, 'rb') as file: + bytes_read = file.read() + print(f"Read these bytes from {file_name}: {bytes_read}") + return bytes_read + + +# create the bytes to be read +create_sample_bytes() +# read the bytes +read_nums = read_bytes() + +# these are some of the datatypes we wish to test with numpy +# i2 = integer 16 signed, >i2 = integer 16 big endian signed, H = integer 16 unsigned +dtypes = ['i2', '>i2', 'H'] + +# create a dictionary to hold the arrays we create with numpy +arrays_by_dtype = {} + +# iterate through the datatypes and create arrays with numpy +for d in dtypes: + numpy_int16 = numpy.frombuffer(read_nums, dtype=d) + print(f"Reading bytes with numpy given datatype: {d}\nArray is listing this as : {numpy_int16.dtype} {numpy_int16}") + arrays_by_dtype[d] = numpy_int16 + +print(f"Arrays by dtype: {arrays_by_dtype}") + +# next we go through the same steps that the ecat converter does but in miniature since a 1x2 array is easier for us +# hairless apes to comprehend than a near as enough to make no difference N dimensional array + +# scale it the calibration factor we have been dealing with in these wustl ecats is 0.7203709074867248 +calibration_factor = 0.7203709074867248 + +scaled_arrays = {} + +for k, v in arrays_by_dtype.items(): + # try recasting the scaled array after the multiplication + scaled_arrays[k] = v * calibration_factor + +print(f"These are the arrays after being scaled by {calibration_factor}: {scaled_arrays}") + +# write these out to nifti's +for k, v in scaled_arrays.items(): + nifti = nibabel.Nifti1Image(v, affine=numpy.eye(4)) + input_data_type = v.dtype + nibabel_data_type = nifti.get_data_dtype() + print(f"Input data type: {input_data_type}, Nibabel data type: {nibabel_data_type}") + nibabel.save(nifti, f"nibabel_{k}.nii.gz") + print(f"Saved array to numpy_{k}.nii.gz") + print(f"loading that array results in the following: {nibabel.load(f'nibabel_{k}.nii.gz').get_fdata()}") + +# what happens if we don't scale the arrays and write them to nifti +for k, v in arrays_by_dtype.items(): + input_data_type = v.dtype + nibabel_data_type = nifti.get_data_dtype() + print(f"Input data type: {input_data_type}, Nibabel data type: {nibabel_data_type}") + nibabel.save(nifti, f"nibabel_{k}_unscaled.nii.gz") + print(f"Saved array to numpy_{k}_unscaled.nii.gz") + print(f"loading that array results in the following: {nibabel.load(f'nibabel_{k}_unscaled.nii.gz').get_fdata()}") diff --git a/ecat_testing/ints/matlab_sample_bytes.bytes b/ecat_testing/ints/matlab_sample_bytes.bytes new file mode 100644 index 00000000..68756cd4 Binary files /dev/null and b/ecat_testing/ints/matlab_sample_bytes.bytes differ diff --git a/ecat_testing/ints/nii_tool.m b/ecat_testing/ints/nii_tool.m new file mode 100755 index 00000000..c0b4540e --- /dev/null +++ b/ecat_testing/ints/nii_tool.m @@ -0,0 +1,1249 @@ +function varargout = nii_tool(cmd, varargin) +% +% Copyright (c) 2016, Xiangrui Li https://github.com/xiangruili/dicm2nii +% BSD-2-Clause License :: commit Mar 8, 2021 +% Basic function to create, load and save NIfTI file. +% +% :format: - rst = nii_tool('cmd', para); +% +% % .. note:: To list all command, type 'nii_tool ?' +% +% To get help information for each command, include '?' in cmd, for example: +% nii_tool init? +% nii_tool('init?') +% +% Here is a list of all command: +% +% nii_tool('default', 'version', 1, 'rgb_dim', 1); +% nii = nii_tool('init', img); +% nii = nii_tool('update', nii, mat); +% nii_tool('save', nii, filename, force_3D); +% hdr = nii_tool('hdr', filename); +% img = nii_tool('img', filename_or_hdr); +% ext = nii_tool('ext', filename_or_hdr); +% nii = nii_tool('load', filename_or_hdr); +% nii = nii_tool('cat3D', filenames); +% nii_tool('RGBStyle', 'afni'); +% +% Detail for each command is described below. +% +% oldVal = nii_tool('default', 'version', 1, 'rgb_dim', 1); +% oldVal = nii_tool('default', struct('version', 1, 'rgb_dim', 1)); +% +% - Set/query default NIfTI version and/or rgb_dim. To check the setting, run +% nii_tool('default') without other input. The input for 'default' command can +% be either a struct with fields of 'version' and/or 'rgb_dim', or +% parameter/value pairs. See nii_tool('RGBstyle') for meaning of rgb_dim. +% +% Note that the setting will be saved for future use. If one wants to change the +% settting temporarily, it is better to return the oldVal, and to restore it +% after done: +% +% oldVal = nii_tool('default', 'version', 2); % set version 2 as default +% % 'init' and 'save' NIfTI using above version +% nii_tool('default', 'version', oldVal); % restore default setting +% +% The default version setting affects 'init' command only. If you 'load' a NIfTI +% file, modify it, and then 'save' it, the version will be the same as the +% original file, unless it is changed explicitly (see help for 'save' command). +% All 'load' command ('load', 'hdr', 'ext', 'img') will read any version +% correctly, regardless of version setting. +% +% nii = nii_tool('init', img, RGB_dim); +% +% - Initialize nii struct based on img, normally 3D or 4D array. Most fields in +% the returned nii.hdr contain default values, and need to be updated based on +% dicom or other information. Important ones include pixdim, s/qform_code and +% related parameters. +% +% The NIfTI datatype will depend on data type of img. Most Matlab data types are +% supported, including 8/16/32/64 bit signed and unsigned integers, single and +% double floating numbers. Single/double complex and logical array are also +% supported. +% +% The optional third input, RGB_dim, is needed only if img contains RGB/RGBA +% data. It specifies which dimension in img encodes RGB or RGBA. In other words, +% if a non-empty RGB_dim is provided, img will be interpreted as RGB/RGBA data. +% +% Another way to signify RGB/RGBA data is to permute color dim to 8th-dim of img +% (RGB_dim of 8 can be omitted then). Since NIfTI img can have up to 7 dim, +% nii_tool chooses to store RGB/RGBA in 8th dim. Although this looks lengthy +% (4th to 7th dim are often all ones), nii_tool can deal with up to 7 dim +% without causing any confusion. This is why the returned nii.img always stores +% RGB in 8th dim. +% +% +% nii = nii_tool('update', nii, mat); +% +% - Update nii.hdr according to nii.img. This is useful if one changes nii.img +% type or dimension. The 'save' command calls this internally, so it is not +% necessary to call this before 'save'. A useful case to call 'update' is that +% one likes to use nii struct without saving it to a file, and 'update' will +% make nii.hdr.dim and others correct. +% +% If the 3rd input, new transformation matrix is provided, it will be set as the +% sform transformation matrix. +% +% +% hdr = nii_tool('hdr', filename); +% +% - Return hdr struct of the provided NIfTI file. This is useful to check NIfTI +% hdr, and it is much faster than 'load', especially for .gz file. +% +% +% img = nii_tool('img', filename_or_hdr); +% +% - Return image data in a NIfTI file. The second input can be NIfTI file name, +% or hdr struct returned by nii_tool('hdr', filename). +% +% +% ext = nii_tool('ext', filename_or_hdr); +% +% - Return NIfTI extension in a NIfTI file. The second input can be NIfTI file +% name, or hdr struct returned by nii_tool('hdr', filename). The returned ext +% will have field 'edata_decoded' if 'ecode' is of known type, such as dicom +% (2), text (4 or 6) or Matlab (40). +% +% Here is an example to add data in myFile.mat as extension to nii struct, which +% can be from 'init' or 'load': +% +% fid = fopen('myFile.mat'); % open the MAT file +% myEdata = fread(fid, inf, '*uint8'); % load all bytes as byte column +% fclose(fid); +% len = int32(numel(myEdata)); % number of bytes in int32 +% myEdata = [typecast(len, 'uint8')'; myEdata]; % include len in myEdata +% nii.ext.ecode = 40; % 40 for Matlab extension +% nii.ext.edata = myEdata; % myEdata must be uint8 array +% +% nii_tool will take care of rest when you 'save' nii to a file. +% +% In case a NIfTI ext causes problem (for example, some FSL builds have problem +% in reading NIfTI img with ecode>30), one can remove the ext easily: +% +% nii = nii_tool('load', 'file_with_ext.nii'); % load the file with ext +% nii.ext = []; % or nii = rmfield(nii, 'ext'); % remove ext +% nii_tool('save', nii, 'file_without_ext.nii'); % save it +% +% +% nii = nii_tool('load', filename_or_hdr); +% +% - Load NIfTI file into nii struct. The returned struct includes NIfTI 'hdr' +% and 'img', as well as 'ext' if the file contains NIfTI extension. +% +% nii_tool returns nii.img with the same data type as stored in the file, while +% numeric values in hdr are in double precision for convenience. +% +% +% nii_tool('save', nii, filename, force_3D); +% +% - Save struct nii into filename. The format of the file is determined by the +% file extension, such as .img, .nii, .img.gz, .nii.gz etc. If filename is not +% provided, nii.hdr.file_name must contain a file name. Note that 'save' command +% always overwrites file in case of name conflict. +% +% If filename has no extension, '.nii' will be used as default. +% +% If the 4th input, force_3D, is true (default false), the output file will be +% 3D only, which means multiple volume data will be split into multiple files. +% This is the format SPM likes. You can use this command to convert 4D into 3D +% by 'load' a 4D file, then 'save' it as 3D files. The 3D file names will have +% 5-digit like '_00001' appended to indicate volume index. +% +% The NIfTI version can be set by nii_tool('default'). One can override the +% default version by specifying it in nii.hdr.version. To convert between +% versions, load a NIfTI file, specify new version, and save it. For example: +% +% nii = nii_tool('load', 'file_nifti1.nii'); % load version 1 file +% nii.hdr.version = 2; % force to NIfTI-2 +% nii_tool('save', nii, 'file_nifti2.nii'); % save as version 2 file +% +% Following example shows how to change data type of a nii file: +% nii = nii_tool('load', 'file_int16.nii'); % load int16 type file +% nii.img = single(nii.img); % change data type to single/float32 +% nii_tool('save', nii, 'file_float.nii'); % nii_tool will take care of hdr +% +% +% nii = nii_tool('cat3D', files); +% +% - Concatenate SPM 3D files into a 4D dataset. The input 'files' can be cellstr +% with file names, or char with wildcards (* or ?). If it is cellstr, the volume +% order in the 4D data corresponds to those files. If wildcards are used, the +% volume order is based on alphabetical order of file names. +% +% Note that the files to be concatenated must have the same datatype, dim, voxel +% size, scaling slope and intercept, transformation matrix, etc. This is +% normally true if files are for the same dicom series. +% +% Following example shows how to convert a series of 3D files into a 4D file: +% +% nii = nii_tool('cat3D', './data/fSubj2-0003*.nii'); % load files for series 3 +% nii_tool('save', nii, './data/fSubj2-0003_4D.nii'); % save as a 4D file +% +% +% oldStyle = nii_tool('RGBStyle', 'afni'); +% +% - Set/query the method to read/save RGB or RGBA NIfTI file. The default method +% can be set by nii_tool('default', 'rgb_dim', dimN), where dimN can be 1, 3 or +% 4, or 'afni', 'mricron' or 'fsl', as explained below. +% +% The default is 'afni' style (or 1), which is defined by NIfTI standard, but is +% not well supported by fslview till v5.0.8 or mricron till v20140804. +% +% If the second input is set to 'mricron' (or 3), nii_tool will save file using +% the old RGB fashion (dim 3 for RGB). This works for mricron v20140804 or +% earlier. +% +% If the second input is set to 'fsl' (or 4), nii_tool will save RGB or RGBA +% layer into 4th dimension, and the file is not encoded as RGB data, but as +% normal 4D NIfTI. This violates the NIfTI rule, but it seems it is the only way +% to work for fslview (at least till fsl v5.0.8). +% +% If no new style (second input) is provided, it means to query the current +% style (one of 'afni', 'mricron' and 'fsl'). +% +% The GUI method to convert between different RGB style can be found in +% nii_viewer. Following shows how to convert other style into fsl style: +% +% nii_tool('RGBStyle', 'afni'); % we are loading afni style RGB +% nii = nii_tool('load', 'afni_style.nii'); % load RGB file +% nii_tool('RGBStyle', 'fsl'); % switch to fsl style for later save +% nii_tool('save', nii, 'fslRGB.nii'); % fsl can read it as RGB +% +% Note that, if one wants to convert fsl style (non-RGB file by NIfTI standard) +% to other styles, an extra step is needed to change the RGB dim from 4th to 8th +% dim before 'save': +% +% nii = nii_tool('load', 'fslStyleFile.nii'); % it is normal NIfTI +% nii.img = permute(nii.img, [1:3 5:8 4]); % force it to be RGB data +% nii_tool('RGBStyle', 'afni'); % switch to NIfTI RGB style if needed +% nii_tool('save', nii, 'afni_RGB.nii'); % now AFNI can read it as RGB +% +% Also note that the setting by nii_tool('RGBStyle') is effective only for +% current Matlab session. If one clears all or starts a new Matlab session, the +% default style by nii_tool('default') will take effect. +% +% See also NII_VIEWER, NII_XFORM, DICM2NII + +% More information for NIfTI format: +% Official NIfTI website: http://nifti.nimh.nih.gov/ +% Another excellent site: http://brainder.org/2012/09/23/the-nifti-file-format/ + +% History (yymmdd) +% 150109 Write it based on Jimmy Shen's NIfTI tool (xiangrui.li@gmail.com) +% 150202 Include renamed pigz files for Windows +% 150203 Fix closeFile and deleteTmpFile order +% 150205 Add hdr.machine: needed for .img fopen +% 150208 Add 4th input for 'save', allowing to save SPM 3D files +% 150210 Add 'cat3D' to load SPM 3D files +% 150226 Assign all 8 char for 'magic' (version 2 needs it) +% 150321 swapbytes(nByte) for ecode=40 with big endian +% 150401 Add 'default' to set/query version and rgb_dim default setting +% 150514 read_ext: decode txt edata by dicm2nii.m +% 150517 func_handle: provide a way to use gunzipOS etc from outside +% 150617 auto detect rgb_dim 1&3 for 'load' etc using ChrisR method +% 151025 Change subfunc img2datatype as 'update' for outside access +% 151109 Include dd.exe from WinAVR-20100110 for partial gz unzip +% 151205 Partial gunzip: fix fname with space & unknown pigz | dd error. +% 151222 Take care of img for intent_code 2003/2004: anyone uses it? +% 160110 Use matlab pref method to replace para file. +% 160120 check_gzip: use "" for included pigz; ignore dd error if err is false. +% 160326 fix setpref for older Octave: set each parameter separately. +% 160531 fopen uses 'W' for 'w': performance benefit according to Yair. +% 160701 subFuncHelp: bug fix for mfile case. +% 161018 gunzipOS: use unique name for outName, to avoid problem with parfor. +% 161025 Make included linux pigz executible; fix "dd" for windows. +% 161031 gunzip_mem(), nii_bytes() for hdr/ext read: read uint8 then parse; +% Replace hdr.machine with hdr.swap_endian. +% 170212 Extract decode_ext() from 'ext' cmd so call it in 'update' cmd. +% 170215 gunzipOS: use -c > rather than copyfile for performance. +% 170322 gzipOS: stop using background gz to avoid file not exist error. +% 170410 read_img(): turn off auto RGB dim detection, and use rgb_dim. +% 170714 'save': force to version 2 if img dim exceeds 2^15-1. +% 170716 Add functionSignatures.json file for tab auto-completion. +% 171031 'LocalFunc' makes eaiser to call local functions. +% 171206 Allow file name ext other than .nii, .hdr, .img. +% 180104 check_gzip: add /usr/local/bin to PATH for unix if needed. +% 180119 use jsystem for better speed. +% 180710 bug fix for cal_max/cal_min in 'update'. +% 210302 take care of unicode char in hdr (Thx Yong). + +persistent C para; % C columns: name, length, format, value, offset +if isempty(C) + [C, para] = niiHeader; + if exist('OCTAVE_VERSION', 'builtin') + warning('off', 'Octave:fopen-mode'); % avoid 'W' warning + more off; + end +end + +if ~ischar(cmd) + error('Provide a string command as the first input for nii_tool'); +end +if any(cmd=='?'), subFuncHelp(mfilename, cmd); return; end + +if strcmpi(cmd, 'init') + if nargin<2, error('nii_tool(''%s'') needs second input', cmd); end + nii.hdr = cell2struct(C(:,4), C(:,1)); + nii.img = varargin{1}; + if numel(size(nii.img))>8 + error('NIfTI img can have up to 7 dimension'); + end + if nargin>2 + i = varargin{2}; + if i<0 || i>8 || mod(i,1)>0, error('Invalid RGB_dim number'); end + nii.img = permute(nii.img, [1:i-1 i+1:8 i]); % RGB to dim8 + end + nii.hdr.file_name = inputname(2); + if isempty(nii.hdr.file_name), nii.hdr.file_name = 'tmpNII'; end + varargout{1} = nii_tool('update', nii); % set datatype etc + +elseif strcmpi(cmd, 'save') + if nargin<2, error('nii_tool(''%s'') needs second input', cmd); end + nii = varargin{1}; + if ~isstruct(nii) || ~isfield(nii, 'hdr') || ~isfield(nii, 'img') + error(['nii_tool(''save'') needs a struct from nii_tool(''init'')' ... + ' or nii_tool(''load'') as the second input']); + end + + % Check file name to save + if nargin>2 + fname = varargin{2}; + if ~ischar(fname), error('Invalid name for NIfTI file: %s', fname); end + elseif isfield(nii.hdr, 'file_name') + fname = nii.hdr.file_name; + else + error('Provide a valid file name as the third input'); + end + if ~ispc && strncmp(fname, '~/', 2) % matlab may err with this abbrevation + fname = [getenv('HOME') fname(2:end)]; + end + [pth, fname, fext] = fileparts(fname); + do_gzip = strcmpi(fext, '.gz'); + if do_gzip + [~, fname, fext] = fileparts(fname); % get .nii .img .hdr + end + if isempty(fext), fext = '.nii'; end % default single .nii file + fname = fullfile(pth, fname); % without file ext + if nargout, varargout{1} = []; end + isNii = strcmpi(fext, '.nii'); % will use .img/.hdr if not .nii + + % Deal with NIfTI version and sizeof_hdr + niiVer = para.version; + if isfield(nii.hdr, 'version'), niiVer = nii.hdr.version; end + if niiVer<2 && any(nii.hdr.dim(2:end) > 32767), niiVer = 2; end + + if niiVer == 1 + nii.hdr.sizeof_hdr = 348; % in case it was loaded from other version + elseif niiVer == 2 + nii.hdr.sizeof_hdr = 540; % version 2 + else + error('Unsupported NIfTI version: %g', niiVer); + end + + if niiVer ~= para.version + C0 = niiHeader(niiVer); + else + C0 = C; + end + + % Update datatype/bitpix/dim in case nii.img is changed + [nii, fmt] = nii_tool('update', nii); + + % This 'if' block: lazy implementation to split into 3D SPM files + if nargin>3 && ~isempty(varargin{3}) && varargin{3} && nii.hdr.dim(5)>1 + if do_gzip, fext = [fext '.gz']; end + nii0 = nii; + for i = 1:nii.hdr.dim(5) + fname0 = sprintf('%s_%05g%s', fname, i, fext); + nii0.img = nii.img(:,:,:,i,:,:,:,:); % one vol + if i==1 && isfield(nii, 'ext'), nii0.ext = nii.ext; + elseif i==2 && isfield(nii0, 'ext'), nii0 = rmfield(nii0, 'ext'); + end + nii_tool('save', nii0, fname0); + end + return; + end + + % re-arrange img for special datatype: RGB/RGBA/Complex. + if any(nii.hdr.datatype == [128 511 2304]) % RGB or RGBA + if para.rgb_dim == 1 % AFNI style + nii.img = permute(nii.img, [8 1:7]); + elseif para.rgb_dim == 3 % old mricron style + nii.img = permute(nii.img, [1 2 8 3:7]); + elseif para.rgb_dim == 4 % for fslview + nii.img = permute(nii.img, [1:3 8 4:7]); % violate nii rule + dim = size(nii.img); + if numel(dim)>6 % dim7 is not 1 + i = find(dim(5:7)==1, 1, 'last') + 4; + nii.img = permute(nii.img, [1:i-1 i+1:8 i]); + end + nii = nii_tool('update', nii); % changed to non-RGB datatype + end + elseif any(nii.hdr.datatype == [32 1792]) % complex single/double + nii.img = [real(nii.img(:))'; imag(nii.img(:))']; + end + + % Check nii extension: update esize to x16 + nExt = 0; esize = 0; + nii.hdr.extension = [0 0 0 0]; % no nii ext + if isfield(nii, 'ext') && isstruct(nii.ext) ... + && isfield(nii.ext(1), 'edata') && ~isempty(nii.ext(1).edata) + nExt = numel(nii.ext); + nii.hdr.extension = [1 0 0 0]; % there is nii ext + for i = 1:nExt + if ~isfield(nii.ext(i), 'ecode') || ~isfield(nii.ext(i), 'edata') + error('NIfTI header ext struct must have ecode and edata'); + end + + n0 = numel(nii.ext(i).edata) + 8; % 8 byte for esize and ecode + n1 = ceil(n0/16) * 16; % esize: multiple of 16 + nii.ext(i).esize = n1; + nii.ext(i).edata(end+(1:n1-n0)) = 0; % pad zeros + esize = esize + n1; + end + end + + % Set magic, vox_offset, and open file for .nii or .hdr + if isNii + % version 1 will take only the first 4 + nii.hdr.magic = sprintf('n+%g%s', niiVer, char([0 13 10 26 10])); + nii.hdr.vox_offset = nii.hdr.sizeof_hdr + 4 + esize; + fid = fopen(strcat(fname, fext), 'W'); + else + nii.hdr.magic = sprintf('ni%g%s', niiVer, char([0 13 10 26 10])); + nii.hdr.vox_offset = 0; + fid = fopen(strcat(fname, '.hdr'), 'W'); + end + + % Write nii hdr + for i = 1:size(C0,1) + if isfield(nii.hdr, C0{i,1}) + val = nii.hdr.(C0{i,1}); + else % niiVer=2 omit some fields, also take care of other cases + val = C0{i,4}; + end + fmt0 = C0{i,3}; + if strcmp(C0{i,3}, 'char') + if ~ischar(val), val = char(val); end % avoid val=[] error etc + val = unicode2native(val); % may have more bytes than numel(val) + fmt0 = 'uint8'; + end + n = numel(val); + len = C0{i,2}; + if n>len + val(len+1:n) = []; % remove extra, normally for char + elseif n0, nByte = hdr.vox_offset + 64; % .nii arbituary +64 + else, nByte = inf; + end + b = nii_bytes(fname, nByte); + varargout{1} = read_ext(b, hdr); + end + if nargout>1, varargout{2} = hdr; end + +elseif strcmpi(cmd, 'RGBStyle') + styles = {'afni' '' 'mricron' 'fsl'}; + curStyle = styles{para.rgb_dim}; + if nargin<2, varargout{1} = curStyle; return; end % query only + irgb = varargin{1}; + if isempty(irgb), irgb = 1; end % default as 'afni' + if ischar(irgb) + if strncmpi(irgb, 'fsl', 3), irgb = 4; + elseif strncmpi(irgb, 'mricron', 4), irgb = 3; + else, irgb = 1; + end + end + if ~any(irgb == [1 3 4]) + error('nii_tool(''RGBStyle'') can have 1, 3, or 4 as second input'); + end + if nargout, varargout{1} = curStyle; end % return old one + para.rgb_dim = irgb; % no save to pref + +elseif strcmpi(cmd, 'cat3D') + if nargin<2, error('nii_tool(''%s'') needs second input', cmd); end + fnames = varargin{1}; + if ischar(fnames) % guess it is like run1*.nii + f = dir(fnames); + f = sort({f.name}); + fnames = strcat([fileparts(fnames) '/'], f); + end + + n = numel(fnames); + if n<2 || (~iscellstr(fnames) && (exist('strings', 'builtin') && ~isstring(fnames))) + error('Invalid input for nii_tool(''cat3D''): %s', varargin{1}); + end + + nii = nii_tool('load', fnames{1}); % all for first file + nii.img(:,:,:,2:n) = 0; % pre-allocate + % For now, omit all consistence check between files + for i = 2:n, nii.img(:,:,:,i) = nii_tool('img', fnames{i}); end + varargout{1} = nii_tool('update', nii); % update dim + +elseif strcmpi(cmd, 'default') + flds = {'version' 'rgb_dim'}; % may add more in the future + pf = getpref('nii_tool_para'); + for i = 1:numel(flds), val.(flds{i}) = pf.(flds{i}); end + if nargin<2, varargout{1} = val; return; end % query only + if nargout, varargout{1} = val; end % return old val + in2 = varargin; + if ~isstruct(in2), in2 = struct(in2{:}); end + nam = fieldnames(in2); + for i = 1:numel(nam) + ind = strcmpi(nam{i}, flds); + if isempty(ind), continue; end + para.(flds{ind}) = in2.(nam{i}); + setpref('nii_tool_para', flds{ind}, in2.(nam{i})); + end + if val.version ~= para.version, C = niiHeader(para.version); end + +elseif strcmpi(cmd, 'update') % old img2datatype subfunction + if nargin<2, error('nii_tool(''%s'') needs second input', cmd); end + nii = varargin{1}; + if ~isstruct(nii) || ~isfield(nii, 'hdr') || ~isfield(nii, 'img') + error(['nii_tool(''update'') needs a struct from nii_tool(''init'')' ... + ' or nii_tool(''load'') as the second input']); + end + + dim = size(nii.img); + ndim = numel(dim); + dim(ndim+1:7) = 1; + + if nargin>2 % set new sform mat + R = varargin{2}; + if size(R,2)~=4, error('Invalid matrix dimension.'); end + nii.hdr.srow_x = R(1,:); + nii.hdr.srow_y = R(2,:); + nii.hdr.srow_z = R(3,:); + end + + if ndim == 8 % RGB/RGBA data. Change img type to uint8/single if needed + valpix = dim(8); + if valpix == 4 % RGBA + typ = 'RGBA'; % error info only + nii.img = uint8(nii.img); % NIfTI only support uint8 for RGBA + elseif valpix == 3 % RGB, must be single or uint8 + typ = 'RGB'; + if max(nii.img(:))>1, nii.img = uint8(nii.img); + else, nii.img = single(nii.img); + end + else + error('Color dimension must have length of 3 for RGB or 4 for RGBA'); + end + + dim(8) = []; % remove color-dim so numel(dim)=7 for nii.hdr + ndim = find(dim>1, 1, 'last'); % update it + elseif isreal(nii.img) + typ = 'real'; + valpix = 1; + else + typ = 'complex'; + valpix = 2; + end + + if islogical(nii.img), imgFmt = 'ubit1'; + else, imgFmt = class(nii.img); + end + ind = find(strcmp(para.format, imgFmt) & para.valpix==valpix); + + if isempty(ind) % only RGB and complex can have this problem + error('nii_tool does not support %s image of ''%s'' type', typ, imgFmt); + elseif numel(ind)>1 % unlikely + error('Non-unique datatype found for %s image of ''%s'' type', typ, imgFmt); + end + + fmt = para.format{ind}; + nii.hdr.datatype = para.datatype(ind); + nii.hdr.bitpix = para.bitpix(ind); + nii.hdr.dim = [ndim dim]; + + mx = double(max(nii.img(:))); + mn = double(min(nii.img(:))); + if nii.hdr.cal_min>mx || nii.hdr.cal_max1, varargout{2} = fmt; end +elseif strcmp(cmd, 'func_handle') % make a local function avail to outside + varargout{1} = str2func(varargin{1}); +elseif strcmp(cmd, 'LocalFunc') % call local function from outside + [varargout{1:nargout}] = feval(varargin{:}); +else + error('Invalid command for nii_tool: %s', cmd); +end +% End of main function + +%% Subfunction: all nii header in the order in NIfTI-1/2 file +function [C, para] = niiHeader(niiVer) +pf = getpref('nii_tool_para'); +if isempty(pf) + setpref('nii_tool_para', 'version', 1); + setpref('nii_tool_para', 'rgb_dim', 1); + pf = getpref('nii_tool_para'); +end +if nargin<1 || isempty(niiVer), niiVer = pf.version; end + +if niiVer == 1 + C = { + % name len format value offset + 'sizeof_hdr' 1 'int32' 348 0 + 'data_type' 10 'char' '' 4 + 'db_name' 18 'char' '' 14 + 'extents' 1 'int32' 16384 32 + 'session_error' 1 'int16' 0 36 + 'regular' 1 'char' 'r' 38 + 'dim_info' 1 'uint8' 0 39 + 'dim' 8 'int16' ones(1,8) 40 + 'intent_p1' 1 'single' 0 56 + 'intent_p2' 1 'single' 0 60 + 'intent_p3' 1 'single' 0 64 + 'intent_code' 1 'int16' 0 68 + 'datatype' 1 'int16' 0 70 + 'bitpix' 1 'int16' 0 72 + 'slice_start' 1 'int16' 0 74 + 'pixdim' 8 'single' zeros(1,8) 76 + 'vox_offset' 1 'single' 352 108 + 'scl_slope' 1 'single' 1 112 + 'scl_inter' 1 'single' 0 116 + 'slice_end' 1 'int16' 0 120 + 'slice_code' 1 'uint8' 0 122 + 'xyzt_units' 1 'uint8' 0 123 + 'cal_max' 1 'single' 0 124 + 'cal_min' 1 'single' 0 128 + 'slice_duration' 1 'single' 0 132 + 'toffset' 1 'single' 0 136 + 'glmax' 1 'int32' 0 140 + 'glmin' 1 'int32' 0 144 + 'descrip' 80 'char' '' 148 + 'aux_file' 24 'char' '' 228 + 'qform_code' 1 'int16' 0 252 + 'sform_code' 1 'int16' 0 254 + 'quatern_b' 1 'single' 0 256 + 'quatern_c' 1 'single' 0 260 + 'quatern_d' 1 'single' 0 264 + 'qoffset_x' 1 'single' 0 268 + 'qoffset_y' 1 'single' 0 272 + 'qoffset_z' 1 'single' 0 276 + 'srow_x' 4 'single' [1 0 0 0] 280 + 'srow_y' 4 'single' [0 1 0 0] 296 + 'srow_z' 4 'single' [0 0 1 0] 312 + 'intent_name' 16 'char' '' 328 + 'magic' 4 'char' 'n+1' 344 + 'extension' 4 'uint8' [0 0 0 0] 348 + }; + +elseif niiVer == 2 + C = { + 'sizeof_hdr' 1 'int32' 540 0 + 'magic' 8 'char' 'n+2' 4 + 'datatype' 1 'int16' 0 12 + 'bitpix' 1 'int16' 0 14 + 'dim' 8 'int64' ones(1,8) 16 + 'intent_p1' 1 'double' 0 80 + 'intent_p2' 1 'double' 0 88 + 'intent_p3' 1 'double' 0 96 + 'pixdim' 8 'double' zeros(1,8) 104 + 'vox_offset' 1 'int64' 544 168 + 'scl_slope' 1 'double' 1 176 + 'scl_inter' 1 'double' 0 184 + 'cal_max' 1 'double' 0 192 + 'cal_min' 1 'double' 0 200 + 'slice_duration' 1 'double' 0 208 + 'toffset' 1 'double' 0 216 + 'slice_start' 1 'int64' 0 224 + 'slice_end' 1 'int64' 0 232 + 'descrip' 80 'char' '' 240 + 'aux_file' 24 'char' '' 320 + 'qform_code' 1 'int32' 0 344 + 'sform_code' 1 'int32' 0 348 + 'quatern_b' 1 'double' 0 352 + 'quatern_c' 1 'double' 0 360 + 'quatern_d' 1 'double' 0 368 + 'qoffset_x' 1 'double' 0 376 + 'qoffset_y' 1 'double' 0 384 + 'qoffset_z' 1 'double' 0 392 + 'srow_x' 4 'double' [1 0 0 0] 400 + 'srow_y' 4 'double' [0 1 0 0] 432 + 'srow_z' 4 'double' [0 0 1 0] 464 + 'slice_code' 1 'int32' 0 496 + 'xyzt_units' 1 'int32' 0 500 + 'intent_code' 1 'int32' 0 504 + 'intent_name' 16 'char' '' 508 + 'dim_info' 1 'uint8' 0 524 + 'unused_str' 15 'char' '' 525 + 'extension' 4 'uint8' [0 0 0 0] 540 + }; +else + error('Nifti version %g is not supported', niiVer); +end +if nargout<2, return; end + +% class datatype bitpix valpix +D = { + 'ubit1' 1 1 1 % neither mricron nor fsl support this + 'uint8' 2 8 1 + 'int16' 4 16 1 + 'int32' 8 32 1 + 'single' 16 32 1 + 'single' 32 64 2 % complex + 'double' 64 64 1 + 'uint8' 128 24 3 % RGB + 'int8' 256 8 1 + 'single' 511 96 3 % RGB, not in NIfTI standard? + 'uint16' 512 16 1 + 'uint32' 768 32 1 + 'int64' 1024 64 1 + 'uint64' 1280 64 1 +% 'float128' 1536 128 1 % long double, for 22nd century? + 'double' 1792 128 2 % complex +% 'float128' 2048 256 2 % long double complex + 'uint8' 2304 32 4 % RGBA + }; + +para.format = D(:,1)'; +para.datatype = [D{:,2}]; +para.bitpix = [D{:,3}]; +para.valpix = [D{:,4}]; +para.rgb_dim = pf.rgb_dim; % dim of RGB/RGBA in NIfTI FILE +para.version = niiVer; + +%% Subfunction: use pigz or system gzip if available (faster) +function gzipOS(fname) +persistent cmd; % command to gzip +if isempty(cmd) + cmd = check_gzip('gzip'); + if ischar(cmd) + cmd = @(nam){cmd '-nf' nam}; + elseif islogical(cmd) && ~cmd + fprintf(2, ['None of system pigz, gzip or Matlab gzip available. ' ... + 'Files are not compressed into gz.\n']); + end +end + +if islogical(cmd) + if cmd, gzip(fname); deleteFile(fname); end + return; +end + +[err, str] = jsystem(cmd(fname)); +if err && ~exist(strcat(fname, '.gz'), 'file') + try + gzip(fname); deleteFile(fname); + catch + fprintf(2, 'Error during compression: %s\n', str); + end +end + +%% Deal with pigz/gzip on path or in nii_tool folder, and matlab gzip/gunzip +function cmd = check_gzip(gz_unzip) +m_dir = fileparts(mfilename('fullpath')); +% away from pwd, so use OS pigz if both exist. Avoid error if pwd changed later +if strcmpi(pwd, m_dir), cd ..; clnObj = onCleanup(@() cd(m_dir)); end +if isunix + pth1 = getenv('PATH'); + if isempty(strfind(pth1, '/usr/local/bin')) + pth1 = [pth1 ':/usr/local/bin']; + setenv('PATH', pth1); + end +end + +% first, try system pigz +[err, ~] = jsystem({'pigz' '-V'}); +if ~err, cmd = 'pigz'; return; end + +% next, try pigz included with nii_tool +cmd = [m_dir '/pigz']; +if ismac % pigz for mac is not included in the package + if strcmp(gz_unzip, 'gzip') + fprintf(2, [' Please install pigz for fast compression: ' ... + 'http://macappstore.org/pigz/\n']); + end +elseif isunix % linux + [st, val] = fileattrib(cmd); + if st && ~val.UserExecute, fileattrib(cmd, '+x'); end +end + +[err, ~] = jsystem({cmd '-V'}); +if ~err, return; end + +% Third, try system gzip/gunzip +[err, ~] = jsystem({gz_unzip '-V'}); % gzip/gunzip on system path? +if ~err, cmd = gz_unzip; return; end + +% Lastly, use Matlab gzip/gunzip if java avail +cmd = usejava('jvm'); + +%% check dd command, return empty if not available +function dd = check_dd +m_dir = fileparts(mfilename('fullpath')); +if strcmpi(pwd, m_dir), cd ..; clnObj = onCleanup(@() cd(m_dir)); end +[err, ~] = jsystem({'dd' '--version'}); +if ~err, dd = 'dd'; return; end % dd with linix/mac, and maybe windows + +if ispc % rename it as exe + dd = [m_dir '\dd']; + [err, ~] = jsystem({dd '--version'}); + if ~err, return; end +end +dd = ''; + +%% Try to use in order of pigz, system gunzip, then matlab gunzip +function outName = gunzipOS(fname, nByte) +persistent cmd dd pth uid; % command to run gupzip, dd tool, and temp_path +if isempty(cmd) + cmd = check_gzip('gunzip'); % gzip -dc has problem in PC + if ischar(cmd) + cmd = @(nam)sprintf('"%s" -nfdc "%s" ', cmd, nam); % f for overwrite + elseif islogical(cmd) && ~cmd + cmd = []; + error('None of system pigz, gunzip or Matlab gunzip is available'); + end + dd = check_dd; + if ~isempty(dd) + dd = @(n,out)sprintf('| "%s" count=%g of="%s"', dd, ceil(n/512), out); + end + + if ispc % matlab tempdir could be slow due to cd in and out + pth = getenv('TEMP'); + if isempty(pth), pth = pwd; end + else + pth = getenv('TMP'); + if isempty(pth), pth = getenv('TMPDIR'); end + if isempty(pth), pth = '/tmp'; end % last resort + end + uid = @()sprintf('_%s_%03x', datestr(now, 'yymmddHHMMSSfff'), randi(999)); +end + +fname = char(fname); +if islogical(cmd) + outName = gunzip(fname, pth); + outName = outName{1}; + return; +end + +[~, outName, ext] = fileparts(fname); +if strcmpi(ext, '.gz') % likely always true + [~, outName, ext1] = fileparts(outName); + outName = [outName uid() ext1]; +else + outName = [outName uid()]; +end +outName = fullfile(pth, outName); +if ~isempty(dd) && nargin>1 && ~isinf(nByte) % unzip only part of data + try + [err, ~] = system([cmd(fname) dd(nByte, outName)]); + if err==0, return; end + end +end + +[err, str] = system([cmd(fname) '> "' outName '"']); +% [err, str] = jsystem({'pigz' '-nfdc' fname '>' outName}); +if err + try + outName = gunzip(fname, pth); + catch + error('Error during gunzip:\n%s', str); + end +end + +%% cast bytes into a type, swapbytes if needed +function out = cast_swap(b, typ, swap) +out = typecast(b, typ); +if swap, out = swapbytes(out); end +out = double(out); % for convenience + +%% subfunction: read hdr +function hdr = read_hdr(b, C, fname) +n = typecast(b(1:4), 'int32'); +if n==348, niiVer = 1; swap = false; +elseif n==540, niiVer = 2; swap = false; +else + n = swapbytes(n); + if n==348, niiVer = 1; swap = true; + elseif n==540, niiVer = 2; swap = true; + else, error('Not valid NIfTI file: %s', fname); + end +end + +if niiVer>1, C = niiHeader(niiVer); end % C defaults for version 1 +for i = 1:size(C,1) + try a = b(C{i,5}+1 : C{i+1,5}); + catch + if C{i,5}==numel(b), a = []; + else, a = b(C{i,5} + (1:C{i,2})); % last item extension is in bytes + end + end + if strcmp(C{i,3}, 'char') + a = deblank(native2unicode(a)); + else + a = cast_swap(a, C{i,3}, swap); + a = double(a); + end + hdr.(C{i,1}) = a; +end + +hdr.version = niiVer; % for 'save', unless user asks to change +hdr.swap_endian = swap; +hdr.file_name = fname; + +%% subfunction: read ext, and decode it if known ecode +function ext = read_ext(b, hdr) +ext = []; % avoid error if no ext but hdr.extension(1) was set +nEnd = hdr.vox_offset; +if nEnd == 0, nEnd = numel(b); end % .hdr file + +swap = hdr.swap_endian; +j = hdr.sizeof_hdr + 4; % 4 for hdr.extension +while j < nEnd + esize = cast_swap(b(j+(1:4)), 'int32', swap); j = j+4; % x16 + if isempty(esize) || mod(esize,16), return; end % just to be safe + i = numel(ext) + 1; + ext(i).esize = esize; %#ok<*AGROW> + ext(i).ecode = cast_swap(b(j+(1:4)), 'int32', swap); j = j+4; + ext(i).edata = b(j+(1:esize-8))'; % -8 for esize & ecode + j = j + esize - 8; +end +ext = decode_ext(ext, swap); + +%% subfunction +function ext = decode_ext(ext, swap) +% Decode edata if we know ecode +for i = 1:numel(ext) + if isfield(ext(i), 'edata_decoded'), continue; end % done + if ext(i).ecode == 40 % Matlab: any kind of matlab variable + nByte = cast_swap(ext(i).edata(1:4), 'int32', swap); % MAT data + tmp = [tempname '.mat']; % temp MAT file to save edata + fid1 = fopen(tmp, 'W'); + fwrite(fid1, ext(i).edata(5:nByte+4)); % exclude padded zeros + fclose(fid1); + deleteMat = onCleanup(@() deleteFile(tmp)); % delete temp file after done + ext(i).edata_decoded = load(tmp); % load into struct + elseif any(ext(i).ecode == [4 6 32 44]) % 4 AFNI, 6 plain text, 32 CIfTI, 44 MRS (json) + str = char(ext(i).edata(:)'); + if isempty(strfind(str, 'dicm2nii.m')) + ext(i).edata_decoded = deblank(str); + else % created by dicm2nii.m + ss = struct; + ind = strfind(str, [';' char([0 10])]); % strsplit error in Octave + ind = [-2 ind]; % -2+3=1: start of first para + for k = 1:numel(ind)-1 + a = str(ind(k)+3 : ind(k+1)); + a(a==0) = []; % to be safe. strtrim wont remove null + a = strtrim(a); + if isempty(a), continue; end + try + eval(['ss.' a]); % put all into struct + catch + try + a = regexp(a, '(.*?)\s*=\s*(.*?);', 'tokens', 'once'); + ss.(a{1}) = a{2}; + catch me + fprintf(2, '%s\n', me.message); + fprintf(2, 'Unrecognized text: %s\n', a); + end + end + end + flds = fieldnames(ss); % make all vector column + for k = 1:numel(flds) + val = ss.(flds{k}); + if isnumeric(val) && isrow(val), ss.(flds{k}) = val'; end + end + ext(i).edata_decoded = ss; + end + elseif ext(i).ecode == 2 % dicom + tmp = [tempname '.dcm']; + fid1 = fopen(tmp, 'W'); + fwrite(fid1, ext(i).edata); + fclose(fid1); + deleteDcm = onCleanup(@() deleteFile(tmp)); + ext(i).edata_decoded = dicm_hdr(tmp); + end +end + +%% subfunction: read img +% memory gunzip may be slow and error for large img, so use file unzip +function img = read_img(hdr, para) +ind = para.datatype == hdr.datatype; +if ~any(ind) + error('Datatype %g is not supported by nii_tool.', hdr.datatype); +end + +dim = hdr.dim(2:8); +dim(hdr.dim(1)+1:7) = 1; % avoid some error in file +dim(dim<1) = 1; +valpix = para.valpix(ind); + +fname = nii_name(hdr.file_name, '.img'); % in case of .hdr/.img pair +fid = fopen(fname); +sig = fread(fid, 2, '*uint8')'; +if isequal(sig, [31 139]) % .gz + fclose(fid); + fname = gunzipOS(fname); + cln = onCleanup(@() deleteFile(fname)); % delete gunzipped file + fid = fopen(fname); +end + +% if ~exist('cln', 'var') && valpix==1 && ~hdr.swap_endian +% m = memmapfile(fname, 'Offset', hdr.vox_offset, ... +% 'Format', {para.format{ind}, dim, 'img'}); +% nii = m.Data; +% return; +% end + +if hdr.swap_endian % switch between LE and BE + [~, ~, ed] = fopen(fid); % default endian: almost always ieee-le + fclose(fid); + if isempty(strfind(ed, '-le')), ed = strrep(ed, '-be', '-le'); %#ok<*STREMP> + else, ed = strrep(ed, '-le', '-be'); + end + fid = fopen(fname, 'r', ed); % re-open with changed endian +end + +fseek(fid, hdr.vox_offset, 'bof'); +img = fread(fid, prod(dim)*valpix, ['*' para.format{ind}]); % * to keep original class +fclose(fid); + +if any(hdr.datatype == [128 511 2304]) % RGB or RGBA +% a = reshape(single(img), valpix, n); % assume rgbrgbrgb... +% d1 = abs(a - a(:,[2:end 1])); % how similar are voxels to their neighbor +% a = reshape(a, prod(dim(1:2)), valpix*prod(dim(3:7))); % rr...rgg...gbb...b +% d2 = abs(a - a([2:end 1],:)); +% j = (sum(d1(:))>sum(d2(:)))*2 + 1; % 1 for afni, 3 for mricron + j = para.rgb_dim; % auto detection may get wrong for noisy background + dim = [dim(1:j-1) valpix dim(j:7)]; % length=8 now + img = reshape(img, dim); + img = permute(img, [1:j-1 j+1:8 j]); % put RGB(A) to dim8 +elseif any(hdr.datatype == [32 1792]) % complex single/double + img = reshape(img, [2 dim]); + img = complex(permute(img(1,:,:,:,:,:,:,:), [2:8 1]), ... % real + permute(img(2,:,:,:,:,:,:,:), [2:8 1])); % imag +else % all others: valpix=1 + if hdr.datatype==1, img = logical(img); end + img = reshape(img, dim); +end + +% RGB triplet in 5th dim OR RGBA quadruplet in 5th dim +c = hdr.intent_code; +if (c == 2003 && dim(5) == 3) || (c == 2004 && dim(5) == 4) + img = permute(img, [1:4 6:8 5]); +end + +%% Return requested fname with ext, useful for .hdr and .img files +function fname = nii_name(fname, ext) +if strcmpi(ext, '.img') + i = regexpi(fname, '.hdr(.gz)?$'); + if ~isempty(i), fname(i(end)+(0:3)) = ext; end +elseif strcmpi(ext, '.hdr') + i = regexpi(fname, '.img(.gz)?$'); + if ~isempty(i), fname(i(end)+(0:3)) = ext; end +end + +%% Read NIfTI file as bytes, gunzip if needed, but ignore endian +function [b, fname] = nii_bytes(fname, nByte) +if nargin<2, nByte = inf; end +[fid, err] = fopen(fname); % system default endian +if fid<1, error([err ': ' fname]); end +b = fread(fid, nByte, '*uint8')'; +fname = fopen(fid); +fclose(fid); +if isequal(b(1:2), [31 139]) % gz, tgz file + b = gunzip_mem(b, fname, nByte)'; +end + +%% subfunction: get help for a command +function subFuncHelp(mfile, cmd) +str = fileread(which(mfile)); +i = regexp(str, '\n\s*%', 'once'); % start of 1st % line +str = regexp(str(i:end), '.*?(?=\n\s*[^%])', 'match', 'once'); % help text +str = regexprep(str, '\r?\n\s*%', '\n'); % remove '\r' and leading % + +dashes = regexp(str, '\n\s*-{1,4}\s+') + 1; % lines starting with 1 to 4 - +if isempty(dashes), disp(str); return; end % Show all help text + +prgrfs = regexp(str, '(\n\s*){2,}'); % blank lines +nTopic = numel(dashes); +topics = ones(1, nTopic+1); +for i = 1:nTopic + ind = regexpi(str(1:dashes(i)), [mfile '\s*\(']); % syntax before ' - ' + if isempty(ind), continue; end % no syntax before ' - ', assume start with 1 + ind = find(prgrfs < ind(end), 1, 'last'); % previous paragraph + if isempty(ind), continue; end + topics(i) = prgrfs(ind) + 1; % start of this topic +end +topics(end) = numel(str); % end of last topic + +cmd = strrep(cmd, '?', ''); % remove ? in case it is in subcmd +if isempty(cmd) % help for main function + disp(str(1:topics(1))); % subfunction list before first topic + return; +end + +expr = [mfile '\s*\(\s*''' cmd '''']; +for i = 1:nTopic + if isempty(regexpi(str(topics(i):dashes(i)), expr, 'once')), continue; end + disp(str(topics(i):topics(i+1))); + return; +end + +fprintf(2, ' Unknown command for %s: %s\n', mfile, cmd); % no cmd found + +%% gunzip bytes in memory if possible. 2nd/3rd input for fallback file gunzip +% Trick: try-block avoid error for partial file unzip. +function bytes = gunzip_mem(gz_bytes, fname, nByte) +bytes = []; +try + bais = java.io.ByteArrayInputStream(gz_bytes); + try, gzis = java.util.zip.GZIPInputStream(bais); %#ok<*NOCOM> + catch, try, gzis = java.util.zip.InflaterInputStream(bais); catch me; end + end + buff = java.io.ByteArrayOutputStream; + try org.apache.commons.io.IOUtils.copy(gzis, buff); catch me; end + gzis.close; + bytes = typecast(buff.toByteArray, 'uint8'); + if isempty(bytes), error(me.message); end +catch + if nargin<3 || isempty(nByte), nByte = inf; end + if nargin<2 || isempty(fname) + fname = [tempname '.gz']; % temp gz file + fid = fopen(fname, 'W'); + if fid<0, return; end + cln = onCleanup(@() deleteFile(fname)); % delete temp gz file + fwrite(fid, gz_bytes, 'uint8'); + fclose(fid); + end + + try %#ok<*TRYNC> + fname = gunzipOS(fname, nByte); + fid = fopen(fname); + bytes = fread(fid, nByte, '*uint8'); + fclose(fid); + deleteFile(fname); % unzipped file + end +end + +%% Delete file in background +function deleteFile(fname) +if ispc, system(['start "" /B del "' fname '"']); +else, system(['rm "' fname '" &']); +end + +%% faster than system: based on https://github.com/avivrosenberg/matlab-jsystem +function [err, out] = jsystem(cmd) +% cmd is cell str, no quotation marks needed for file names with space. +cmd = cellstr(cmd); +try + pb = java.lang.ProcessBuilder(cmd); + pb.redirectErrorStream(true); % ErrorStream to InputStream + process = pb.start(); + scanner = java.util.Scanner(process.getInputStream).useDelimiter('\A'); + if scanner.hasNext(), out = char(scanner.next()); else, out = ''; end + err = process.exitValue; % err = process.waitFor() may hang + if err, error('java.lang.ProcessBuilder error'); end +catch % fallback to system() if java fails like for Octave + cmd = regexprep(cmd, '.+? .+', '"$0"'); % double quotes if with middle space + [err, out] = system(sprintf('%s ', cmd{:}, '2>&1')); +end + +%% Return true if input is char or single string (R2016b+) +function tf = ischar(A) +tf = builtin('ischar', A); +if tf, return; end +if exist('strings', 'builtin'), tf = isstring(A) && numel(A)==1; end +%% \ No newline at end of file diff --git a/ecat_testing/ints/python_sample_bytes.bytes b/ecat_testing/ints/python_sample_bytes.bytes new file mode 100644 index 00000000..68756cd4 Binary files /dev/null and b/ecat_testing/ints/python_sample_bytes.bytes differ diff --git a/ecat_testing/ints/template.variables.env b/ecat_testing/ints/template.variables.env new file mode 100644 index 00000000..a5a3f304 --- /dev/null +++ b/ecat_testing/ints/template.variables.env @@ -0,0 +1,4 @@ +# place paths to real files here and rename this file to variables.env +WELL_BEHAVED_ECAT_FILE= +WUSTL_FBP_ECAT= +WUSTL_OSEM_ECAT= \ No newline at end of file diff --git a/ecat_testing/read_matlab_nii.py b/ecat_testing/read_matlab_nii.py new file mode 100644 index 00000000..0de93b83 --- /dev/null +++ b/ecat_testing/read_matlab_nii.py @@ -0,0 +1,42 @@ +import nibabel +import pathlib +import numpy + +steps_dir = pathlib.Path(__file__).parent / 'steps' + +def first_middle_last_frames_to_text(four_d_array_like_object, output_folder, step_name='_step_name_'): + frames = [0, four_d_array_like_object.shape[-1] // 2, four_d_array_like_object.shape[-1] - 1] + frames_to_record = [] + for f in frames: + frames_to_record.append(four_d_array_like_object[:, :, :, f]) + + # now collect a single 2d slice from the "middle" of the 3d frames in frames_to_record + for index, frame in enumerate(frames_to_record): + numpy.savetxt(output_folder / f"{step_name}_frame_{frames[index]}.tsv", + frames_to_record[index][:, :, frames_to_record[index].shape[2] // 2], + delimiter="\t", fmt='%s') + +path_to_matlab_nii = pathlib.Path('matlab_nii.nii') +path_to_python_nii = pathlib.Path('python_nii.nii.gz') + + +python_nii = nibabel.load(path_to_python_nii) +matlab_nii = nibabel.load(path_to_matlab_nii) + +python_data = python_nii.get_fdata() +matlab_data = matlab_nii.get_fdata() + +# compare the two arrays in each +print(numpy.allclose(python_data, matlab_data, rtol=0.01)) + + +# subtract the two arrays +diff = python_data - matlab_data +print(f"difference max and min: {diff.max()}, {diff.min()}") +print(f"mean difference: {np.sqrt(np.mean(diff ** 2))}") + +# save diff as nii +diff_nii = python_nii.__class__(diff, python_nii.affine, python_nii.header) +nibabel.save(diff_nii, steps_dir / '12_diff_between_written_niis.nii.gz') + +first_middle_last_frames_to_text(diff, steps_dir, step_name='13_diff_between_written_niis') \ No newline at end of file diff --git a/ecat_testing/test_nii_vs_ecat.m b/ecat_testing/test_nii_vs_ecat.m new file mode 100644 index 00000000..dab8da71 --- /dev/null +++ b/ecat_testing/test_nii_vs_ecat.m @@ -0,0 +1,22 @@ +[mh,sh,data] = readECAT7; % read in p9816fvat1_osem.ecat +nii = MRIread('p9816fvat1_osem.nii'); + +figure; imagesc(rot90(fliplr(squeeze(data{10}(:,:,40))))) +figure; imagesc(squeeze(nii.vol(:,:,40,10))) + +ecat = rot90(fliplr(squeeze(data{10}(:,:,40)))); +turku = squeeze(nii.vol(:,:,40,10)); + +val_ecat = ecat(10,6); +val_turku = turku(10,6); + +scale_factor = sh{10}.scale_factor; +calibration_factor = mh.ecat_calibration_factor; + +cf = scale_factor*calibration_factor; + +print(val_ecatcf,val_turku) + +vinci = [12055.6,50513.8,22912.1]; +figure; plot(turku(24:26,65),vinci,'.') +mdl = fitlm(vinci,turku(24:26,65)); \ No newline at end of file diff --git a/matlab/ecat2nii.m b/matlab/ecat2nii.m index 2a059eb4..a780f21f 100644 --- a/matlab/ecat2nii.m +++ b/matlab/ecat2nii.m @@ -48,6 +48,16 @@ gz = true; % compress nifti savemat = false; % save ecat data as .mat +%% debebugging +% ------------ +ecat_save_steps = getenv('ECAT_SAVE_STEPS'); +ecat_save_steps_dir = mfilename('fullpath'); +if contains(ecat_save_steps_dir, 'LiveEditorEvaluationHelper') + ecat_save_steps_dir = matlab.desktop.editor.getActiveFilename; +end +parts = strsplit(ecat_save_steps_dir, filesep); +ecat_save_steps_dir = strjoin([parts(1:end-2), 'ecat_testing', 'steps'], filesep); + %% check inputs % ------------ @@ -143,7 +153,10 @@ end pet_file = [pet_file ext]; - [mh,sh] = readECAT7([pet_path filesep pet_file]); % loading the whole file here and iterating to flipdim below only minuimally improves time (0.6sec on NRU server) + [mh,sh,data] = readECAT7([pet_path filesep pet_file]); % loading the whole file here and iterating to flipdim below only minuimally improves time (0.6sec on NRU server) + if (ecat_save_steps == '1') + first_middle_last_frames_to_text_cell(data,ecat_save_steps_dir, '6_ecat2nii_matlab'); + end if sh{1}.data_type ~= 6 error('Conversion for 16 bit data only (type 6 in ecat file) - error loading ecat file'); end @@ -166,7 +179,13 @@ Randoms(i) = 0; end end - + + % save debugging steps 6 and 7 + if (ecat_save_steps == '1') + first_middle_last_frames_to_text_cell(data,ecat_save_steps_dir, '6_ecat2nii_matlab'); + first_middle_last_frames_to_text(img_temp,ecat_save_steps_dir, '7_flip_ecat2nii_matlab'); + end + % rescale to 16 bits rg = max(img_temp(:))-min(img_temp(:)); if rg ~= 32767 % same as range(img_temp(:)) but no need of stats toolbox @@ -178,8 +197,22 @@ img_temp = img_temp/MinImg*(-32768); Sca = Sca*MinImg/(-32768); end + + % save scaling factor to file located at ecat_save_steps_dir/8.5_sca_matlab.txt + fid = fopen([ecat_save_steps_dir filesep '8.5_sca_matlab.txt'],'w'); + fprintf(fid,'Scaling factor: %10e\n',Sca); + x = mh.ecat_calibration_factor * Sca; + fprintf(fid,'Scaling factor * ECAT Cal Factor: %10.10f\n',x); + fclose(fid); end - + + + + % save debugging step 8 - rescale to 16 bits + if (ecat_save_steps == '1') + first_middle_last_frames_to_text(img_temp,ecat_save_steps_dir, '8_rescale_to_16_ecat2nii_matlab'); + end + % check names if ~exist('FileListOut','var') [~,pet_filename] = fileparts(pet_file); @@ -215,8 +248,12 @@ end % save raw data - if savemat - ecat = img_temp.*(Sca*mh.ecat_calibration_factor); + if savemat or ecat_save_steps == '1' + if mh.calibration_units == 1 % see line 337 + ecat = img_temp.*Sca; + else + ecat = img_temp.*(Sca*mh.ecat_calibration_factor); + end save([filenameout '.ecat.mat'],'ecat','-v7.3'); end @@ -235,10 +272,23 @@ end else % annotation is blank - no info on method - warning('no reconstruction method information found - invalid BIDS metadata') - info.ReconMethodParameterLabels = {'lower_threshold', 'upper_threshold'}; - info.ReconMethodParameterUnits = {'keV', 'keV'}; - info.ReconMethodParameterValues = [mh.lwr_true_thres, mh.upr_true_thres]; + if isfield(info.ReconMethodName) % user provided + [info.ReconMethodName,i,s] = get_recon_method(deblank(info.ReconMethodName)); + if ~isempty(i) && ~isempty(s) + info.ReconMethodParameterLabels = {'iterations', 'subsets', 'lower_threshold', 'upper_threshold'}; + info.ReconMethodParameterUnits = {'none', 'none', 'keV', 'keV'}; + info.ReconMethodParameterValues = [str2double(i), str2double(s), mh.lwr_true_thres, mh.upr_true_thres]; + else % some method without iteration and subset e.g. back projection + info.ReconMethodParameterLabels = {'lower_threshold', 'upper_threshold'}; + info.ReconMethodParameterUnits = {'keV', 'keV'}; + info.ReconMethodParameterValues = [mh.lwr_true_thres, mh.upr_true_thres]; + end + else + warning('no reconstruction method information found - invalid BIDS metadata') + info.ReconMethodParameterLabels = {'lower_threshold', 'upper_threshold'}; + info.ReconMethodParameterUnits = {'keV', 'keV'}; + info.ReconMethodParameterValues = [mh.lwr_true_thres, mh.upr_true_thres]; + end end else % no info on method @@ -330,7 +380,16 @@ warning('the json file is BIDS invalid') end - img_temp = single(round(img_temp).*(Sca*mh.ecat_calibration_factor)); + if mh.calibration_units == 1 % do calibrate + img_temp = single(round(img_temp).*(Sca*mh.ecat_calibration_factor)); % scale and dose calibrated + if ecat_save_steps == '1' + first_middle_last_frames_to_text(img_temp,ecat_save_steps_dir, '9_scal_cal_units_ecat2nii_matlab'); + end + warning('it looks like the source data are not dose calibrated - ecat2nii is thus scaling the data') + else % do not calibrate + img_temp = single(round(img_temp).*Sca); % just the 16 bit scaling, img_temp is already dose calibrated + warning('it looks like the source data are already dose calibrated - ecat2nii is thus not scaling the data') + end info.Datatype = 'single'; info.BitsPerPixel = 32; info.SpaceUnits = 'Millimeter'; @@ -405,9 +464,19 @@ if gz fnm = [fnm '.gz']; %#ok<*AGROW> end - + + + if ecat_save_steps == '1' + first_middle_last_frames_to_text(img_temp, ecat_save_steps_dir, '10_save_nii_cat2nii_matlab'); + end nii_tool('save', nii, fnm); FileListOut{j} = fnm; + + if ecat_save_steps == '1' + % open the nifti file just written and check to see if the written values differ from the original + nii = nii_tool('load', fnm); + first_middle_last_frames_to_text(nii.img, ecat_save_steps_dir, '11_read_saved_nii_matlab'); + end % optionally one can use niftiwrite from the Image Processing Toolbox % warning different versions of matlab may provide different nifti results @@ -422,7 +491,7 @@ % end catch conversionerr - FileListOut{j} = sprintf('%s failed to convert:%s',FileListIn{j},conversionerr.message); + FileListOut{j} = sprintf('%s failed to convert:%s',FileListIn{j},conversionerr.message, conversionerr.stack.line); end if exist('newfile','var') % i.e. decompresed .nii.gz diff --git a/matlab/first_middle_last_frames_to_text.m b/matlab/first_middle_last_frames_to_text.m new file mode 100644 index 00000000..18a3b01d --- /dev/null +++ b/matlab/first_middle_last_frames_to_text.m @@ -0,0 +1,32 @@ +function [output_folder, step_name] = first_middle_last_frames_to_text(four_d_array_like_object,output_folder, step_name) +%takes a times series of 3d images and writes out sub sections of that time +%series as 2D frames. Frames are labled as zero indexed to aligne with +%python conventions. Up to 3 frames are selected from the time series +%corresponding to the first, middle, and last frames. +% four_d_array_like_object: input time series +% output_folder: path to write the selected 2D frames to +% step_name: name to apply to output files +% +output_folder = output_folder; +step_name = step_name; +data = four_d_array_like_object; +data_size = size(data); + +% get first, middle, and last frame of data +frames = [ 1, floor(data_size(4)/2) + 1, data_size(4)] +frames_to_record = cell(length(frames)); +for i = 1:length(frames) + frame_number = frames(i); + frame_to_slice = data(:,:,:,frame_number); + size_of_frame_to_slice = size(frame_to_slice) + middle_of_frame = int16(size_of_frame_to_slice(3)/2); + slice = data(:,:,middle_of_frame,frame_number); + frames_to_record{i} = slice; +end + +for i = 1:length(frames_to_record) + frame = frames_to_record{i}; + frame_string = string(frames(i) - 1); + filename = strcat(output_folder, filesep, step_name, '_frame_', frame_string, '.tsv'); + writematrix(frame, filename, 'Delimiter', 'tab', 'FileType', 'text'); +end diff --git a/matlab/first_middle_last_frames_to_text_cell.m b/matlab/first_middle_last_frames_to_text_cell.m new file mode 100644 index 00000000..cc17dd5d --- /dev/null +++ b/matlab/first_middle_last_frames_to_text_cell.m @@ -0,0 +1,39 @@ +function [output_folder, step_name] = first_middle_last_frames_to_text(four_d_array_like_object,output_folder, step_name) +%takes a times series of 3d images and writes out sub sections of that time +%series as 2D frames. Frames are labled as zero indexed to aligne with +%python conventions. Up to 3 frames are selected from the time series +%corresponding to the first, middle, and last frames. +% four_d_array_like_object: input time series +% output_folder: path to write the selected 2D frames to +% step_name: name to apply to output files +% +output_folder = output_folder; +step_name = step_name; +data = four_d_array_like_object; + +% get first, middle, and last frame of data +frames = [ 1, floor(length(data)/2) + 1, length(data)]; +frames_to_record = cell(length(frames)); +for i = 1:length(frames) + frame_number = frames(i); + try + frame_to_slice = data{frame_number}; + catch + frame_to_slice = data(frame_number); + end + size_of_frame_to_slice = size(frame_to_slice); + middle_of_frame = int16(size_of_frame_to_slice(3)/2); + try + slice = data{frame_number}(:,:,middle_of_frame); + catch + slice = data(:,:,middle_of_frame,frame_number); + end + frames_to_record{i} = slice; +end + +for i = 1:length(frames_to_record) + frame = frames_to_record{i}; + frame_string = string(frames(i) - 1); + filename = strcat(output_folder, filesep, step_name, '_frame_', frame_string, '.tsv'); + writematrix(frame, filename, 'Delimiter', 'tab', 'FileType', 'text'); +end diff --git a/matlab/get_pet_metadata.m b/matlab/get_pet_metadata.m index 102fd1fd..b3f9a78a 100644 --- a/matlab/get_pet_metadata.m +++ b/matlab/get_pet_metadata.m @@ -1,5 +1,4 @@ function metadata = get_pet_metadata(varargin) - % Routine that outputs PET scanner metadata following % `BIDS `_. % @@ -16,8 +15,8 @@ % all info is necessarily needed\) % :param inputs: a series of key/value pairs are expected % :returns metadata: a structure with BIDS fields filled \(such structure is ready -% to be writen as json file using e.g. the bids matlab jsonwrite -% function, typically associated with the *_pet.nii file\) +% to be writen as json file using e.g. the bids matlab jsonwrite +% function, typically associated with the \*_pet.nii file\) % % :format: metadata = get_pet_metadata(key,value) % @@ -25,7 +24,7 @@ % % Mandatory inputs are as follows\: % -% - *Scanner* name of scanner, map to a *parameters.txt file e.g. 'Scanner', 'SiemensBiograph' +% - *Scanner* name of scanner, map to a \*parameters.txt file e.g. 'Scanner', 'SiemensBiograph' % - *TimeZero* when was the tracer injected e.g. 'TimeZero','11:05:01' % - *ModeOfAdministration* e.g. 'ModeOfAdministration', 'bolus' % - *TracerName* which tracer was used e.g. 'TracerName','DASB' @@ -55,7 +54,7 @@ % FrameTimesStart = 0; % % .. note:: -% TimeZero also can be [] or 'ScanStart' indicating that the scanning time +% TimeZero also can be [] or 'ScanStart' indicating that the scanning time % should be used as TimeZero. If TimeZero is not the scan time, we strongly % advice to input ScanStart and InjectionStart making sure timing is correct % @@ -228,7 +227,7 @@ % evaluate key-value pairs for optional arguments from txt file parameter_file = fullfile(fileparts(which('get_pet_metadata.m')),[Scanner 'parameters.txt']); - if ~any(cellfun(@exist, optional)) + if any(cellfun(@exist, optional)) if exist(parameter_file,'file') setmetadata = importdata(parameter_file); for opt = 1:length(setmetadata) diff --git a/matlab/readECAT7.m b/matlab/readECAT7.m index f9f056d4..fe365e44 100644 --- a/matlab/readECAT7.m +++ b/matlab/readECAT7.m @@ -55,6 +55,8 @@ fs = []; end +%ecat_save_steps = '1' + if length(fs) == 0 origpwd = pwd; if length(lastpath_) > 0; cd(lastpath_); end @@ -77,7 +79,8 @@ end % open file as ieee big-endian -fid = fopen(fs,'r','ieee-be'); +file_endianess = 'ieee-be'; +fid = fopen(fs,'r', file_endianess); if (fid < 0) error('readECAT: Problem opening file %s', fs); end @@ -86,6 +89,23 @@ % Read in the main header info mh = readMainheader(fid); +ecat_save_steps = getenv('ECAT_SAVE_STEPS'); +ecat_save_steps_dir = mfilename('fullpath'); +if contains(ecat_save_steps_dir, 'LiveEditorEvaluationHelper') + ecat_save_steps_dir = matlab.desktop.editor.getActiveFilename; +end +parts = strsplit(ecat_save_steps_dir, filesep); +ecat_save_steps_dir = strjoin([parts(1:end-2), 'ecat_testing', 'steps'], filesep); + +% save main header output to json +if (ecat_save_steps == '1') + % save main header + main_header_json = (jsonencode(mh, PrettyPrint=true)); + step_1_file = fopen([ecat_save_steps_dir filesep '1_read_mh_ecat_matlab.json'], 'w'); + fprintf(step_1_file, '%s', main_header_json); + fclose(step_1_file); +end + if (nargout == 1) return end @@ -129,6 +149,7 @@ sh = cell(nmat,1); data = cell(nmat,1); +pixel_data_type = ''; % select proper sh routine switch mh.file_type @@ -173,16 +194,19 @@ switch sh{m}.data_type case 5 % IEEE float (32 bit) data{m} = fread(fid, [sz(1)*sz(2) sz(3)],'float32'); + pixel_data_type = 'float32'; case 6 % SUN int16 if (matlabVersion(1) < 6) warning('old matlab version, using int16 to read') data{m} = int16(fread(fid, [sz(1)*sz(2) sz(3)],'int16')); + pixel_data_type = 'int16'; else - % REALLY MAKE IT A UINT16 - data{m} = fread(fid, [sz(1)*sz(2) sz(3)], 'uint16'); + data{m} = fread(fid, [sz(1)*sz(2) sz(3)],'int16=>int16'); + pixel_data_type = 'int16=>int16'; end otherwise warning('readECAT7: unrecognized data type'); + pixel_data_type = 'unrecognized data type'; end % other sh{m}.data_type: 2 = VAX int16, 3 = VAX int32, 4 = VAX F-float (32 bit), 7 = SUN int32 @@ -197,6 +221,31 @@ end % loop over matrices fclose(fid); +if (ecat_save_steps == '1') + % save subheaders header + sub_header_json = (jsonencode(sh, PrettyPrint=true)); + step_2_file = fopen([ecat_save_steps_dir filesep '1_read_sh_ecat_matlab.json'], 'w'); + fprintf(step_2_file, '%s', sub_header_json); + fclose(step_2_file); + + % write out endianess and datatype of the pixel data matrix + step_3_struct = {}; + x.data_type = class(data{1}); + x.endianess = file_endianess; + step_3_json = (jsonencode(x, PrettyPrint=true)); + step_3_file = fopen([ecat_save_steps_dir filesep '3_determine_data_type_matlab.json'], 'w'); + fprintf(step_3_file, '%s', step_3_json); + fclose(step_3_file); + + first_middle_last_frames_to_text_cell(data, ecat_save_steps_dir, '4_read_img_ecat_matlab') + + % scale if calibrated + % data has already been saved in the previous step (step 4) but we go ahead and do this step once more because the + % numbering system that's been imposed upon us to test these outputs + if calibrated ~0 + first_middle_last_frames_to_text_cell(data, ecat_save_steps_dir, '5_scale_img_ecat_matlab') + end +end return diff --git a/matlab/updatejsonpetfile.m b/matlab/updatejsonpetfile.m index e7bdc759..380ba8ae 100644 --- a/matlab/updatejsonpetfile.m +++ b/matlab/updatejsonpetfile.m @@ -79,14 +79,17 @@ [filemetadata,updated] = update_arrays(filemetadata); if updated && exist('jsonfilename','var') warning('some scalars were changed to array') + if strcmpi(filemetadata.ReconFilterType,"none") + filemetadata.ReconFilterSize = 0; % not necessary once the validator takes conditinonal + end jsonwrite(jsonfilename,orderfields(filemetadata)); end - % -------------- only check --------------- + % -------------- only check -------------- for m=length(petmetadata.mandatory):-1:1 test(m) = isfield(filemetadata,petmetadata.mandatory{m}); end - + if sum(test)~=length(petmetadata.mandatory) status.state = 0; missing = find(test==0); @@ -360,8 +363,12 @@ else % returns none if actually seen as empty by get_recon_method filemetadata.ReconMethodParameterLabels = "none"; filemetadata.ReconMethodParameterUnits = "none"; - if isempty(filemetadata.ReconMethodParameterValues) % in cas user passes info - filemetadata.ReconMethodParameterValues = 0; % if none should be 0 + try + if isempty(filemetadata.ReconMethodParameterValues) % in case user passes info + filemetadata.ReconMethodParameterValues = 0; % if none should be 0 + end + catch + filemetadata.ReconMethodParameterValues = 0; end end end @@ -383,10 +390,13 @@ elseif isfield(filemetadata,'ReconFilterType') && isfield(filemetadata,'ReconFilterSize') if strcmp(filemetadata.ReconFilterType,filemetadata.ReconFilterSize) filtername = filemetadata.ReconFilterType; %% because if was set matching DICOM and BIDS + if strcmp(filemetadata.ReconFilterType,"none") + filemetadata.ReconFilterSize = 0; + end end else filemetadata.ReconFilterType = "none"; - % filemetadata.ReconFilterSize = 0; % conditional on ReconFilterType + filemetadata.ReconFilterSize = 0; % conditional on ReconFilterType end if exist('filtername','var') @@ -423,7 +433,7 @@ end else filemetadata.ReconFilterType = "none"; - % filemetadata.ReconFilterSize = 0; % conditional on ReconFilterType + filemetadata.ReconFilterSize = 0; % conditional on ReconFilterType end function [filemetadata,updated] = update_arrays(filemetadata) diff --git a/metadata/PET_reconstruction_methods.json b/metadata/PET_reconstruction_methods.json index 471df6ec..a6f9ec32 100644 --- a/metadata/PET_reconstruction_methods.json +++ b/metadata/PET_reconstruction_methods.json @@ -54,6 +54,22 @@ 10 ] }, + { + "contents": "OP_OSEM3D", + "ReconMethodName": "Ordinary Poisson 3D Ordered Subset Expectation Maximization", + "ReconMethodParameterUnits": [ + null, + null + ], + "ReconMethodParameterLabels": [ + "subsets", + "iterations" + ], + "ReconMethodParameterValues": [ + null, + null + ] + }, { "contents": "LOR-RAMLA", "subsets": null, diff --git a/pypet2bids/README.md b/pypet2bids/README.md new file mode 100644 index 00000000..e95babcd --- /dev/null +++ b/pypet2bids/README.md @@ -0,0 +1,132 @@ +# PET2BIDS is a code library to convert source Brain PET data to BIDS + +[![python](https://github.com/openneuropet/PET2BIDS/actions/workflows/python.yaml/badge.svg)](https://github.com/openneuropet/PET2BIDS/actions/workflows/python.yaml) +[![Matlab PET2BIDS Tests](https://github.com/openneuropet/PET2BIDS/actions/workflows/matlab.yaml/badge.svg)](https://github.com/openneuropet/PET2BIDS/actions/workflows/matlab.yaml) +[![Documentation Status](https://readthedocs.org/projects/pet2bids/badge/?version=latest)](https://pet2bids.readthedocs.io/en/latest/?badge=latest) +[![phantoms](https://github.com/openneuropet/PET2BIDS/actions/workflows/phantoms.yaml/badge.svg)](https://github.com/openneuropet/PET2BIDS/actions/workflows/phantoms.yaml) + +This repository is hosting tools to curate PET brain data using the [Brain Imaging Data Structure Specification](https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/09-positron-emission-tomography.html). The work to create these tools is funded by [Novo Nordisk Foundation](https://novonordiskfonden.dk/en/) (NNF20OC0063277) and the [BRAIN initiative](https://braininitiative.nih.gov/) (MH002977-01). + +For DICOM image conversion, we rely on [dcm2niix](https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage), +collaborating with Prof. Chris Rorden without whom we could not convert your data! For more information on dcm2niix +and nifti please see [The first step for neuroimaging data analysis: DICOM to NIfTI conversion](https://www.ncbi.nlm.nih.gov/pubmed/26945974) paper. + + +# Documentation + +For **more detailed** (and most likely helpful) documentation visit the Read the Docs site for this project at: + +[https://pet2bids.readthedocs.io](https://pet2bids.readthedocs.io/en/latest/index.html#) + +## Installation + +Simply download the repository - follow the specific Matlab or Python explanations. Matlab and Python codes provide the same functionalities. + +### matlab + +[![asciicast](https://asciinema.org/a/RPxiHW6afISPmWYFBOGKWNem1.svg)](https://asciinema.org/a/RPxiHW6afISPmWYFBOGKWNem1) + +1) remember to set the path to the PET2BIDS/matlab folder, you will find the source code to use here. +2) if converting DICOM files, make sure you have dcm2niix (for windows users, edit dcm2niix4pet.m to set the right paths to the .exe) +3) start using the code! more info [here](https://github.com/openneuropet/PET2BIDS/tree/main/matlab#readme) + +### pypet2bids + +Use pip to install this library directly from PyPI: + +[![asciicast](https://asciinema.org/a/TZJg5BglDMFM2fEEX9dSpnJEy.svg)](https://asciinema.org/a/TZJg5BglDMFM2fEEX9dSpnJEy) + +If you wish to install directly from this repository see the instructions below to either build +a packaged version of `pypet2bids` or how to run the code from source. + +
+Build Package Locally and Install with PIP + +We use [poetry](https://python-poetry.org/) to build this package, no other build methods are supported, +further we encourage the use of [GNU make](https://www.gnu.org/software/make/) and a bash-like shell to simplify the +build process. + +After installing poetry, you can build and install this package to your local version of Python with the following +commands (keep in mind the commands below are executed in a bash-like shell): + +```bash +cd PET2BIDS +cp -R metadata/ pypet2bids/pypet2bids/metadata +cp pypet2bids/pyproject.toml pypet2bids/pypet2bids/pyproject.toml +cd pypet2bids && poetry lock && poetry build +pip install dist/pypet2bids-X.X.X-py3-none-any.whl +``` + +Why is all the above required? Well, because this is a monorepo and we just have to work around that sometimes. + + +[!NOTE] +Make and the additional scripts contained in the `scripts/` directory are for the convenience of +non-windows users. + +If you have GNU make installed and are using a bash or something bash-like in you your terminal of choice, run the +following: + +```bash +cd PET2BIDS +make installpoetry buildpackage installpackage +``` + +
+ +
+Run Directly From Source + +Lastly, if one wishes run pypet2bids directly from the source code in this repository or to help contribute to the python portion of this project or any of the documentation they can do so via the following options: + +```bash +cd PET2BIDS/pypet2bids +poetry install +``` + +Or they can install the dependencies only using pip: + +```bash +cd PET2BIDS/pypet2bids +pip install . +``` + +After either poetry or pip installation of dependencies modules can be executed as follows: + +```bash +cd PET2BIDS/pypet2bids +python dcm2niix4pet.py --help +``` + +
+ +**Note:** +*We recommend using dcm2niix v1.0.20220720 or newer; we rely on metadata included in these later releases. It's best to +collect releases from the [rorden lab/dcm2niix/releases](https://github.com/rordenlab/dcm2niix/releases) page. We have +observed that package managers such as yum or apt or apt-get often install much older versions of dcm2niix e.g. +v1.0.2017XXXX, v1.0.2020XXXXX. You may run into invalid-BIDS or errors with this software with older versions.* + + +### spreadsheet_conversion (custom and pmod) + +This folder contains spreadsheets templates and examples of metadata and matlab and python code to convert them to json files. Often, metadata such as Frame durations, InjectedRadioactivity, etc are stored in spreadsheets and we have made those function to create json files automatically for 1 or many subjects at once to go with the nifti imaging data. Note, we also have conversion for pmod files (also spreadsheets) allowing to export to blood.tsv files. + +### metadata + +A small collection of json files for our metadata information. + +### user metadata + +No matter the way you prefer inputting metadata (passing all arguments, using txt or env file, using spreadsheets), you are always right! DICOM values will be ignored - BUT they are checked and the code tells you if there is inconsistency between your inputs and what DICOM says. + +### ecat_validation + +This folder contains code generating Siemens HRRT scanner data using ecat file format and validating the matlab and python conversion tools (i.e. giving the data generated as ecat, do our nifti images reflect acurately the data). + +## Citation + +Please [cite us](CITATION.cff) when using PET2BIDS. + +## Contribute + +Anyone is welcome to contribute ! check here [how you can get involved](contributing.md), the [code of conduct](code_of_conduct.md). Contributors are listed [here](contributors.md) diff --git a/pypet2bids/poetry.lock b/pypet2bids/poetry.lock index 4cbf664c..f0e7fba1 100644 --- a/pypet2bids/poetry.lock +++ b/pypet2bids/poetry.lock @@ -24,13 +24,13 @@ files = [ [[package]] name = "babel" -version = "2.13.1" +version = "2.14.0" description = "Internationalization utilities" optional = false python-versions = ">=3.7" files = [ - {file = "Babel-2.13.1-py3-none-any.whl", hash = "sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed"}, - {file = "Babel-2.13.1.tar.gz", hash = "sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900"}, + {file = "Babel-2.14.0-py3-none-any.whl", hash = "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"}, + {file = "Babel-2.14.0.tar.gz", hash = "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363"}, ] [package.dependencies] @@ -64,13 +64,13 @@ virtualenv = ["virtualenv (>=20.0.35)"] [[package]] name = "certifi" -version = "2023.11.17" +version = "2024.2.2" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, - {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, ] [[package]] @@ -207,37 +207,27 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.2.0" +version = "1.2.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, - {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, ] [package.extras] test = ["pytest (>=6)"] -[[package]] -name = "future" -version = "0.18.3" -description = "Clean single-source support for Python 3 and 2" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"}, -] - [[package]] name = "idna" -version = "3.6" +version = "3.7" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, - {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] [[package]] @@ -253,32 +243,32 @@ files = [ [[package]] name = "importlib-metadata" -version = "6.8.0" +version = "7.1.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_metadata-6.8.0-py3-none-any.whl", hash = "sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb"}, - {file = "importlib_metadata-6.8.0.tar.gz", hash = "sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743"}, + {file = "importlib_metadata-7.1.0-py3-none-any.whl", hash = "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570"}, + {file = "importlib_metadata-7.1.0.tar.gz", hash = "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2"}, ] [package.dependencies] zipp = ">=0.5" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] [[package]] name = "importlib-resources" -version = "6.1.1" +version = "6.4.0" description = "Read resources from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_resources-6.1.1-py3-none-any.whl", hash = "sha256:e8bf90d8213b486f428c9c39714b920041cb02c184686a3dee24905aaa8105d6"}, - {file = "importlib_resources-6.1.1.tar.gz", hash = "sha256:3893a00122eafde6894c59914446a512f728a0c1a45f9bb9b63721b6bacf0b4a"}, + {file = "importlib_resources-6.4.0-py3-none-any.whl", hash = "sha256:50d10f043df931902d4194ea07ec57960f66a80449ff867bfe782b4c486ba78c"}, + {file = "importlib_resources-6.4.0.tar.gz", hash = "sha256:cdb2b453b8046ca4e3798eb1d84f3cce1446a0e8e7b5ef4efb600f19fc398145"}, ] [package.dependencies] @@ -286,7 +276,7 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-ruff", "zipp (>=3.17)"] +testing = ["jaraco.test (>=5.4)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)", "zipp (>=3.17)"] [[package]] name = "iniconfig" @@ -301,13 +291,13 @@ files = [ [[package]] name = "jinja2" -version = "3.1.2" +version = "3.1.3" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, ] [package.dependencies] @@ -318,13 +308,13 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "joblib" -version = "1.3.2" +version = "1.4.0" description = "Lightweight pipelining with Python functions" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "joblib-1.3.2-py3-none-any.whl", hash = "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"}, - {file = "joblib-1.3.2.tar.gz", hash = "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1"}, + {file = "joblib-1.4.0-py3-none-any.whl", hash = "sha256:42942470d4062537be4d54c83511186da1fc14ba354961a2114da91efa9a4ed7"}, + {file = "joblib-1.4.0.tar.gz", hash = "sha256:1eb0dc091919cd384490de890cb5dfd538410a6d4b3b54eef09fb8c50b409b1c"}, ] [[package]] @@ -354,101 +344,101 @@ altgraph = ">=0.17" [[package]] name = "markupsafe" -version = "2.1.3" +version = "2.1.5" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.7" files = [ - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, - {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, ] [[package]] name = "nibabel" -version = "5.1.0" +version = "5.2.1" description = "Access a multitude of neuroimaging data formats" optional = false python-versions = ">=3.8" files = [ - {file = "nibabel-5.1.0-py3-none-any.whl", hash = "sha256:b3deb8130c835b9d26e80880b0d5e443d9e3f30972b3b0302dd2fafa3ca629f8"}, - {file = "nibabel-5.1.0.tar.gz", hash = "sha256:ce73ca5e957209e7219a223cb71f77235c9df2acf4d3f27f861ba38e9481ac53"}, + {file = "nibabel-5.2.1-py3-none-any.whl", hash = "sha256:2cbbc22985f7f9d39d050df47249771dfb8d48447f5e7a993177e4cabfe047f0"}, + {file = "nibabel-5.2.1.tar.gz", hash = "sha256:b6c80b2e728e4bc2b65f1142d9b8d2287a9102a8bf8477e115ef0d8334559975"}, ] [package.dependencies] importlib-resources = {version = ">=1.3", markers = "python_version < \"3.9\""} -numpy = ">=1.19" +numpy = ">=1.20" packaging = ">=17" [package.extras] -all = ["nibabel[dev,dicomfs,doc,minc2,spm,style,test,zstd]"] -dev = ["gitpython", "nibabel[style]", "twine"] +all = ["nibabel[dicomfs,minc2,spm,zstd]"] +dev = ["tox"] dicom = ["pydicom (>=1.0.0)"] dicomfs = ["nibabel[dicom]", "pillow"] -doc = ["matplotlib (>=1.5.3)", "numpydoc", "sphinx (>=5.3,<6.0)", "texext", "tomli"] -doctest = ["nibabel[doc,test]"] +doc = ["matplotlib (>=1.5.3)", "numpydoc", "sphinx", "texext", "tomli"] +doctest = ["tox"] minc2 = ["h5py"] spm = ["scipy"] -style = ["blue", "flake8", "isort"] -test = ["coverage", "pytest (!=5.3.4)", "pytest-cov", "pytest-doctestplus", "pytest-httpserver", "pytest-xdist"] -typing = ["importlib-resources", "mypy", "pydicom", "pytest", "pyzstd", "types-pillow", "types-setuptools"] +style = ["tox"] +test = ["pytest", "pytest-cov", "pytest-doctestplus", "pytest-httpserver", "pytest-xdist"] +typing = ["tox"] zstd = ["pyzstd (>=0.14.3)"] [[package]] @@ -504,13 +494,13 @@ et-xmlfile = "*" [[package]] name = "packaging" -version = "23.2" +version = "24.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, ] [[package]] @@ -588,13 +578,13 @@ tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [[package]] name = "pluggy" -version = "1.3.0" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ - {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, - {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] [package.extras] @@ -603,13 +593,13 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pydicom" -version = "2.4.3" +version = "2.4.4" description = "A pure Python package for reading and writing DICOM data" optional = false python-versions = ">=3.7" files = [ - {file = "pydicom-2.4.3-py3-none-any.whl", hash = "sha256:797e84f7b22e5f8bce403da505935b0787dca33550891f06495d14b3f6c70504"}, - {file = "pydicom-2.4.3.tar.gz", hash = "sha256:51906e0b9fb6e184a0f56298cb43ed716b7cf7edc00f6b71d5c769bc1f982402"}, + {file = "pydicom-2.4.4-py3-none-any.whl", hash = "sha256:f9f8e19b78525be57aa6384484298833e4d06ac1d6226c79459131ddb0bd7c42"}, + {file = "pydicom-2.4.4.tar.gz", hash = "sha256:90b4801d851ce65be3df520e16bbfa3d6c767cf2a3a3b1c18f6780e6b670b87a"}, ] [package.extras] @@ -665,24 +655,29 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2023.10" +version = "2024.5" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.7" files = [ - {file = "pyinstaller-hooks-contrib-2023.10.tar.gz", hash = "sha256:4b4a998036abb713774cb26534ca06b7e6e09e4c628196017a10deb11a48747f"}, - {file = "pyinstaller_hooks_contrib-2023.10-py2.py3-none-any.whl", hash = "sha256:6dc1786a8f452941245d5bb85893e2a33632ebdcbc4c23eea41f2ee08281b0c0"}, + {file = "pyinstaller_hooks_contrib-2024.5-py2.py3-none-any.whl", hash = "sha256:0852249b7fb1e9394f8f22af2c22fa5294c2c0366157969f98c96df62410c4c6"}, + {file = "pyinstaller_hooks_contrib-2024.5.tar.gz", hash = "sha256:aa5dee25ea7ca317ad46fa16b5afc8dba3b0e43f2847e498930138885efd3cab"}, ] +[package.dependencies] +importlib-metadata = {version = ">=4.6", markers = "python_version < \"3.10\""} +packaging = ">=22.0" +setuptools = ">=42.0.0" + [[package]] name = "pyparsing" -version = "3.1.1" +version = "3.1.2" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = false python-versions = ">=3.6.8" files = [ - {file = "pyparsing-3.1.1-py3-none-any.whl", hash = "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb"}, - {file = "pyparsing-3.1.1.tar.gz", hash = "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"}, + {file = "pyparsing-3.1.2-py3-none-any.whl", hash = "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"}, + {file = "pyparsing-3.1.2.tar.gz", hash = "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad"}, ] [package.extras] @@ -690,13 +685,13 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "7.4.3" +version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, - {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, ] [package.dependencies] @@ -712,13 +707,13 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no [[package]] name = "python-dateutil" -version = "2.8.2" +version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, ] [package.dependencies] @@ -740,13 +735,13 @@ cli = ["click (>=5.0)"] [[package]] name = "pytz" -version = "2023.3.post1" +version = "2024.1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, - {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, ] [[package]] @@ -794,57 +789,57 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "scipy" -version = "1.10.1" +version = "1.9.3" description = "Fundamental algorithms for scientific computing in Python" optional = false -python-versions = "<3.12,>=3.8" +python-versions = ">=3.8" files = [ - {file = "scipy-1.10.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e7354fd7527a4b0377ce55f286805b34e8c54b91be865bac273f527e1b839019"}, - {file = "scipy-1.10.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4b3f429188c66603a1a5c549fb414e4d3bdc2a24792e061ffbd607d3d75fd84e"}, - {file = "scipy-1.10.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1553b5dcddd64ba9a0d95355e63fe6c3fc303a8fd77c7bc91e77d61363f7433f"}, - {file = "scipy-1.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c0ff64b06b10e35215abce517252b375e580a6125fd5fdf6421b98efbefb2d2"}, - {file = "scipy-1.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:fae8a7b898c42dffe3f7361c40d5952b6bf32d10c4569098d276b4c547905ee1"}, - {file = "scipy-1.10.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0f1564ea217e82c1bbe75ddf7285ba0709ecd503f048cb1236ae9995f64217bd"}, - {file = "scipy-1.10.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:d925fa1c81b772882aa55bcc10bf88324dadb66ff85d548c71515f6689c6dac5"}, - {file = "scipy-1.10.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaea0a6be54462ec027de54fca511540980d1e9eea68b2d5c1dbfe084797be35"}, - {file = "scipy-1.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15a35c4242ec5f292c3dd364a7c71a61be87a3d4ddcc693372813c0b73c9af1d"}, - {file = "scipy-1.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:43b8e0bcb877faf0abfb613d51026cd5cc78918e9530e375727bf0625c82788f"}, - {file = "scipy-1.10.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5678f88c68ea866ed9ebe3a989091088553ba12c6090244fdae3e467b1139c35"}, - {file = "scipy-1.10.1-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:39becb03541f9e58243f4197584286e339029e8908c46f7221abeea4b749fa88"}, - {file = "scipy-1.10.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bce5869c8d68cf383ce240e44c1d9ae7c06078a9396df68ce88a1230f93a30c1"}, - {file = "scipy-1.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07c3457ce0b3ad5124f98a86533106b643dd811dd61b548e78cf4c8786652f6f"}, - {file = "scipy-1.10.1-cp38-cp38-win_amd64.whl", hash = "sha256:049a8bbf0ad95277ffba9b3b7d23e5369cc39e66406d60422c8cfef40ccc8415"}, - {file = "scipy-1.10.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cd9f1027ff30d90618914a64ca9b1a77a431159df0e2a195d8a9e8a04c78abf9"}, - {file = "scipy-1.10.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:79c8e5a6c6ffaf3a2262ef1be1e108a035cf4f05c14df56057b64acc5bebffb6"}, - {file = "scipy-1.10.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51af417a000d2dbe1ec6c372dfe688e041a7084da4fdd350aeb139bd3fb55353"}, - {file = "scipy-1.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b4735d6c28aad3cdcf52117e0e91d6b39acd4272f3f5cd9907c24ee931ad601"}, - {file = "scipy-1.10.1-cp39-cp39-win_amd64.whl", hash = "sha256:7ff7f37b1bf4417baca958d254e8e2875d0cc23aaadbe65b3d5b3077b0eb23ea"}, - {file = "scipy-1.10.1.tar.gz", hash = "sha256:2cf9dfb80a7b4589ba4c40ce7588986d6d5cebc5457cad2c2880f6bc2d42f3a5"}, + {file = "scipy-1.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1884b66a54887e21addf9c16fb588720a8309a57b2e258ae1c7986d4444d3bc0"}, + {file = "scipy-1.9.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:83b89e9586c62e787f5012e8475fbb12185bafb996a03257e9675cd73d3736dd"}, + {file = "scipy-1.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a72d885fa44247f92743fc20732ae55564ff2a519e8302fb7e18717c5355a8b"}, + {file = "scipy-1.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d01e1dd7b15bd2449c8bfc6b7cc67d630700ed655654f0dfcf121600bad205c9"}, + {file = "scipy-1.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:68239b6aa6f9c593da8be1509a05cb7f9efe98b80f43a5861cd24c7557e98523"}, + {file = "scipy-1.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b41bc822679ad1c9a5f023bc93f6d0543129ca0f37c1ce294dd9d386f0a21096"}, + {file = "scipy-1.9.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:90453d2b93ea82a9f434e4e1cba043e779ff67b92f7a0e85d05d286a3625df3c"}, + {file = "scipy-1.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c06e62a390a9167da60bedd4575a14c1f58ca9dfde59830fc42e5197283dab"}, + {file = "scipy-1.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abaf921531b5aeaafced90157db505e10345e45038c39e5d9b6c7922d68085cb"}, + {file = "scipy-1.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:06d2e1b4c491dc7d8eacea139a1b0b295f74e1a1a0f704c375028f8320d16e31"}, + {file = "scipy-1.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5a04cd7d0d3eff6ea4719371cbc44df31411862b9646db617c99718ff68d4840"}, + {file = "scipy-1.9.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:545c83ffb518094d8c9d83cce216c0c32f8c04aaf28b92cc8283eda0685162d5"}, + {file = "scipy-1.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d54222d7a3ba6022fdf5773931b5d7c56efe41ede7f7128c7b1637700409108"}, + {file = "scipy-1.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cff3a5295234037e39500d35316a4c5794739433528310e117b8a9a0c76d20fc"}, + {file = "scipy-1.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:2318bef588acc7a574f5bfdff9c172d0b1bf2c8143d9582e05f878e580a3781e"}, + {file = "scipy-1.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d644a64e174c16cb4b2e41dfea6af722053e83d066da7343f333a54dae9bc31c"}, + {file = "scipy-1.9.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:da8245491d73ed0a994ed9c2e380fd058ce2fa8a18da204681f2fe1f57f98f95"}, + {file = "scipy-1.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4db5b30849606a95dcf519763dd3ab6fe9bd91df49eba517359e450a7d80ce2e"}, + {file = "scipy-1.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c68db6b290cbd4049012990d7fe71a2abd9ffbe82c0056ebe0f01df8be5436b0"}, + {file = "scipy-1.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:5b88e6d91ad9d59478fafe92a7c757d00c59e3bdc3331be8ada76a4f8d683f58"}, + {file = "scipy-1.9.3.tar.gz", hash = "sha256:fbc5c05c85c1a02be77b1ff591087c83bc44579c6d2bd9fb798bb64ea5e1a027"}, ] [package.dependencies] -numpy = ">=1.19.5,<1.27.0" +numpy = ">=1.18.5,<1.26.0" [package.extras] -dev = ["click", "doit (>=0.36.0)", "flake8", "mypy", "pycodestyle", "pydevtool", "rich-click", "typing_extensions"] -doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-design (>=0.2.0)"] -test = ["asv", "gmpy2", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +dev = ["flake8", "mypy", "pycodestyle", "typing_extensions"] +doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-panels (>=0.5.2)", "sphinx-tabs"] +test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] [[package]] name = "setuptools" -version = "69.0.2" +version = "69.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.0.2-py3-none-any.whl", hash = "sha256:1e8fdff6797d3865f37397be788a4e3cba233608e9b509382a2777d25ebde7f2"}, - {file = "setuptools-69.0.2.tar.gz", hash = "sha256:735896e78a4742605974de002ac60562d286fa8051a7e2299445e8e8fbb01aa6"}, + {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, + {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -997,17 +992,16 @@ test = ["flake8", "mypy", "pytest"] [[package]] name = "sphinxcontrib-matlabdomain" -version = "0.13.0" +version = "0.21.5" description = "Sphinx \"matlabdomain\" extension" optional = false python-versions = "*" files = [ - {file = "sphinxcontrib-matlabdomain-0.13.0.tar.gz", hash = "sha256:b5e2b3ae54e50876f34a20344bac2d1c0f8b7d882787da5cc6cf9ca6feaa4c14"}, - {file = "sphinxcontrib_matlabdomain-0.13.0-py3-none-any.whl", hash = "sha256:d7ed4496923849dfc43530693207f1cd159401dd1cef17d6cef10065b346621e"}, + {file = "sphinxcontrib-matlabdomain-0.21.5.tar.gz", hash = "sha256:b203e5bbe164429ac758a530ef020076749c94c8d871807153159c3186da8330"}, + {file = "sphinxcontrib_matlabdomain-0.21.5-py3-none-any.whl", hash = "sha256:161e9fb824c0084d4da6ef084e6b11cfa036e01fa7bcb12cebaa69b793e4ef5e"}, ] [package.dependencies] -future = ">=0.16.0" Pygments = ">=2.0.1" Sphinx = ">=4.0.0" @@ -1041,16 +1035,6 @@ files = [ lint = ["docutils-stubs", "flake8", "mypy"] test = ["pytest"] -[[package]] -name = "termcolor" -version = "1.1.0" -description = "ANSII Color formatting for output in terminal." -optional = false -python-versions = "*" -files = [ - {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, -] - [[package]] name = "toml" version = "0.10.2" @@ -1075,17 +1059,18 @@ files = [ [[package]] name = "urllib3" -version = "2.1.0" +version = "2.2.1" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.1.0-py3-none-any.whl", hash = "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3"}, - {file = "urllib3-2.1.0.tar.gz", hash = "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"}, + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, ] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -1107,20 +1092,20 @@ test = ["pytest", "pytest-cov"] [[package]] name = "zipp" -version = "3.17.0" +version = "3.18.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.17.0-py3-none-any.whl", hash = "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31"}, - {file = "zipp-3.17.0.tar.gz", hash = "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"}, + {file = "zipp-3.18.1-py3-none-any.whl", hash = "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b"}, + {file = "zipp-3.18.1.tar.gz", hash = "sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [metadata] lock-version = "2.0" -python-versions = ">=3.8,<=3.11" -content-hash = "b4fdb5b852b5d932b8899848b94dfbc118d12b11d53e1782e72aecf5f0c293fc" +python-versions = ">=3.8,<=3.12" +content-hash = "d5e5bf221e17dd550fa3411f510dc5df546aac9dd2b97a40e98fd2cd8fe834c2" diff --git a/pypet2bids/pypet2bids/convert_pmod_to_blood.py b/pypet2bids/pypet2bids/convert_pmod_to_blood.py index ecda62ed..00bbf1dc 100644 --- a/pypet2bids/pypet2bids/convert_pmod_to_blood.py +++ b/pypet2bids/pypet2bids/convert_pmod_to_blood.py @@ -9,6 +9,7 @@ | *Authors: Anthony Galassi* | *Copyright: OpenNeuroPET team* """ + import json import logging import textwrap @@ -30,7 +31,8 @@ except ModuleNotFoundError: import pypet2bids.helper_functions as helper_functions -epilog = textwrap.dedent(''' +epilog = textwrap.dedent( + """ example usage: @@ -39,9 +41,11 @@ convert-pmod-to-blood --whole blood-path wholeblood.bld --parent-fraction parentfraction.bld --outputh-path sub-01/pet For more extensive examples rerun this program with the --show-example flag -''') +""" +) -example1 = textwrap.dedent(''' +example1 = textwrap.dedent( + """ Usage examples are below that verbosely describe and show the input consumed an the output generated by this program. Additonal arguments/fields are passed via the kwargs flag in key value pairs. @@ -93,7 +97,8 @@ } -''') +""" +) def cli(): @@ -110,32 +115,38 @@ def cli(): :param json: create a json sidecar/data-dictionary file along with output tsv's, default is set to True :param engine: engine used to read excel files, ignore this option as it will most likely be deprecated in the future :param kwargs: additional key pair arguments one wishes to include, such as extra entries for plasma or blood PET BIDS - fields that aren't in PMOD blood files + fields that aren't in PMOD blood files :param show-examples: shows an example of how to run this module as well as the outputs :return: collected arguments :rtype: argparse.namespace + """ - parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Converts PMOD blood files to PET BIDS compliant tsv's and json " + "_blood.* files", + ) parser.add_argument( "--whole-blood-path", - '-w', + "-w", help="Path to pmod whole blood file.", required=False, - type=Path + type=Path, ) parser.add_argument( "--parent-fraction-path", "-f", help="Path to pmod parent fraction path.", required=False, - type=Path + type=Path, ) parser.add_argument( "--plasma-activity-path", - "-p", help="Path to pmod plasma file.", + "-p", + help="Path to pmod plasma file.", required=False, type=Path, - default=None + default=None, ) parser.add_argument( "--output-path", @@ -144,35 +155,35 @@ def cli(): BIDS path containing subject id and session id those values will be extracted an used to name the output files.""", type=Path, - default=None + default=None, ) parser.add_argument( "--json", "-j", help="Output a json data dictionary along with tsv files (default True)", default=True, - type=bool + type=bool, ) parser.add_argument( "--engine", "-e", help="Engine for loading PMOD files, see options for pandas.read_excel. Defaults to None.", - default='', - type=str + default="", + type=str, ) parser.add_argument( - '--kwargs', - '-k', - nargs='*', + "--kwargs", + "-k", + nargs="*", action=helper_functions.ParseKwargs, - help="Pass additional arguments not enumerated in this help menu, see documentation online" + - " for more details.", - default={} + help="Pass additional arguments not enumerated in this help menu, see documentation online" + + " for more details.", + default={}, ) parser.add_argument( - '--show-examples', + "--show-examples", help="Show additional examples (verbose) of how to use this interface.", - action='store_true' + action="store_true", ) args = parser.parse_args() @@ -195,9 +206,9 @@ def type_cast_cli_input(kwarg_arg): return var except (ValueError, SyntaxError): # try truthy evals if the input doesn't evalute as a python type listed in the try statement - if kwarg_arg.lower() in ['true', 't', 'yes']: + if kwarg_arg.lower() in ["true", "t", "yes"]: return True - elif kwarg_arg.lower() in ['false', 'f', 'no']: + elif kwarg_arg.lower() in ["false", "f", "no"]: return False else: return kwarg_arg @@ -223,18 +234,19 @@ class PmodToBlood: """ def __init__( - self, - whole_blood_activity: Path, - parent_fraction: Path = Path(), - plasma_activity: Path = Path(), - output_path: Path = None, - output_json: bool = False, - engine='', - **kwargs): + self, + whole_blood_activity: Path, + parent_fraction: Path = Path(), + plasma_activity: Path = Path(), + output_path: Path = None, + output_json: bool = False, + engine="", + **kwargs, + ): if kwargs: try: - self.kwargs = kwargs['kwargs'] + self.kwargs = kwargs["kwargs"] except KeyError: self.kwargs = kwargs else: @@ -251,46 +263,62 @@ def __init__( self.units = None self.engine = engine - # if given an output name run with that, otherwise we construct a name from the parent path the .bld files were + # if given an output name run with that, otherwise we construct a name from the parent path the .bld files were # found at. if output_path: self.output_path = Path(output_path) if not self.output_path.is_dir(): - raise FileNotFoundError(f"The output_path {output_path} must be an existing directory.") + raise FileNotFoundError( + f"The output_path {output_path} must be an existing directory." + ) else: self.output_path = Path(whole_blood_activity).parent # check the output name for subject and session id - if helper_functions.collect_bids_part('sub', str(self.output_path)): - self.subject_id = helper_functions.collect_bids_part('sub', self.output_path) + if helper_functions.collect_bids_part("sub", str(self.output_path)): + self.subject_id = helper_functions.collect_bids_part( + "sub", self.output_path + ) else: print("Subject id not found in output_path, checking key pair input.") - self.subject_id = self.kwargs.get('subject_id', '') + self.subject_id = self.kwargs.get("subject_id", "") - if helper_functions.collect_bids_part('ses', str(self.output_path)): - self.session_id = helper_functions.collect_bids_part('ses', self.output_path) + if helper_functions.collect_bids_part("ses", str(self.output_path)): + self.session_id = helper_functions.collect_bids_part( + "ses", self.output_path + ) else: print("Session id not found in output_path, checking key pair input.") - self.session_id = self.kwargs.get('session_id', '') + self.session_id = self.kwargs.get("session_id", "") self.output_json = output_json self.auto_sampled = [] self.manually_sampled = [] self.blood_series = {} - self.duplicates = {} # list of times that are duplicated across manual and automatic samples + self.duplicates = ( + {} + ) # list of times that are duplicated across manual and automatic samples if whole_blood_activity.is_file(): - self.blood_series['whole_blood_activity'] = self.load_pmod_file(whole_blood_activity, engine=self.engine) + self.blood_series["whole_blood_activity"] = self.load_pmod_file( + whole_blood_activity, engine=self.engine + ) if parent_fraction.is_file(): - self.blood_series['parent_fraction'] = self.load_pmod_file(parent_fraction, engine=self.engine) + self.blood_series["parent_fraction"] = self.load_pmod_file( + parent_fraction, engine=self.engine + ) self.multiply_plasma = False # plasma activity is not required, but is used if provided if plasma_activity.is_file(): - self.blood_series['plasma_activity'] = self.load_pmod_file(plasma_activity, engine=self.engine) - if 'whole' in str.lower(plasma_activity.name) and 'ratio' in str.lower(plasma_activity.name): + self.blood_series["plasma_activity"] = self.load_pmod_file( + plasma_activity, engine=self.engine + ) + if "whole" in str.lower(plasma_activity.name) and "ratio" in str.lower( + plasma_activity.name + ): self.multiply_plasma = True # one may encounter data collected manually and/or automatically, we vary our logic depending on the case @@ -304,10 +332,10 @@ def __init__( self.data_collection[blood_sample] = kwargs.get(var) for measure, collection_method in self.data_collection.items(): - if collection_method == 'manual': - self.manually_sampled.append({'name': measure}) - if collection_method == 'automatic': - self.auto_sampled.append({'name': measure}) + if collection_method == "manual": + self.manually_sampled.append({"name": measure}) + if collection_method == "automatic": + self.auto_sampled.append({"name": measure}) # scale time to seconds rename columns self.scale_time_rename_columns() @@ -320,7 +348,7 @@ def __init__( # remove duplicates from automatically sampled data, this data will then be written out to manual versions # of each recording - #self.deduplicate() + # self.deduplicate() self.write_out_tsvs() @@ -328,7 +356,7 @@ def __init__( self.write_out_jsons() @staticmethod - def load_pmod_file(pmod_blood_file: Path, engine=''): + def load_pmod_file(pmod_blood_file: Path, engine=""): """ Loads a pmod .bld blood file in with pandas. @@ -342,7 +370,9 @@ def load_pmod_file(pmod_blood_file: Path, engine=''): if pmod_blood_file.is_file() and pmod_blood_file.exists(): loaded_file = helper_functions.open_meta_data(pmod_blood_file) # drop unnamed columns - loaded_file = loaded_file.loc[:, ~loaded_file.columns.str.contains('^Unnamed')] + loaded_file = loaded_file.loc[ + :, ~loaded_file.columns.str.contains("^Unnamed") + ] return loaded_file else: raise FileNotFoundError(str(pmod_blood_file)) @@ -362,9 +392,11 @@ def check_time_info(self): row_lengths[key] = len(bld_data) if len(set(row_lengths.values())) > 1: - err_message = f"Sampling method for all PMOD blood files (.bld) given as " \ - f"{list(set(self.data_collection.values()))[0]} must be of the same dimensions" \ - f" row-wise!\n" + err_message = ( + f"Sampling method for all PMOD blood files (.bld) given as " + f"{list(set(self.data_collection.values()))[0]} must be of the same dimensions" + f" row-wise!\n" + ) for key, value in row_lengths.items(): err_message += f"{key} file has {value} rows\n" @@ -373,93 +405,124 @@ def check_time_info(self): raise Exception(err_message) # lastly make sure the same time points exist across each input file/dataframe - whole_blood_activity = self.blood_series.pop('whole_blood_activity') + whole_blood_activity = self.blood_series.pop("whole_blood_activity") for key, dataframe in self.blood_series.items(): try: - assert whole_blood_activity['time'].equals(dataframe['time']) + assert whole_blood_activity["time"].equals(dataframe["time"]) except AssertionError: - raise AssertionError(f"Time(s) must have same values between input files, check time columns.") + raise AssertionError( + f"Time(s) must have same values between input files, check time columns." + ) # if it all checks out put the whole blood activity back into our blood series object - self.blood_series['whole_blood_activity'] = whole_blood_activity + self.blood_series["whole_blood_activity"] = whole_blood_activity # checks to make sure that an auto-sampled file has more entries in it than a manually sampled file, # John Henry must lose. - elif len(self.blood_series) >= 2 and len(set(self.data_collection.values())) > 1: + elif ( + len(self.blood_series) >= 2 and len(set(self.data_collection.values())) > 1 + ): # check to make sure auto sampled .bld files have more entries than none autosampled compare_lengths = [] for key, sampling_type in self.data_collection.items(): compare_lengths.append( - {'name': key, 'sampling_type': sampling_type, 'sample_length': len(self.blood_series[key])}) + { + "name": key, + "sampling_type": sampling_type, + "sample_length": len(self.blood_series[key]), + } + ) auto_sampled, manually_sampled = [], [] for each in compare_lengths: - if 'auto' in str.lower(each['sampling_type']): + if "auto" in str.lower(each["sampling_type"]): auto_sampled.append(each) - elif 'manual' in str.lower(each['sampling_type']): + elif "manual" in str.lower(each["sampling_type"]): manually_sampled.append(each) for auto in auto_sampled: for manual in manually_sampled: - if auto['sample_length'] < manual['sample_length']: + if auto["sample_length"] < manual["sample_length"]: logging.warning( f"Autosampled .bld input for {auto['name']} has {auto['sample_length']} rows" f" and Manually sampled input has {manual['sample_length']} rows. Autosampled blood " - f"files should have more rows than manually sampled input files. Check .bld inputs.") + f"files should have more rows than manually sampled input files. Check .bld inputs." + ) # we want to make sure that in the case of mixed data we don't include manual time points in the autosampled # curves, first we find duplicates. - whole_blood_activity = self.blood_series.get('whole_blood_activity', pandas.DataFrame()) - plasma_activity = self.blood_series.get('plasma_activity', pandas.DataFrame()) - parent_fraction = self.blood_series.get('parent_fraction', pandas.DataFrame()) + whole_blood_activity = self.blood_series.get( + "whole_blood_activity", pandas.DataFrame() + ) + plasma_activity = self.blood_series.get( + "plasma_activity", pandas.DataFrame() + ) + parent_fraction = self.blood_series.get( + "parent_fraction", pandas.DataFrame() + ) duplicates = {} # wp is short for whole blood and plasma activity, we're just using wp to make our conditional statements # look a bit cleaner while retaining the namespace from which activity series come from. Hanging onto these # names in this dictionary makes creating manual files later during de-duplication wp = { - 'whole_blood_activity': whole_blood_activity, - 'plasma_activity': plasma_activity - } + "whole_blood_activity": whole_blood_activity, + "plasma_activity": plasma_activity, + } # case 1 whole blood time == plasma time && parent fraction time exists - if len(wp['whole_blood_activity']) == len(wp['plasma_activity']) and parent_fraction: + if ( + len(wp["whole_blood_activity"]) == len(wp["plasma_activity"]) + and parent_fraction + ): # get ready for duplicates for key in wp.keys(): - if self.data_collection.get(key) == 'automatic': + if self.data_collection.get(key) == "automatic": duplicates[key] = [] for activity_series in duplicates.keys(): - for pf_time in parent_fraction.get('time', []): - for activity_time in wp.get(key).get('time', []): + for pf_time in parent_fraction.get("time", []): + for activity_time in wp.get(key).get("time", []): if numpy.isclose(pf_time, activity_time, atol=self.atol): duplicates[activity_series].append(activity_time) # case 2 whole blood time > plasma time && parent fraction time exists - elif len(wp['whole_blood_activity']) > len(wp['plasma_activity']) and parent_fraction is not None: + elif ( + len(wp["whole_blood_activity"]) > len(wp["plasma_activity"]) + and parent_fraction is not None + ): # get ready for duplicates for key in wp.keys(): - if self.data_collection.get(key) == 'automatic': + if self.data_collection.get(key) == "automatic": duplicates[key] = [] - - if len(wp['plasma_activity']) == (len(parent_fraction) - 1): - if parent_fraction['time'][0] == wp['whole_blood_activity']['time'][0] and parent_fraction['time'][0] == 0: + if len(wp["plasma_activity"]) == (len(parent_fraction) - 1): + if ( + parent_fraction["time"][0] + == wp["whole_blood_activity"]["time"][0] + and parent_fraction["time"][0] == 0 + ): if self.multiply_plasma: - wp['plasma_activity'].loc[-1] = [0, 1] + wp["plasma_activity"].loc[-1] = [0, 1] else: - wp['plasma_activity'].loc[-1] = [0, 0] - - wp['plasma_activity'].index = wp['plasma_activity'].index + 1 - wp['plasma_activity'] = wp['plasma_activity'].sort_values('time').reset_index(drop=True) - self.blood_series['plasma_activity'] = wp['plasma_activity'] + wp["plasma_activity"].loc[-1] = [0, 0] + + wp["plasma_activity"].index = wp["plasma_activity"].index + 1 + wp["plasma_activity"] = ( + wp["plasma_activity"] + .sort_values("time") + .reset_index(drop=True) + ) + self.blood_series["plasma_activity"] = wp["plasma_activity"] else: - raise Exception(f"Parent fraction and plasma times do not match, we cannot figure out which " - f"time points to extract in whole blood.") + raise Exception( + f"Parent fraction and plasma times do not match, we cannot figure out which " + f"time points to extract in whole blood." + ) for activity_series in duplicates.keys(): - for pf_time in parent_fraction.get('time', []): - for df_time in wp.get(key).get('time', []): + for pf_time in parent_fraction.get("time", []): + for df_time in wp.get(key).get("time", []): if numpy.isclose(pf_time, df_time, atol=self.atol): duplicates[activity_series].append(df_time) @@ -481,64 +544,97 @@ def check_time_info(self): # series_is_auto_sampled = False # we only want to remove rows if they exist in autosampled data, but are clearly manually sampled - if self.data_collection.get(activity_series) == 'automatic': + if self.data_collection.get(activity_series) == "automatic": removed_rows = pandas.DataFrame( - columns=self.blood_series.get(activity_series, pandas.DataFrame()).columns) + columns=self.blood_series.get( + activity_series, pandas.DataFrame() + ).columns + ) for time in duplicate_times: - index = helper_functions.get_coordinates_containing(time, self.blood_series[activity_series], - exact=False) + index = helper_functions.get_coordinates_containing( + time, self.blood_series[activity_series], exact=False + ) if index: row = index[0][0] - dropped_row = helper_functions.drop_row(self.blood_series[activity_series], row) + dropped_row = helper_functions.drop_row( + self.blood_series[activity_series], row + ) # using the loc indexer, append the series to the end of the df removed_rows.loc[len(removed_rows)] = dropped_row # update the blood series object with the popped/dropped row new_string = f"{activity_series}_manually_popped" - self.blood_series[new_string] = removed_rows.sort_values('time').reset_index(drop=True) - self.manually_sampled.append({'name': new_string}) - self.data_collection[new_string] = 'manual' + self.blood_series[new_string] = removed_rows.sort_values( + "time" + ).reset_index(drop=True) + self.manually_sampled.append({"name": new_string}) + self.data_collection[new_string] = "manual" if len(self.blood_series[new_string]) == (len(parent_fraction) - 1): - if (self.blood_series['parent_fraction']['time'][0] != self.blood_series[new_string]['time'][0] - and parent_fraction['time'][0] == 0): + if ( + self.blood_series["parent_fraction"]["time"][0] + != self.blood_series[new_string]["time"][0] + and parent_fraction["time"][0] == 0 + ): self.blood_series[new_string].loc[-1] = [0, 0] - self.blood_series[new_string].index = self.blood_series[new_string].index + 1 - self.blood_series[new_string] = self.blood_series[new_string].sort_values('time').reset_index(drop=True) + self.blood_series[new_string].index = ( + self.blood_series[new_string].index + 1 + ) + self.blood_series[new_string] = ( + self.blood_series[new_string] + .sort_values("time") + .reset_index(drop=True) + ) else: - raise Exception(f"Parent fraction and plasma times do not match, we cannot figure out which " - f"time points to extract in whole blood.") + raise Exception( + f"Parent fraction and plasma times do not match, we cannot figure out which " + f"time points to extract in whole blood." + ) # if provided with the plasma ratio we multiply the whole blood by it, math time if self.multiply_plasma: # get our plasma activity, we get the manually popped if it exists pa = self.blood_series.get( - 'plasma_activity_manually_popped', - self.blood_series['plasma_activity']) - - if self.blood_series.get('whole_blood_activity_manually_popped', pandas.DataFrame()).shape[0] != 0: - - wba = self.blood_series['whole_blood_activity_manually_popped'] - new_plasma = wba['whole_blood_radioactivity'] * self.blood_series['plasma_activity']['plasma_radioactivity'] - - temp = pandas.DataFrame({'time': self.blood_series['plasma_activity']['time'], - 'plasma_radioactivity': new_plasma}) + "plasma_activity_manually_popped", self.blood_series["plasma_activity"] + ) + + if ( + self.blood_series.get( + "whole_blood_activity_manually_popped", pandas.DataFrame() + ).shape[0] + != 0 + ): + + wba = self.blood_series["whole_blood_activity_manually_popped"] + new_plasma = ( + wba["whole_blood_radioactivity"] + * self.blood_series["plasma_activity"]["plasma_radioactivity"] + ) + + temp = pandas.DataFrame( + { + "time": self.blood_series["plasma_activity"]["time"], + "plasma_radioactivity": new_plasma, + } + ) - self.blood_series['plasma_activity'] = temp + self.blood_series["plasma_activity"] = temp else: - self.blood_series['plasma_activity'] = self.blood_series['plasma_activity'] * self.blood_series['whole_blood'] - + self.blood_series["plasma_activity"] = ( + self.blood_series["plasma_activity"] + * self.blood_series["whole_blood"] + ) def scale_time_rename_columns(self): """ - Scales time info if it's not in seconds and renames dataframe column to 'time' instead of given column name in + Scales time info if it's not in seconds and renames dataframe column to 'time' instead of given column name in .bld file. Renames radioactivity column to BIDS compliant column name if it's in units Bq/cc or Bq/mL. """ # scale time info to seconds if it's minutes @@ -547,66 +643,99 @@ def scale_time_rename_columns(self): radioactivity_column_header_name = [] parent_fraction_column_header_name = [] time_scalar = 1.0 - time_column_header_name = [header for header in list(dataframe.columns) if 'sec' in str.lower(header)] + time_column_header_name = [ + header + for header in list(dataframe.columns) + if "sec" in str.lower(header) + ] if not time_column_header_name: - time_column_header_name = [header for header in list(dataframe.columns) if 'min' in str.lower(header)] + time_column_header_name = [ + header + for header in list(dataframe.columns) + if "min" in str.lower(header) + ] if time_column_header_name: time_scalar = 60.0 if time_column_header_name and len(time_column_header_name) == 1: - dataframe.rename(columns={time_column_header_name[0]: 'time'}, inplace=True) + dataframe.rename( + columns={time_column_header_name[0]: "time"}, inplace=True + ) else: - raise Exception("Unable to locate time column in blood file, make sure input files are formatted " - "to include a single time column in minutes or seconds.") + raise Exception( + "Unable to locate time column in blood file, make sure input files are formatted " + "to include a single time column in minutes or seconds." + ) # scale the time column to seconds - dataframe['time'] = dataframe['time'] * time_scalar + dataframe["time"] = dataframe["time"] * time_scalar self.blood_series[name] = dataframe # locate parent fraction column - parent_fraction_column_header_name = [header for header in dataframe.columns if - 'parent' in str.lower(header)] + parent_fraction_column_header_name = [ + header for header in dataframe.columns if "parent" in str.lower(header) + ] parent_fraction_is_suspicous = False if len(parent_fraction_column_header_name) >= 1: pf = str(parent_fraction_column_header_name[0]).lower() - if 'bq' in pf or 'ml' in pf: - logging.warning(f"Found {parent_fraction_column_header_name[0]} in {name} input file, this column " - f"must be unitless") + if "bq" in pf or "ml" in pf: + logging.warning( + f"Found {parent_fraction_column_header_name[0]} in {name} input file, this column " + f"must be unitless" + ) parent_fraction_is_suspicous = True if not parent_fraction_column_header_name or parent_fraction_is_suspicous: # locate radioactivity column - radioactivity_column_header_name = [header for header in dataframe.columns if - 'bq' and 'cc' in str.lower(header)] + radioactivity_column_header_name = [ + header + for header in dataframe.columns + if "bq" and "cc" in str.lower(header) + ] # run through radio updating conversion if not percent parent if radioactivity_column_header_name and len(time_column_header_name) == 1: - sub_ml_for_cc = re.sub('cc', 'mL', radioactivity_column_header_name[0]) - extracted_units = re.search(r'\[(.*?)\]', sub_ml_for_cc) + sub_ml_for_cc = re.sub("cc", "mL", radioactivity_column_header_name[0]) + extracted_units = re.search(r"\[(.*?)\]", sub_ml_for_cc) second_column_name = None - if 'plasma' in str.lower(radioactivity_column_header_name[0]) or \ - ('bq/cc' in str.lower(radioactivity_column_header_name[0]) and name == 'plasma_activity'): - second_column_name = 'plasma_radioactivity' - - if name == 'whole_blood_activity': - if 'whole' in str.lower(radioactivity_column_header_name[0]) or 'blood' in str.lower( - radioactivity_column_header_name[0]): - second_column_name = 'whole_blood_radioactivity' + if "plasma" in str.lower(radioactivity_column_header_name[0]) or ( + "bq/cc" in str.lower(radioactivity_column_header_name[0]) + and name == "plasma_activity" + ): + second_column_name = "plasma_radioactivity" + + if name == "whole_blood_activity": + if "whole" in str.lower( + radioactivity_column_header_name[0] + ) or "blood" in str.lower(radioactivity_column_header_name[0]): + second_column_name = "whole_blood_radioactivity" if second_column_name: - dataframe = dataframe.rename({radioactivity_column_header_name[0]: second_column_name}, - axis='columns') + dataframe = dataframe.rename( + {radioactivity_column_header_name[0]: second_column_name}, + axis="columns", + ) if extracted_units: self.units = extracted_units.group(1) else: raise Exception( "Unable to determine radioactivity entries from .bld column name. Column name/units must be in " - "Bq/cc or Bq/mL") + "Bq/cc or Bq/mL" + ) # if percent parent rename column accordingly - if parent_fraction_column_header_name and len( - parent_fraction_column_header_name) == 1 and name != 'plasma_activity': - dataframe = dataframe.rename({parent_fraction_column_header_name[0]: 'metabolite_parent_fraction'}, - axis='columns') + if ( + parent_fraction_column_header_name + and len(parent_fraction_column_header_name) == 1 + and name != "plasma_activity" + ): + dataframe = dataframe.rename( + { + parent_fraction_column_header_name[ + 0 + ]: "metabolite_parent_fraction" + }, + axis="columns", + ) self.blood_series[name] = dataframe.copy(deep=True) def ask_recording_type(self, recording: str): @@ -620,22 +749,27 @@ def ask_recording_type(self, recording: str): :rtype: None """ how = None - while how != 'a' or how != 'm': - how = input(f"How was the {recording} data sampled?:\nEnter A for automatically or M for manually\n") - if str.lower(how) == 'm': - self.data_collection[recording] = 'manual' + while how != "a" or how != "m": + how = input( + f"How was the {recording} data sampled?:\nEnter A for automatically or M for manually\n" + ) + if str.lower(how) == "m": + self.data_collection[recording] = "manual" break - elif str.lower(how) == 'a': - self.data_collection[recording] = 'automatic' + elif str.lower(how) == "a": + self.data_collection[recording] = "automatic" break - elif str.lower(how) == 'y': - self.data_collection[recording] = 'manual' + elif str.lower(how) == "y": + self.data_collection[recording] = "manual" warnings.warn( f"Received {how} as input, assuming input recieved from cli w/ '-y' option on bash/zsh etc, " - f"defaulting to manual input") + f"defaulting to manual input" + ) break else: - print(f"You entered {how}; please enter either M or A to exit this prompt") + print( + f"You entered {how}; please enter either M or A to exit this prompt" + ) def write_out_tsvs(self): """ @@ -648,34 +782,40 @@ def write_out_tsvs(self): # first we combine the various blood datas into one or two dataframes, the autosampled data goes into a # recording_autosample, and the manually sampled data goes into a recording_manual if they exist if self.subject_id: - file_path = join(self.output_path, self.subject_id + '_') + file_path = join(self.output_path, self.subject_id + "_") if self.session_id: - file_path += self.session_id + '_' - manual_path = file_path + 'recording-manual_blood.tsv' - automatic_path = file_path + 'recording-automatic_blood.tsv' + file_path += self.session_id + "_" + manual_path = file_path + "recording-manual_blood.tsv" + automatic_path = file_path + "recording-automatic_blood.tsv" else: - manual_path = join(self.output_path, 'recording-manual_blood.tsv') - automatic_path = join(self.output_path, 'recording-automatic_blood.tsv') + manual_path = join(self.output_path, "recording-manual_blood.tsv") + automatic_path = join(self.output_path, "recording-automatic_blood.tsv") # first combine autosampled data if self.auto_sampled: - first_auto_sampled = self.blood_series[self.auto_sampled.pop()['name']] + first_auto_sampled = self.blood_series[self.auto_sampled.pop()["name"]] for remaining_auto in self.auto_sampled: - remaining_auto = self.blood_series[remaining_auto['name']] - column_difference = remaining_auto.columns.difference(first_auto_sampled.columns) + remaining_auto = self.blood_series[remaining_auto["name"]] + column_difference = remaining_auto.columns.difference( + first_auto_sampled.columns + ) for column in list(column_difference): first_auto_sampled[column] = remaining_auto[column] - first_auto_sampled.to_csv(automatic_path, sep='\t', index=False) + first_auto_sampled.to_csv(automatic_path, sep="\t", index=False) # combine any additional manually sampled dataframes if self.manually_sampled: - first_manually_sampled = self.blood_series[self.manually_sampled.pop()['name']] + first_manually_sampled = self.blood_series[ + self.manually_sampled.pop()["name"] + ] for remaining_manual in self.manually_sampled: - remaining_manual = self.blood_series[remaining_manual['name']] - column_difference = remaining_manual.columns.difference(first_manually_sampled.columns) + remaining_manual = self.blood_series[remaining_manual["name"]] + column_difference = remaining_manual.columns.difference( + first_manually_sampled.columns + ) for column in list(column_difference): first_manually_sampled[column] = remaining_manual[column] - first_manually_sampled.to_csv(manual_path, sep='\t', index=False) + first_manually_sampled.to_csv(manual_path, sep="\t", index=False) def write_out_jsons(self): """ @@ -685,104 +825,124 @@ def write_out_jsons(self): :rtype: None """ if self.subject_id: - file_path = join(self.output_path, self.subject_id + '_') + file_path = join(self.output_path, self.subject_id + "_") if self.session_id: - file_path += self.session_id + '_' - file_path += 'blood.json' + file_path += self.session_id + "_" + file_path += "blood.json" else: - file_path = join(self.output_path, 'blood.json') + file_path = join(self.output_path, "blood.json") side_car_template = { "Time": { "Description": "Time in relation to time zero defined by the _pet.json", - "Units": "s" + "Units": "s", }, "whole_blood_radioactivity": { - "Description": 'Radioactivity in whole blood samples.', - "Units": self.units + "Description": "Radioactivity in whole blood samples.", + "Units": self.units, }, "metabolite_parent_fraction": { - "Description": 'Parent fraction of the radiotracer', - "Units": 'arbitrary' + "Description": "Parent fraction of the radiotracer", + "Units": "arbitrary", }, } - if self.kwargs.get('MetaboliteMethod', None): - side_car_template['MetaboliteMethod'] = self.kwargs.get('MetaboliteMethod'), - elif self.kwargs.get('MetaboliteRecoveryCorrectionApplied', None): - side_car_template['MetaboliteRecoveryCorrectionApplied'] = self.kwargs.get( - 'MetaboliteRecoveryCorrectionApplied') - elif self.kwargs.get('DispersionCorrected', None): - side_car_template['DispersionCorrected'] = self.kwargs.get('DispersionCorrected') - - side_car_template['MetaboliteAvail'] = True - - if self.kwargs.get('MetaboliteMethod', None): - side_car_template['MetaboliteMethod'] = self.kwargs.get('MetaboliteMethod') + if self.kwargs.get("MetaboliteMethod", None): + side_car_template["MetaboliteMethod"] = ( + self.kwargs.get("MetaboliteMethod"), + ) + elif self.kwargs.get("MetaboliteRecoveryCorrectionApplied", None): + side_car_template["MetaboliteRecoveryCorrectionApplied"] = self.kwargs.get( + "MetaboliteRecoveryCorrectionApplied" + ) + elif self.kwargs.get("DispersionCorrected", None): + side_car_template["DispersionCorrected"] = self.kwargs.get( + "DispersionCorrected" + ) + + side_car_template["MetaboliteAvail"] = True + + if self.kwargs.get("MetaboliteMethod", None): + side_car_template["MetaboliteMethod"] = self.kwargs.get("MetaboliteMethod") else: - warnings.warn("Parent fraction is available, but MetaboliteMethod is not specified, which is not BIDS " - "compliant.") - - if self.kwargs.get('DispersionCorrected'): - side_car_template['DispersionCorrected'] = self.kwargs.get('DispersionCorrected') + warnings.warn( + "Parent fraction is available, but MetaboliteMethod is not specified, which is not BIDS " + "compliant." + ) + + if self.kwargs.get("DispersionCorrected"): + side_car_template["DispersionCorrected"] = self.kwargs.get( + "DispersionCorrected" + ) else: - warnings.warn('Parent fraction is available, but there is no information if DispersionCorrected was' + - 'applied, which is not BIDS compliant') - - if self.blood_series.get('plasma_activity', None) is type(pd.DataFrame): - side_car_template['PlasmaAvail'] = True - side_car_template['plasma_radioactivity'] = { - 'Description': 'Radioactivity in plasma samples', - 'Units': self.units + warnings.warn( + "Parent fraction is available, but there is no information if DispersionCorrected was" + + "applied, which is not BIDS compliant" + ) + + if self.blood_series.get("plasma_activity", None) is type(pd.DataFrame): + side_car_template["PlasmaAvail"] = True + side_car_template["plasma_radioactivity"] = { + "Description": "Radioactivity in plasma samples", + "Units": self.units, } - with open(file_path, 'w') as out_json: + with open(file_path, "w") as out_json: json.dump(side_car_template, out_json, indent=4) def check_for_interpolated_data(self): # check to see if there may exist (emphasis on may) interpolated plasma values - if type(self.blood_series.get('plasma_activity', None)) is not None: + if type(self.blood_series.get("plasma_activity", None)) is not None: # extract parent fraction/metabolite fraction as series from dataframes - metabolite_parent_fraction_series = pandas.Series(dtype='float64') + metabolite_parent_fraction_series = pandas.Series(dtype="float64") for name, dataframe in self.blood_series.items(): columns = dataframe.columns for entry in columns: - if 'parent' in str.lower(entry) or 'fraction' in str.lower(entry): + if "parent" in str.lower(entry) or "fraction" in str.lower(entry): metabolite_parent_fraction_series = dataframe[str(entry)] break if len(metabolite_parent_fraction_series) > 0: break # check dataframes for plasma activity - plasma_activity_series = pandas.Series(dtype='float64') + plasma_activity_series = pandas.Series(dtype="float64") for name, dataframe in self.blood_series.items(): columns = dataframe.columns for entry in columns: - if 'plasma_radioactivity' in entry: + if "plasma_radioactivity" in entry: plasma_activity_series = dataframe[str(entry)] break if len(plasma_activity_series) > 0: break - if len(plasma_activity_series) > 0 and len(metabolite_parent_fraction_series) > 0: - if len(plasma_activity_series) == len(metabolite_parent_fraction_series): - logging.warning(f"plasma_activity_series and metabolite_parent_fraction have same " - f"length {len(plasma_activity_series)}, plasma data may be interpolated, BIDS " - f"prefers raw data.") + if ( + len(plasma_activity_series) > 0 + and len(metabolite_parent_fraction_series) > 0 + ): + if len(plasma_activity_series) == len( + metabolite_parent_fraction_series + ): + logging.warning( + f"plasma_activity_series and metabolite_parent_fraction have same " + f"length {len(plasma_activity_series)}, plasma data may be interpolated, BIDS " + f"prefers raw data." + ) def check_fraction_is_fraction(self): # collect parent fractions wherever they may exist fraction_series = [] for name, dataframe in self.blood_series.items(): for column in dataframe.columns: - if 'fraction' in str(column).lower(): + if "fraction" in str(column).lower(): fraction_series.append({name: dataframe[column]}) for fraction_column in fraction_series: for name, values in fraction_column.items(): for value in values: - logging.warning(f"No, no my friend parent/metabolite fraction must be less than or equal to 1," - f" found {value} in {name}.") + logging.warning( + f"No, no my friend parent/metabolite fraction must be less than or equal to 1," + f" found {value} in {name}." + ) def main(): @@ -804,10 +964,12 @@ def main(): output_path=cli_args.output_path, output_json=cli_args.json, engine=cli_args.engine, - kwargs=cli_args.kwargs + kwargs=cli_args.kwargs, ) else: - raise Exception(f"--whole-blood-path (-w) and --parent-fraction-path (-p) are both required!") + raise Exception( + f"--whole-blood-path (-w) and --parent-fraction-path (-p) are both required!" + ) if __name__ == "__main__": diff --git a/pypet2bids/pypet2bids/dcm2niix4pet.py b/pypet2bids/pypet2bids/dcm2niix4pet.py index 1b6320c5..d68a3c98 100644 --- a/pypet2bids/pypet2bids/dcm2niix4pet.py +++ b/pypet2bids/pypet2bids/dcm2niix4pet.py @@ -11,278 +11,87 @@ | *Authors: Anthony Galassi* | *Copyright OpenNeuroPET team* """ + import pathlib import sys -import os import textwrap -from json_maj.main import JsonMAJ, load_json_or_dict +from json_maj.main import JsonMAJ from platform import system import subprocess import pandas as pd from os.path import join -from os import listdir, walk +from os import listdir, walk, environ from pathlib import Path import json import pydicom import re from tempfile import TemporaryDirectory import shutil -from dateutil import parser -from termcolor import colored import argparse import importlib -import dotenv -import logging try: import helper_functions import is_pet + from update_json_pet_file import ( + check_json, + update_json_with_dicom_value, + update_json_with_dicom_value_cli, + get_radionuclide, + check_meta_radio_inputs, + metadata_dictionaries, + get_metadata_from_spreadsheet, + ) except ModuleNotFoundError: import pypet2bids.helper_functions as helper_functions import pypet2bids.is_pet as is_pet + from pypet2bids.update_json_pet_file import ( + check_json, + update_json_with_dicom_value, + update_json_with_dicom_value_cli, + get_radionuclide, + check_meta_radio_inputs, + metadata_dictionaries, + get_metadata_from_spreadsheet, + ) logger = helper_functions.logger("pypet2bids") -# fields to check for module_folder = Path(__file__).parent.resolve() python_folder = module_folder.parent pet2bids_folder = python_folder.parent -metadata_folder = join(pet2bids_folder, 'metadata') - -try: - # collect metadata jsons in dev mode - metadata_jsons = \ - [Path(join(metadata_folder, metadata_json)) for metadata_json - in listdir(metadata_folder) if '.json' in metadata_json] -except FileNotFoundError: - metadata_jsons = \ - [Path(join(module_folder, 'metadata', metadata_json)) for metadata_json - in listdir(join(module_folder, 'metadata')) if '.json' in metadata_json] +metadata_folder = join(pet2bids_folder, "metadata") # check to see if config file exists home_dir = Path.home() -pypet2bids_config = home_dir / '.pet2bidsconfig' +pypet2bids_config = home_dir / ".pet2bidsconfig" if pypet2bids_config.exists(): # check to see if the template json var is set and valid - default_metadata_json = helper_functions.check_pet2bids_config('DEFAULT_METADATA_JSON') + default_metadata_json = helper_functions.check_pet2bids_config( + "DEFAULT_METADATA_JSON" + ) if default_metadata_json and Path(default_metadata_json).exists(): # do nothing pass -else: - # if it doesn't exist use the default one included in this library - helper_functions.modify_config_file('DEFAULT_METADATA_JSON', module_folder / 'template_json.json') - -# create a dictionary to house the PET metadata files -metadata_dictionaries = {} - -for metadata_json in metadata_jsons: - try: - with open(metadata_json, 'r') as infile: - dictionary = json.load(infile) - - metadata_dictionaries[metadata_json.name] = dictionary - except FileNotFoundError as err: - raise Exception(f"Missing pet metadata file {metadata_json} in {metadata_folder}, unable to validate metadata.") - except json.decoder.JSONDecodeError as err: - raise IOError(f"Unable to read from {metadata_json}") - - -def check_json(path_to_json, items_to_check=None, silent=False, spreadsheet_metadata={}, **additional_arguments): - """ - This method opens a json and checks to see if a set of mandatory values is present within that json, optionally it - also checks for recommended key value pairs. If fields are not present a warning is raised to the user. - - :param spreadsheet_metadata: - :type spreadsheet_metadata: - :param path_to_json: path to a json file e.g. a BIDS sidecar file created after running dcm2niix - :param items_to_check: a dictionary with items to check for within that json. If None is supplied defaults to the - PET_metadata.json contained in this repository - :param silent: Raises warnings or errors to stdout if this flag is set to True - :return: dictionary of items existence and value state, if key is True/False there exists/(does not exist) a - corresponding entry in the json the same can be said of value - """ - - if silent: - logger.disable = True else: - logger.disabled = False - - # check if path exists - path_to_json = Path(path_to_json) - if not path_to_json.exists(): - raise FileNotFoundError(path_to_json) - - # check for default argument for dictionary of items to check - if items_to_check is None: - items_to_check = metadata_dictionaries['PET_metadata.json'] - # remove blood tsv data from items to check - if items_to_check.get('blood_recording_fields', None): - items_to_check.pop('blood_recording_fields') - - # open the json - with open(path_to_json, 'r') as in_file: - json_to_check = json.load(in_file) - - # initialize warning colors and warning storage dictionary - storage = {} - flattened_spreadsheet_metadata = {} - flattened_spreadsheet_metadata.update(spreadsheet_metadata.get('nifti_json', {})) - flattened_spreadsheet_metadata.update(spreadsheet_metadata.get('blood_json', {})) - flattened_spreadsheet_metadata.update(spreadsheet_metadata.get('blood_tsv', {})) - - for requirement in items_to_check.keys(): - for item in items_to_check[requirement]: - all_good = False - if item in json_to_check.keys() and json_to_check.get(item, None) or item in additional_arguments or item in flattened_spreadsheet_metadata.keys(): - # this json has both the key and a non-blank value do nothing - all_good = True - pass - elif item in json_to_check.keys() and not json_to_check.get(item, None): - logger.warning(f"{item} present but has null value.") - storage[item] = {'key': True, 'value': False} - elif not all_good: - logger.warning(f"{item} is not present in {path_to_json}. This will have to be " - f"corrected post conversion.") - storage[item] = {'key': False, 'value': False} - - return storage - - -def update_json_with_dicom_value( - path_to_json, - missing_values, - dicom_header, - dicom2bids_json=None, - **additional_arguments -): - """ - We go through all of the missing values or keys that we find in the sidecar json and attempt to extract those - missing entities from the dicom source. This function relies on many heuristics a.k.a. many unique conditionals and - simply is what it is, hate the game not the player. - - :param path_to_json: path to the sidecar json to check - :param missing_values: dictionary output from check_json indicating missing fields and/or values - :param dicom_header: the dicom or dicoms that may contain information not picked up by dcm2niix - :param dicom2bids_json: a json file that maps dicom header entities to their corresponding BIDS entities - :return: a dictionary of sucessfully updated (written to the json file) fields and values - """ - - # load the sidecar json - sidecar_json = load_json_or_dict(str(path_to_json)) - - # purely to clean up the generated read the docs page from sphinx, otherwise the entire json appears in the - # read the docs page. - if dicom2bids_json is None: - dicom2bids_json = metadata_dictionaries['dicom2bids.json'] - - # Units gets written as Unit in older versions of dcm2niix here we check for missing Units and present Unit entity - units = missing_values.get('Units', None) - if units: try: - # Units is missing, check to see if Unit is present - if sidecar_json.get('Unit', None): - temp = JsonMAJ(path_to_json, {'Units': sidecar_json.get('Unit')}, bids_null=True) - temp.remove('Unit') - else: # we source the Units value from the dicom header and update the json - JsonMAJ(path_to_json, {'Units': dicom_header.Units}, bids_null=True) - except AttributeError: - logger.error(f"Dicom is missing Unit(s) field, are you sure this is a PET dicom?") - # pair up dicom fields with bids sidecar json field, we do this in a separate json file - # it's loaded when this script is run and stored in metadata dictionaries - dcmfields = dicom2bids_json['dcmfields'] - jsonfields = dicom2bids_json['jsonfields'] - - regex_cases = ["ReconstructionMethod", "ConvolutionKernel"] - - # strip excess characters from dcmfields - dcmfields = [re.sub('[^0-9a-zA-Z]+', '', field) for field in dcmfields] - paired_fields = {} - for index, field in enumerate(jsonfields): - paired_fields[field] = dcmfields[index] - - logger.info("Attempting to locate missing BIDS fields in dicom header") - # go through missing fields and reach into dicom to pull out values - json_updater = JsonMAJ(json_path=path_to_json, bids_null=True) - for key, value in paired_fields.items(): - missing_bids_field = missing_values.get(key, None) - # if field is missing look into dicom - if missing_bids_field and key not in additional_arguments: - # there are a few special cases that require regex splitting of the dicom entries - # into several bids sidecar entities - try: - dicom_field = getattr(dicom_header, value) - logger.info(f"FOUND {value} corresponding to BIDS {key}: {dicom_field}") - except AttributeError: - dicom_field = None - logger.info(f"NOT FOUND {value} corresponding to BIDS {key} in dicom header.") - - if dicom_field and value in regex_cases: - # if it exists get rid of it, we don't want no part of it. - if sidecar_json.get('ReconMethodName', None): - json_updater.remove('ReconstructionMethod') - if dicom_header.get('ReconstructionMethod', None): - reconstruction_method = dicom_header.ReconstructionMethod - json_updater.remove('ReconstructionMethod') - reconstruction_method = helper_functions.get_recon_method(reconstruction_method) - - json_updater.update(reconstruction_method) - - elif dicom_field: - # update json - json_updater.update({key: dicom_field}) - - # Additional Heuristics are included below - - # See if time zero is missing in json or additional args - if missing_values.get('TimeZero', None): - if missing_values.get('TimeZero')['key'] is False or missing_values.get('TimeZero')['value'] is False: - time_parser = parser - if sidecar_json.get('AcquisitionTime', None): - acquisition_time = time_parser.parse(sidecar_json.get('AcquisitionTime')).time().strftime("%H:%M:%S") - else: - acquisition_time = time_parser.parse(dicom_header['SeriesTime'].value).time().strftime("%H:%M:%S") - - json_updater.update({'TimeZero': acquisition_time}) - json_updater.remove('AcquisitionTime') - json_updater.update({'ScanStart': 0}) - else: - pass - - if missing_values.get('ScanStart', None): - if missing_values.get('ScanStart')['key'] is False or missing_values.get('ScanStart')['value'] is False: - json_updater.update({'ScanStart': 0}) - if missing_values.get('InjectionStart', None): - if missing_values.get('InjectionStart')['key'] is False \ - or missing_values.get('InjectionStart')['value'] is False: - json_updater.update({'InjectionStart': 0}) - - # check to see if units are BQML - json_updater = JsonMAJ(str(path_to_json), bids_null=True) - if json_updater.get('Units') == 'BQML': - json_updater.update({'Units': 'Bq/mL'}) - - # Add radionuclide to json - Radionuclide = get_radionuclide(dicom_header) - if Radionuclide: - json_updater.update({'TracerRadionuclide': Radionuclide}) - - # remove scandate if it exists - json_updater.remove('ScanDate') - - # after updating raise warnings to user if values in json don't match values in dicom headers, only warn! - updated_values = json.load(open(path_to_json, 'r')) - for key, value in paired_fields.items(): - try: - json_field = updated_values.get(key) - dicom_field = dicom_header.__getattr__(key) - if json_field != dicom_field: - logger.info(f"WARNING!!!! JSON Field {key} with value {json_field} does not match dicom value " - f"of {dicom_field}") - except AttributeError: - pass + shutil.copy( + Path(metadata_folder) / "template_json.json", default_metadata_json + ) + except FileNotFoundError: + shutil.copy(module_folder / "template_json.json", default_metadata_json) +else: + # if it doesn't exist use the default one included in this library + helper_functions.modify_config_file( + "DEFAULT_METADATA_JSON", module_folder / "template_json.json" + ) + # set the default metadata json to the template json included in this library in our environment so + # we don't trip up retrival of the default metadata json later + environ["DEFAULT_METADATA_JSON"] = str(module_folder / "template_json.json") + default_metadata_json = module_folder / "template_json.json" -def dicom_datetime_to_dcm2niix_time(dicom=None, date='', time=''): +def dicom_datetime_to_dcm2niix_time(dicom=None, date="", time=""): """ Dcm2niix provides the option of outputing the scan data and time into the .nii and .json filename at the time of conversion if '%t' is provided following the '-f' flag. The result is the addition of a date time string of the @@ -294,8 +103,8 @@ def dicom_datetime_to_dcm2niix_time(dicom=None, date='', time=''): :return: a datetime string that corresponds to the converted filenames from dcm2niix when used with the `-f %t` flag """ - parsed_time = '' - parsed_date = '' + parsed_time = "" + parsed_date = "" if dicom: if type(dicom) is pydicom.dataset.FileDataset: # do nothing @@ -305,8 +114,10 @@ def dicom_datetime_to_dcm2niix_time(dicom=None, date='', time=''): dicom_path = Path(dicom) dicom = pydicom.dcmread(dicom_path) except TypeError: - raise TypeError(f"dicom {dicom} must be either a pydicom.dataset.FileDataSet object or a " - f"valid path to a dicom file") + raise TypeError( + f"dicom {dicom} must be either a pydicom.dataset.FileDataSet object or a " + f"valid path to a dicom file" + ) parsed_date = dicom.StudyDate parsed_time = str(round(float(dicom.StudyTime))) @@ -317,7 +128,7 @@ def dicom_datetime_to_dcm2niix_time(dicom=None, date='', time=''): if len(parsed_time) < 6: zeros_to_pad = 6 - len(parsed_time) - parsed_time = zeros_to_pad * '0' + parsed_time + parsed_time = zeros_to_pad * "0" + parsed_time return parsed_date + parsed_time @@ -331,7 +142,7 @@ def collect_date_time_from_file_name(file_name): dcm2niix :return: a date and time object """ - date_time_string = re.search(r'(?!\_)[0-9]{14}(?=\_)', file_name) + date_time_string = re.search(r"(?!\_)[0-9]{14}(?=\_)", file_name) if date_time_string: date = date_time_string[0][0:8] time = date_time_string[0][8:] @@ -342,9 +153,17 @@ def collect_date_time_from_file_name(file_name): class Dcm2niix4PET: - def __init__(self, image_folder, destination_path=None, metadata_path=None, - metadata_translation_script=None, additional_arguments={}, file_format='%p_%i_%t_%s', - silent=False, tempdir_location=None): + def __init__( + self, + image_folder, + destination_path=None, + metadata_path=None, + metadata_translation_script=None, + additional_arguments={}, + file_format="%p_%i_%t_%s", + silent=False, + tempdir_location=None, + ): """ This class is a simple wrapper for dcm2niix and contains methods to do the following in order: - Convert a set of dicoms into .nii and .json sidecar files @@ -386,69 +205,102 @@ def __init__(self, image_folder, destination_path=None, metadata_path=None, self.dcm2niix_path = self.check_for_dcm2niix() if not self.dcm2niix_path: - raise FileNotFoundError("dcm2niix not found, this module depends on it for conversions, exiting.") + logger.error( + "dcm2niix not found, this module depends on it for conversions, exiting." + ) + sys.exit(1) # check for the version of dcm2niix - minimum_version = 'v1.0.20220720' - version_string = subprocess.run([self.dcm2niix_path, '-v'], capture_output=True) + minimum_version = "v1.0.20220720" + version_string = subprocess.run([self.dcm2niix_path, "-v"], capture_output=True) version = re.search(r"v[0-9].[0-9].{8}[0-9]", str(version_string.stdout)) if version: # compare with minimum version if version[0] < minimum_version: - logger.warning(f"Minimum version {minimum_version} of dcm2niix is recommended, found " - f"installed version {version[0]} at {self.dcm2niix_path}.") + logger.warning( + f"Minimum version {minimum_version} of dcm2niix is recommended, found " + f"installed version {version[0]} at {self.dcm2niix_path}." + ) # check if user provided a custom tempdir location self.tempdir_location = tempdir_location self.image_folder = Path(image_folder) - if destination_path: - self.destination_path = Path(destination_path) - else: - self.destination_path = self.image_folder + self.destination_folder = None # if we're provided an entire file path just us that no matter what, we're assuming the user knows what they # are doing in that case self.full_file_path_given = False - for part in self.destination_path.parts: - if '.nii' in part or '.nii.gz' in part: + + for part in Path(destination_path).parts: + if ".nii" in part or ".nii.gz" in part: self.full_file_path_given = True + self.destination_folder = Path(destination_path).parent # replace .nii and .nii.gz - self.destination_path = Path(str(self.destination_path).replace('.nii', '').replace('.gz', '')) + self.destination_path = Path( + str(destination_path).replace(".nii", "").replace(".gz", "") + ) break # replace the suffix in the destination path with '' if a non-nifti full file path is give - if Path(self.destination_path).suffix: + if Path(destination_path).suffix: self.full_file_path_given = True - self.destination_path = Path(self.destination_path).with_suffix('') + self.destination_folder = Path(destination_path).parent + self.destination_path = Path(destination_path).with_suffix("") - # extract PET filename parts from destination path if given - self.subject_id = helper_functions.collect_bids_part('sub', str(self.destination_path)) - self.session_id = helper_functions.collect_bids_part('ses', str(self.destination_path)) - self.task = helper_functions.collect_bids_part('task', str(self.destination_path)) - self.tracer = helper_functions.collect_bids_part('trc', str(self.destination_path)) - self.reconstruction_method = helper_functions.collect_bids_part('rec', str(self.destination_path)) - self.run_id = helper_functions.collect_bids_part('run', str(self.destination_path)) + if not self.full_file_path_given: + if not destination_path: + self.destination_path = self.image_folder + self.destination_folder = self.image_folder + else: + self.destination_folder = Path(destination_path) + self.destination_path = self.destination_folder - # once we've groked all the parts (entities) or ultimate path of the output files (blood, nifti, json, etc) - # we will save that to this variable. - self.new_file_name_with_entities = None + # extract PET filename parts from destination path if given + self.subject_id = helper_functions.collect_bids_part( + "sub", str(self.destination_path) + ) + self.session_id = helper_functions.collect_bids_part( + "ses", str(self.destination_path) + ) + self.task = helper_functions.collect_bids_part( + "task", str(self.destination_path) + ) + self.tracer = helper_functions.collect_bids_part( + "trc", str(self.destination_path) + ) + self.reconstruction_method = helper_functions.collect_bids_part( + "rec", str(self.destination_path) + ) + self.run_id = helper_functions.collect_bids_part( + "run", str(self.destination_path) + ) + + self.file_name_slug = None # we keep track of PET metadata in this spreadsheet metadata_dict, that includes nifti, _blood.json, and # _blood.tsv data - self.spreadsheet_metadata = {} + self.spreadsheet_metadata = { + "nifti_json": {}, + "blood_json": {}, + "blood_tsv": {}, + } self.dicom_headers = self.extract_dicom_headers() # we consider values stored in a default JSON file to be additional arguments, we load those # values first and then overwrite them with any user supplied values # load config file - default_json_path = helper_functions.check_pet2bids_config('DEFAULT_METADATA_JSON') + default_json_path = helper_functions.check_pet2bids_config( + "DEFAULT_METADATA_JSON" + ) if default_json_path and Path(default_json_path).exists(): - with open(default_json_path, 'r') as json_file: + with open(default_json_path, "r") as json_file: try: self.spreadsheet_metadata.update(json.load(json_file)) except json.decoder.JSONDecodeError: - logger.warning(f"Unable to load default metadata json file at {default_json_path}, skipping.") + logger.warning( + f"Unable to load default metadata json file at {default_json_path}, skipping." + ) self.additional_arguments = additional_arguments @@ -458,68 +310,34 @@ def __init__(self, image_folder, destination_path=None, metadata_path=None, self.metadata_path = Path(metadata_path) self.metadata_translation_script = Path(metadata_translation_script) - if self.metadata_path.exists() and self.metadata_translation_script.exists(): + if ( + self.metadata_path.exists() + and self.metadata_translation_script.exists() + ): # load the spreadsheet into a dataframe self.extract_metadata() # next we use the loaded python script to extract the information we need self.load_spread_sheet_data() elif metadata_path and not metadata_translation_script or metadata_path == "": - spread_sheet_values = {} - - if Path(metadata_path).is_file(): - spread_sheet_values = helper_functions.single_spreadsheet_reader( - metadata_path, - dicom_metadata=self.dicom_headers[next(iter(self.dicom_headers))], - **self.additional_arguments) - - if Path(metadata_path).is_dir() or metadata_path == "": - # we accept folder input as well as no input, in the - # event of no input we search for spreadsheets in the - # image folder - if metadata_path == "": - metadata_path = self.image_folder - - spreadsheets = helper_functions.collect_spreadsheets(metadata_path) - pet_spreadsheets = [spreadsheet for spreadsheet in spreadsheets if is_pet.pet_file(spreadsheet)] - spread_sheet_values = {} - - for pet_spreadsheet in pet_spreadsheets: - spread_sheet_values.update( - helper_functions.single_spreadsheet_reader( - pet_spreadsheet, - dicom_metadata=self.dicom_headers[next(iter(self.dicom_headers))], - **self.additional_arguments)) - - - # check for any blood (tsv) data or otherwise in the given spreadsheet values - blood_tsv_columns = ['time', 'plasma_radioactivity', 'metabolite_parent_fraction', - 'whole_blood_radioactivity'] - blood_json_columns = ['PlasmaAvail', 'WholeBloodAvail', 'MetaboliteAvail', 'MetaboliteMethod', - 'MetaboliteRecoveryCorrectionApplied', 'DispersionCorrected'] - - # check for existing tsv columns - for column in blood_tsv_columns: - try: - values = spread_sheet_values[column] - self.spreadsheet_metadata['blood_tsv'][column] = values - # pop found data from spreadsheet values after it's been found - spread_sheet_values.pop(column) - except KeyError: - pass - - # check for existing blood json values - for column in blood_json_columns: - try: - values = spread_sheet_values[column] - self.spreadsheet_metadata['blood_json'][column] = values - # pop found data from spreadsheet values after it's been found - spread_sheet_values.pop(column) - except KeyError: - pass - - if not self.spreadsheet_metadata.get('nifti_json', None): - self.spreadsheet_metadata['nifti_json'] = {} - self.spreadsheet_metadata['nifti_json'].update(spread_sheet_values) + if not self.spreadsheet_metadata.get("nifti_json", None): + self.spreadsheet_metadata["nifti_json"] = {} + + load_spreadsheet_data = get_metadata_from_spreadsheet( + metadata_path=metadata_path, + image_folder=self.image_folder, + image_header_dict=self.dicom_headers[next(iter(self.dicom_headers))], + **self.additional_arguments, + ) + + self.spreadsheet_metadata["nifti_json"].update( + load_spreadsheet_data["nifti_json"] + ) + self.spreadsheet_metadata["blood_tsv"].update( + load_spreadsheet_data["blood_tsv"] + ) + self.spreadsheet_metadata["blood_json"].update( + load_spreadsheet_data["blood_json"] + ) self.file_format = file_format # we may want to include additional information to the sidecar, tsv, or json files generated after conversion @@ -532,16 +350,40 @@ def __init__(self, image_folder, destination_path=None, metadata_path=None, @staticmethod def check_posix(): - check = subprocess.run("dcm2niix -h", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + check = subprocess.run( + "dcm2niix -h", + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) if check.returncode == 0: - dcm2niix_path = subprocess.run('which dcm2niix', - shell=True, - capture_output=True).stdout.decode('utf-8').strip() + dcm2niix_path = ( + subprocess.run("which dcm2niix", shell=True, capture_output=True) + .stdout.decode("utf-8") + .strip() + ) + # check to see if dcm2niix is set in the config file, default to that if it's the case and alert user + set_in_config = helper_functions.check_pet2bids_config() + if set_in_config: + logger.warning( + f"dcm2niix found on system path, but dcm2niix path is also set in ~/.pet2bidsconfig." + f" Defaulting to dcm2niix path set in config at {set_in_config}" + ) + dcm2niix_path = set_in_config + else: + dcm2niix_path = helper_functions.check_pet2bids_config() + + if not dcm2niix_path: pkged = "https://github.com/rordenlab/dcm2niix/releases" instructions = "https://github.com/rordenlab/dcm2niix#install" - no_dcm2niix = f"""Dcm2niix does not appear to be installed. Installation instructions can be found here - {instructions} and packaged versions can be found at {pkged}""" + no_dcm2niix = f"""Unable to locate Dcm2niix on your system $PATH or using the path specified in + $HOME/.pypet2bidsconfig. Installation instructions for dcm2niix can be found here + {instructions} + and packaged versions can be found at + {pkged} + Alternatively, you can set the path to dcm2niix in the config file at $HOME/.pet2bidsconfig + using the command dcm2niix4pet --set-dcm2niix-path.""" logger.error(no_dcm2niix) dcm2niix_path = None @@ -554,13 +396,13 @@ def check_for_dcm2niix(self): :return: status code of the command dcm2niix -h """ - if system().lower() != 'windows': + if system().lower() != "windows": dcm2niix_path = self.check_posix() # fall back and check the config file if it's not on the path if not dcm2niix_path: - dcm2niix_path = helper_functions.check_pet2bids_config('DCM2NIIX_PATH') - elif system().lower() == 'windows': - dcm2niix_path = helper_functions.check_pet2bids_config('DCM2NIIX_PATH') + dcm2niix_path = helper_functions.check_pet2bids_config("DCM2NIIX_PATH") + elif system().lower() == "windows": + dcm2niix_path = helper_functions.check_pet2bids_config("DCM2NIIX_PATH") else: dcm2niix_path = None @@ -616,17 +458,29 @@ def run_dcm2niix(self): convert = subprocess.run(cmd, shell=True, capture_output=True) if convert.returncode != 0: - print("Check output .nii files, dcm2iix returned these errors during conversion:") - if bytes("Skipping existing file name", "utf-8") not in convert.stdout or convert.stderr: + print( + "Check output .nii files, dcm2iix returned these errors during conversion:" + ) + if ( + bytes("Skipping existing file name", "utf-8") not in convert.stdout + or convert.stderr + ): print(convert.stderr) - elif convert.returncode != 0 and bytes("Error: Check sorted order", - "utf-8") in convert.stdout or convert.stderr: - print("Possible error with frame order, is this a phillips dicom set?") + elif ( + convert.returncode != 0 + and bytes("Error: Check sorted order", "utf-8") in convert.stdout + or convert.stderr + ): + print( + "Possible error with frame order, is this a phillips dicom set?" + ) print(convert.stdout) print(convert.stderr) # collect contents of the tempdir - files_created_by_dcm2niix = [join(tempdir_pathlike, file) for file in listdir(tempdir_pathlike)] + files_created_by_dcm2niix = [ + join(tempdir_pathlike, file) for file in listdir(tempdir_pathlike) + ] # make sure destination path exists if not try creating it. try: @@ -642,28 +496,38 @@ def run_dcm2niix(self): # iterate through created files to supplement sidecar jsons for created in files_created_by_dcm2niix: created_path = Path(created) - if created_path.suffix == '.json': + if created_path.suffix == ".json": # we want to pair up the headers to the files created in the output directory in case # dcm2niix has created files from multiple sessions - matched_dicoms_and_headers = self.match_dicom_header_to_file(destination_path=tempdir_pathlike) + matched_dicoms_and_headers = self.match_dicom_header_to_file( + destination_path=tempdir_pathlike + ) # we check to see what's missing from our recommended and required jsons by gathering the # output of check_json silently if self.additional_arguments: - check_for_missing = check_json(created_path, - silent=True, - spreadsheet_metadata=self.spreadsheet_metadata, - **self.additional_arguments) + check_for_missing = check_json( + created_path, + silent=True, + spreadsheet_metadata=self.spreadsheet_metadata, + **self.additional_arguments, + ) else: - check_for_missing = check_json(created_path, - silent=True, - spreadsheet_metadata=self.spreadsheet_metadata) + check_for_missing = check_json( + created_path, + silent=True, + spreadsheet_metadata=self.spreadsheet_metadata, + ) # we do our best to extra information from the dicom header and insert these values # into the sidecar json # first do a reverse lookup of the key the json corresponds to - lookup = [key for key, value in matched_dicoms_and_headers.items() if str(created_path) in value] + lookup = [ + key + for key, value in matched_dicoms_and_headers.items() + if str(created_path) in value + ] if lookup: dicom_header = self.dicom_headers[lookup[0]] @@ -671,14 +535,17 @@ def run_dcm2niix(self): created_path, check_for_missing, dicom_header, - dicom2bids_json=metadata_dictionaries['dicom2bids.json'], - **self.additional_arguments) + dicom2bids_json=metadata_dictionaries["dicom2bids.json"], + **self.additional_arguments, + ) # if we have entities in our metadata spreadsheet that we've used we update - if self.spreadsheet_metadata.get('nifti_json', None): - update_json = JsonMAJ(json_path=str(created), - update_values=self.spreadsheet_metadata['nifti_json'], - bids_null=True) + if self.spreadsheet_metadata.get("nifti_json", None): + update_json = JsonMAJ( + json_path=str(created), + update_values=self.spreadsheet_metadata["nifti_json"], + bids_null=True, + ) update_json.update() # check to see if frame duration is a single value, if so convert it to list @@ -689,23 +556,29 @@ def run_dcm2niix(self): # additional arguments we run this step after updating the sidecar with those additional user # arguments - sidecar_json = JsonMAJ(json_path=str(created), - bids_null=True, - update_values=self.additional_arguments) # load all supplied and now written sidecar data in + sidecar_json = JsonMAJ( + json_path=str(created), + bids_null=True, + update_values=self.additional_arguments, + ) # load all supplied and now written sidecar data in sidecar_json.update() - check_metadata_radio_inputs = check_meta_radio_inputs(sidecar_json.json_data) # run logic + check_metadata_radio_inputs = check_meta_radio_inputs( + sidecar_json.json_data + ) # run logic - sidecar_json.update(check_metadata_radio_inputs) # update sidecar json with results of logic + sidecar_json.update( + check_metadata_radio_inputs + ) # update sidecar json with results of logic # should be list/array types in the json should_be_array = [ - 'FrameDuration', - 'ScatterFraction', - 'FrameTimesStart', - 'DecayCorrectionFactor', - 'ReconFilterSize' + "FrameDuration", + "ScatterFraction", + "FrameTimesStart", + "DecayCorrectionFactor", + "ReconFilterSize", ] for should in should_be_array: @@ -716,40 +589,59 @@ def run_dcm2niix(self): # next we check to see if any of the additional user supplied arguments (kwargs) correspond to # any of the missing tags in our sidecars if self.additional_arguments: - update_json = JsonMAJ(json_path=str(created), - update_values=self.additional_arguments, - bids_null=True) + update_json = JsonMAJ( + json_path=str(created), + update_values=self.additional_arguments, + bids_null=True, + ) update_json.update() # check to see if convolution kernel is present sidecar_json = JsonMAJ(json_path=str(created), bids_null=True) - if sidecar_json.get('ConvolutionKernel'): - if sidecar_json.get('ReconFilterType') and sidecar_json.get('ReconFilterSize'): - sidecar_json.remove('ConvolutionKernel') + if sidecar_json.get("ConvolutionKernel"): + if sidecar_json.get("ReconFilterType") and sidecar_json.get( + "ReconFilterSize" + ): + sidecar_json.remove("ConvolutionKernel") else: # collect filter size - recon_filter_size = '' - if re.search(r'\d+.\d+', sidecar_json.get('ConvolutionKernel')): - recon_filter_size = re.search(r'\d+.\d*', sidecar_json.get('ConvolutionKernel'))[0] + recon_filter_size = "" + if re.search( + r"\d+.\d+", sidecar_json.get("ConvolutionKernel") + ): + recon_filter_size = re.search( + r"\d+.\d*", sidecar_json.get("ConvolutionKernel") + )[0] recon_filter_size = float(recon_filter_size) - sidecar_json.update({'ReconFilterSize': float(recon_filter_size)}) + sidecar_json.update( + {"ReconFilterSize": float(recon_filter_size)} + ) # collect just the filter type by popping out the filter size if it exists - recon_filter_type = re.sub(str(recon_filter_size), '', - sidecar_json.get('ConvolutionKernel')) + recon_filter_type = re.sub( + str(recon_filter_size), + "", + sidecar_json.get("ConvolutionKernel"), + ) # further sanitize the recon filter type string - recon_filter_type = re.sub(r'[^a-zA-Z0-9]', ' ', recon_filter_type) - recon_filter_type = re.sub(r' +', ' ', recon_filter_type) + recon_filter_type = re.sub( + r"[^a-zA-Z0-9]", " ", recon_filter_type + ) + recon_filter_type = re.sub(r" +", " ", recon_filter_type) # update the json - sidecar_json.update({'ReconFilterType': recon_filter_type}) + sidecar_json.update({"ReconFilterType": recon_filter_type}) # remove non bids field - sidecar_json.remove('ConvolutionKernel') + sidecar_json.remove("ConvolutionKernel") # check the input args again as our logic is applied after parsing user inputs if self.additional_arguments: recon_filter_user_input = { - 'ReconFilterSize': self.additional_arguments.get('ReconFilterSize', None), - 'ReconFilterType': self.additional_arguments.get('ReconFilterType', None) + "ReconFilterSize": self.additional_arguments.get( + "ReconFilterSize", None + ), + "ReconFilterType": self.additional_arguments.get( + "ReconFilterType", None + ), } for key, value in recon_filter_user_input.items(): if value: @@ -758,56 +650,102 @@ def run_dcm2niix(self): pass # tag json with additional conversion software - conversion_software = sidecar_json.get('ConversionSoftware') - conversion_software_version = sidecar_json.get('ConversionSoftwareVersion') + conversion_software = sidecar_json.get("ConversionSoftware") + conversion_software_version = sidecar_json.get( + "ConversionSoftwareVersion" + ) - sidecar_json.update({'ConversionSoftware': [conversion_software, 'pypet2bids']}) + sidecar_json.update( + {"ConversionSoftware": [conversion_software, "pypet2bids"]} + ) sidecar_json.update( { - 'ConversionSoftwareVersion': [conversion_software_version, helper_functions.get_version()] - }) + "ConversionSoftwareVersion": [ + conversion_software_version, + helper_functions.get_version(), + ] + } + ) # if this looks familiar, that's because it is, we re-run this to override any changes # made by this software as the input provided by the user is "the correct input" - sidecar_json.update(self.spreadsheet_metadata.get('nifti_json', {})) + sidecar_json.update(self.spreadsheet_metadata.get("nifti_json", {})) sidecar_json.update(self.additional_arguments) + # this is mostly for ezBIDS, but it helps us to make better use of the series description that + # dcm2niix generates by default for PET imaging + collect_these_fields = { + "ProtocolName": "", + "SeriesDescription": "", + "TracerName": "trc", + "InjectedRadioactivity": "", + "InjectedRadioactivityUnits": "", + "ReconMethodName": "rec", + "TimeZero": "", + } + collection_of_fields = {} + for field, entity_string in collect_these_fields.items(): + if sidecar_json.get(field): + # if there's a shortened entity string for the field use that + if entity_string != "": + collection_of_fields[entity_string] = sidecar_json.get( + field + ) + else: + collection_of_fields[field] = sidecar_json.get(field) + + if self.session_id: + collection_of_fields["ses"] = self.session_id + + hash_string = helper_functions.hash_fields(**collection_of_fields) + + sidecar_json.update({"SeriesDescription": hash_string}) + # if there's a subject id rename the output file to use it if self.subject_id: - if 'nii.gz' in created_path.name: - suffix = '.nii.gz' + if "nii.gz" in created_path.name: + suffix = ".nii.gz" else: suffix = created_path.suffix if self.session_id: - session_id = '_' + self.session_id + session_id = "_" + self.session_id else: - session_id = '' + session_id = "" if self.task: - task = '_' + self.task + task = "_" + self.task else: - task = '' + task = "" if self.tracer: - trc = '_' + self.tracer + trc = "_" + self.tracer else: - trc = '' + trc = "" if self.reconstruction_method: - rec = '_' + self.reconstruction_method + rec = "_" + self.reconstruction_method else: - rec = '' + rec = "" if self.run_id: - run = '_' + self.run_id + run = "_" + self.run_id else: - run = '' + run = "" if self.full_file_path_given: new_path = self.destination_path.with_suffix(suffix) + self.destination_folder = self.destination_path.parent else: - new_path = self.destination_path / Path(self.subject_id + session_id + task + trc + rec + - run + '_pet' + suffix) + new_path = self.destination_path / Path( + self.subject_id + + session_id + + task + + trc + + rec + + run + + "_pet" + + suffix + ) try: new_path.parent.mkdir(parents=True, exist_ok=True) @@ -828,37 +766,54 @@ def post_dcm2niix(self): # for now we will just assume that if the user supplied a blood tsv then it is manual recording_entity = "_recording-manual" - if '_pet' in self.new_file_name_with_entities.name: - if self.new_file_name_with_entities.suffix == '.gz' and len(self.new_file_name_with_entities.suffixes) > 1: - self.new_file_name_with_entities = self.new_file_name_with_entities.with_suffix('').with_suffix('') - - blood_file_name = self.new_file_name_with_entities.stem.replace('_pet', recording_entity + '_blood') + if "_pet" in self.new_file_name_with_entities.name: + if ( + self.new_file_name_with_entities.suffix == ".gz" + and len(self.new_file_name_with_entities.suffixes) > 1 + ): + self.new_file_name_with_entities = ( + self.new_file_name_with_entities.with_suffix("").with_suffix("") + ) + + blood_file_name = self.new_file_name_with_entities.stem.replace( + "_pet", recording_entity + "_blood" + ) else: - blood_file_name = self.new_file_name_with_entities.stem + recording_entity + '_blood' + blood_file_name = ( + self.new_file_name_with_entities.stem + recording_entity + "_blood" + ) - if self.spreadsheet_metadata.get('blood_tsv', {}) != {}: - blood_tsv_data = self.spreadsheet_metadata.get('blood_tsv') + if self.spreadsheet_metadata.get("blood_tsv", {}) != {}: + blood_tsv_data = self.spreadsheet_metadata.get("blood_tsv") if type(blood_tsv_data) is pd.DataFrame or type(blood_tsv_data) is dict: if type(blood_tsv_data) is dict: blood_tsv_data = pd.DataFrame(blood_tsv_data) # write out blood_tsv using pandas csv write - blood_tsv_data.to_csv(join(self.destination_path, blood_file_name + ".tsv") - , sep='\t', - index=False) + blood_tsv_data.to_csv( + join(self.destination_folder, blood_file_name + ".tsv"), + sep="\t", + index=False, + ) elif type(blood_tsv_data) is str: # write out with write - with open(join(self.destination_path, blood_file_name + ".tsv"), 'w') as outfile: + with open( + join(self.destination_folder, blood_file_name + ".tsv"), "w" + ) as outfile: outfile.writelines(blood_tsv_data) else: - raise (f"blood_tsv dictionary is incorrect type {type(blood_tsv_data)}, must be type: " - f"pandas.DataFrame or str\nCheck return type of translate_metadata in " - f"{self.metadata_translation_script}") + raise ( + f"blood_tsv dictionary is incorrect type {type(blood_tsv_data)}, must be type: " + f"pandas.DataFrame or str\nCheck return type of translate_metadata in " + f"{self.metadata_translation_script}" + ) # if there's blood data in the tsv then write out the sidecar file too - if self.spreadsheet_metadata.get('blood_json', {}) != {} \ - and self.spreadsheet_metadata.get('blood_tsv', {}) != {}: - blood_json_data = self.spreadsheet_metadata.get('blood_json') + if ( + self.spreadsheet_metadata.get("blood_json", {}) != {} + and self.spreadsheet_metadata.get("blood_tsv", {}) != {} + ): + blood_json_data = self.spreadsheet_metadata.get("blood_json") if type(blood_json_data) is dict: # write out to file with json dump pass @@ -866,11 +821,15 @@ def post_dcm2niix(self): # write out to file with json dumps blood_json_data = json.loads(blood_json_data) else: - raise (f"blood_json dictionary is incorrect type {type(blood_json_data)}, must be type: dict or str" - f"pandas.DataFrame or str\nCheck return type of translate_metadata in " - f"{self.metadata_translation_script}") - - with open(join(self.destination_path, blood_file_name + '.json'), 'w') as outfile: + raise ( + f"blood_json dictionary is incorrect type {type(blood_json_data)}, must be type: dict or str" + f"pandas.DataFrame or str\nCheck return type of translate_metadata in " + f"{self.metadata_translation_script}" + ) + + with open( + join(self.destination_folder, blood_file_name + ".json"), "w" + ) as outfile: json.dump(blood_json_data, outfile, indent=4) def convert(self): @@ -888,8 +847,11 @@ def match_dicom_header_to_file(self, destination_path=None): """ if not destination_path: destination_path = self.destination_path - # first collect all of the files in the output directory - output_files = [join(destination_path, output_file) for output_file in listdir(destination_path)] + # first collect all the files in the output directory + output_files = [ + join(destination_path, output_file) + for output_file in listdir(destination_path) + ] # create empty dictionary to store pairings headers_to_files = {} @@ -897,11 +859,17 @@ def match_dicom_header_to_file(self, destination_path=None): # collect study date and time from header for each in self.dicom_headers: header_study_date = self.dicom_headers[each].StudyDate - header_acquisition_time = str(round(float(self.dicom_headers[each].StudyTime))) + header_acquisition_time = str( + round(float(self.dicom_headers[each].StudyTime)) + ) if len(header_acquisition_time) < 6: - header_acquisition_time = (6 - len(header_acquisition_time)) * "0" + header_acquisition_time + header_acquisition_time = ( + 6 - len(header_acquisition_time) + ) * "0" + header_acquisition_time - header_date_time = dicom_datetime_to_dcm2niix_time(date=header_study_date, time=header_acquisition_time) + header_date_time = dicom_datetime_to_dcm2niix_time( + date=header_study_date, time=header_acquisition_time + ) for output_file in output_files: if header_date_time in output_file: @@ -927,13 +895,10 @@ def open_meta_data(self, extension): :param extension: The extension of the file :return: a pandas dataframe representation of the spreadsheet/metadatafile """ - methods = { - 'excel': pd.read_excel, - 'csv': pd.read_csv - } + methods = {"excel": pd.read_excel, "csv": pd.read_csv} - if 'xls' in extension: - proper_method = 'excel' + if "xls" in extension: + proper_method = "excel" else: proper_method = extension @@ -949,226 +914,26 @@ def load_spread_sheet_data(self): try: # this is where the goofiness happens, we allow the user to create their own custom script to manipulate # data from their particular spreadsheet wherever that file is located. - spec = importlib.util.spec_from_file_location("metadata_translation_script", - self.metadata_translation_script) + spec = importlib.util.spec_from_file_location( + "metadata_translation_script", self.metadata_translation_script + ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) text_file_data = module.translate_metadata(self.metadata_dataframe) except AttributeError as err: print(f"Unable to locate metadata_translation_script") - self.spreadsheet_metadata['blood_tsv'] = text_file_data.get('blood_tsv', {}) - self.spreadsheet_metadata['blood_json'] = text_file_data.get('blood_json', {}) - self.spreadsheet_metadata['nifti_json'] = text_file_data.get('nifti_json', {}) + self.spreadsheet_metadata["blood_tsv"] = text_file_data.get("blood_tsv", {}) + self.spreadsheet_metadata["blood_json"] = text_file_data.get( + "blood_json", {} + ) + self.spreadsheet_metadata["nifti_json"] = text_file_data.get( + "nifti_json", {} + ) -def check_meta_radio_inputs(kwargs: dict) -> dict: - """ - Executes very specific PET logic, author does not recall everything it does. - :param kwargs: metadata key pair's to examine - :type kwargs: dict - :return: fitted/massaged metadata corresponding to logic steps below, return type is an update on input `kwargs` - :rtype: dict +epilog = textwrap.dedent( """ - InjectedRadioactivity = kwargs.get('InjectedRadioactivity', None) - InjectedMass = kwargs.get("InjectedMass", None) - SpecificRadioactivity = kwargs.get("SpecificRadioactivity", None) - MolarActivity = kwargs.get("MolarActivity", None) - MolecularWeight = kwargs.get("MolecularWeight", None) - - data_out = {} - - if InjectedRadioactivity and InjectedMass: - data_out['InjectedRadioactivity'] = InjectedRadioactivity - data_out['InjectedRadioactivityUnits'] = 'MBq' - data_out['InjectedMass'] = InjectedMass - data_out['InjectedMassUnits'] = 'ug' - # check for strings where there shouldn't be strings - numeric_check = [helper_functions.is_numeric(str(InjectedRadioactivity)), - helper_functions.is_numeric(str(InjectedMass))] - if False in numeric_check: - data_out['InjectedMass'] = 'n/a' - data_out['InjectedMassUnits'] = 'n/a' - else: - tmp = (InjectedRadioactivity * 10 ** 6) / (InjectedMass * 10 ** 6) - if SpecificRadioactivity: - if SpecificRadioactivity != tmp: - logger.warning("Inferred SpecificRadioactivity in Bq/g doesn't match InjectedRadioactivity " - "and InjectedMass, could be a unit issue") - data_out['SpecificRadioactivity'] = SpecificRadioactivity - data_out['SpecificRadioactivityUnits'] = kwargs.get('SpecificRadioactivityUnityUnits', 'n/a') - else: - data_out['SpecificRadioactivity'] = tmp - data_out['SpecificRadioactivityUnits'] = 'Bq/g' - - if InjectedRadioactivity and SpecificRadioactivity: - data_out['InjectedRadioactivity'] = InjectedRadioactivity - data_out['InjectedRadioactivityUnits'] = 'MBq' - data_out['SpecificRadioactivity'] = SpecificRadioactivity - data_out['SpecificRadioactivityUnits'] = 'Bq/g' - numeric_check = [helper_functions.is_numeric(str(InjectedRadioactivity)), - helper_functions.is_numeric(str(SpecificRadioactivity))] - if False in numeric_check: - data_out['InjectedMass'] = 'n/a' - data_out['InjectedMassUnits'] = 'n/a' - else: - tmp = ((InjectedRadioactivity * (10 ** 6) / SpecificRadioactivity) * (10 ** 6)) - if InjectedMass: - if InjectedMass != tmp: - logger.warning("Inferred InjectedMass in ug doesn't match InjectedRadioactivity and " - "InjectedMass, could be a unit issue") - data_out['InjectedMass'] = InjectedMass - data_out['InjectedMassUnits'] = kwargs.get('InjectedMassUnits', 'n/a') - else: - data_out['InjectedMass'] = tmp - data_out['InjectedMassUnits'] = 'ug' - - if InjectedMass and SpecificRadioactivity: - data_out['InjectedMass'] = InjectedMass - data_out['InjectedMassUnits'] = 'ug' - data_out['SpecificRadioactivity'] = SpecificRadioactivity - data_out['SpecificRadioactivityUnits'] = 'Bq/g' - numeric_check = [helper_functions.is_numeric(str(SpecificRadioactivity)), - helper_functions.is_numeric(str(InjectedMass))] - if False in numeric_check: - data_out['InjectedRadioactivity'] = 'n/a' - data_out['InjectedRadioactivityUnits'] = 'n/a' - else: - tmp = ((InjectedMass / (10 ** 6)) * SpecificRadioactivity) / ( - 10 ** 6) # ((ug / 10 ^ 6) / Bq / g)/10 ^ 6 = MBq - if InjectedRadioactivity: - if InjectedRadioactivity != tmp: - logger.warning("Inferred InjectedRadioactivity in MBq doesn't match SpecificRadioactivity " - "and InjectedMass, could be a unit issue") - data_out['InjectedRadioactivity'] = InjectedRadioactivity - data_out['InjectedRadioactivityUnits'] = kwargs.get('InjectedRadioactivityUnits', 'n/a') - else: - data_out['InjectedRadioactivity'] = tmp - data_out['InjectedRadioactivityUnits'] = 'MBq' - - if MolarActivity and MolecularWeight: - data_out['MolarActivity'] = MolarActivity - data_out['MolarActivityUnits'] = 'GBq/umol' - data_out['MolecularWeight'] = MolecularWeight - data_out['MolecularWeightUnits'] = 'g/mol' - numeric_check = [helper_functions.is_numeric(str(MolarActivity)), - helper_functions.is_numeric(str(MolecularWeight))] - if False in numeric_check: - data_out['SpecificRadioactivity'] = 'n/a' - data_out['SpecificRadioactivityUnits'] = 'n/a' - else: - tmp = (MolarActivity * (10 ** 3)) / MolecularWeight # (GBq / umol * 10 ^ 6) / (g / mol / * 10 ^ 6) = Bq / g - if SpecificRadioactivity: - if SpecificRadioactivity != tmp: - logger.warning( - "Inferred SpecificRadioactivity in MBq/ug doesn't match Molar Activity and Molecular " - "Weight, could be a unit issue") - data_out['SpecificRadioactivity'] = SpecificRadioactivity - data_out['SpecificRadioactivityUnits'] = kwargs.get('SpecificRadioactivityUnityUnits', 'n/a') - else: - data_out['SpecificRadioactivity'] = tmp - data_out['SpecificRadioactivityUnits'] = 'Bq/g' - - if MolarActivity and SpecificRadioactivity: - data_out['SpecificRadioactivity'] = SpecificRadioactivity - data_out['SpecificRadioactivityUnits'] = 'MBq/ug' - data_out['MolarActivity'] = MolarActivity - data_out['MolarActivityUnits'] = 'GBq/umol' - numeric_check = [helper_functions.is_numeric(str(SpecificRadioactivity)), - helper_functions.is_numeric(str(MolarActivity))] - if False in numeric_check: - data_out['MolecularWeight'] = 'n/a' - data_out['MolecularWeightUnits'] = 'n/a' - else: - tmp = (MolarActivity * 1000) / SpecificRadioactivity # (MBq / ug / 1000) / (GBq / umol) = g / mol - if MolecularWeight: - if MolecularWeight != tmp: - logger.warning("Inferred MolecularWeight in MBq/ug doesn't match Molar Activity and " - "Molecular Weight, could be a unit issue") - - data_out['MolecularWeight'] = tmp - data_out['MolecularWeightUnits'] = kwargs.get('MolecularWeightUnits', 'n/a') - else: - data_out['MolecularWeight'] = tmp - data_out['MolecularWeightUnits'] = 'g/mol' - - if MolecularWeight and SpecificRadioactivity: - data_out['SpecificRadioactivity'] = SpecificRadioactivity - data_out['SpecificRadioactivityUnits'] = 'MBq/ug' - data_out['MolecularWeight'] = MolarActivity - data_out['MolecularWeightUnits'] = 'g/mol' - numeric_check = [helper_functions.is_numeric(str(SpecificRadioactivity)), - helper_functions.is_numeric(str(MolecularWeight))] - if False in numeric_check: - data_out['MolarActivity'] = 'n/a' - data_out['MolarActivityUnits'] = 'n/a' - else: - tmp = MolecularWeight * (SpecificRadioactivity / 1000) # g / mol * (MBq / ug / 1000) = GBq / umol - if MolarActivity: - if MolarActivity != tmp: - logger.warning("Inferred MolarActivity in GBq/umol doesn't match Specific Radioactivity and " - "Molecular Weight, could be a unit issue") - data_out['MolarActivity'] = MolarActivity - data_out['MolarActivityUnits'] = kwargs.get('MolarActivityUnits', 'n/a') - else: - data_out['MolarActivity'] = tmp - data_out['MolarActivityUnits'] = 'GBq/umol' - - return data_out - - -def get_radionuclide(pydicom_dicom): - """ - Gets the radionuclide if given a pydicom_object if - pydicom_object.RadiopharmaceuticalInformationSequence[0].RadionuclideCodeSequence exists - - :param pydicom_dicom: dicom object collected by pydicom.dcmread("dicom_file.img") - :return: Labeled Radionuclide e.g. 11Carbon, 18Flourine - """ - radionuclide = "" - try: - radiopharmaceutical_information_sequence = pydicom_dicom.RadiopharmaceuticalInformationSequence - radionuclide_code_sequence = radiopharmaceutical_information_sequence[0].RadionuclideCodeSequence - code_value = radionuclide_code_sequence[0].CodeValue - code_meaning = radionuclide_code_sequence[0].CodeMeaning - extraction_good = True - except AttributeError: - logger.info("Unable to extract RadioNuclideCodeSequence from RadiopharmaceuticalInformationSequence") - extraction_good = False - - if extraction_good: - # check to see if these nucleotides appear in our verified values - verified_nucleotides = metadata_dictionaries['dicom2bids.json']['RadionuclideCodes'] - - check_code_value = "" - check_code_meaning = "" - - if code_value in verified_nucleotides.keys(): - check_code_value = code_value - else: - logger.warning(f"Radionuclide Code {code_value} does not match any known codes in dcm2bids.json\n" - f"will attempt to infer from code meaning {code_meaning}") - - if code_meaning in verified_nucleotides.values(): - radionuclide = re.sub(r'\^', "", code_meaning) - check_code_meaning = code_meaning - else: - logger.warning(f"Radionuclide Meaning {code_meaning} not in known values in dcm2bids json") - if code_value in verified_nucleotides.keys(): - radionuclide = re.sub(r'\^', "", verified_nucleotides[code_value]) - - # final check - if check_code_meaning and check_code_value: - pass - else: - logger.warning( - f"WARNING!!!! Possible mismatch between nuclide code meaning {code_meaning} and {code_value} in dicom " - f"header") - - return radionuclide - - -epilog = textwrap.dedent(''' example usage: @@ -1176,7 +941,8 @@ def get_radionuclide(pydicom_dicom): dcm2niix4pet folder_with_pet_dicoms/ --destination-path sub-ValidBidsSubject/pet --metadata-path metadata.xlsx \ # use with an input spreadsheet -''') +""" +) def cli(): @@ -1189,48 +955,125 @@ def cli(): :param -d, --destination-path: path to place outputfiles post conversion from dicom to nifti + json :return: arguments collected from argument parser """ - parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog) - parser.add_argument('folder', nargs='?', type=str, - help="Folder path containing imaging data") - parser.add_argument('--metadata-path', '-m', type=str, default=None, const='', nargs='?', - help="Path to metadata file for scan") - parser.add_argument('--translation-script-path', '-t', default=None, - help="Path to a script written to extract and transform metadata from a spreadsheet to BIDS" + - " compliant text files (tsv and json)") - parser.add_argument('--destination-path', '-d', type=str, default=None, - help="Destination path to send converted imaging and metadata files to. If subject id and " - "session id is included in the path files created by dcm2niix4pet will be named as such. " - "e.g. sub-NDAR123/ses-ABCD/pet will yield fields named sub-NDAR123_ses-ABCD_*. If " + - "omitted defaults to using the path supplied to folder path. If destination path " + - "doesn't exist an attempt to create it will be made.", required=False) - parser.add_argument('--tempdir', type=str, default=None, - help="User-specified tempdir location (overrides default system tempfile default)", - required=False) - parser.add_argument('--kwargs', '-k', nargs='*', action=helper_functions.ParseKwargs, default={}, - help="Include additional values in the nifti sidecar json or override values extracted from " - "the supplied nifti. e.g. including `--kwargs TimeZero=\"12:12:12\"` would override the " - "calculated TimeZero. Any number of additional arguments can be supplied after --kwargs " - "e.g. `--kwargs BidsVariable1=1 BidsVariable2=2` etc etc." - "Note: the value portion of the argument (right side of the equal's sign) should always" - "be surrounded by double quotes BidsVarQuoted=\"[0, 1 , 3]\"") - parser.add_argument('--silent', '-s', action="store_true", default=False, - help="Hide missing metadata warnings and errors to stdout/stderr") - parser.add_argument('--show-examples', '-E', '--HELP', '-H', help="Shows example usage of this cli.", - action='store_true') - - parser.add_argument('--set-dcm2niix-path', help="Provide a path to a dcm2niix install/exe, writes path to config " - f"file {Path.home()}/.pet2bidsconfig under the variable " - f"DCM2NIIX_PATH", type=pathlib.Path) - parser.add_argument('--set-default-metadata-json', help="Provide a path to a default metadata file json file." - "This file will be used to fill in missing metadata not" - "contained within dicom headers or spreadsheet metadata." - "Sets given path to DEFAULT_METADATA_JSON var in " - f"{Path.home()}/.pet2bidsconfig") + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=epilog, + description="Given a set of PET dicoms and additional metadata dcm2niix converts " + "them to BIDS compliant nifti (using dcm2niix), json, and tsv files.", + ) + parser.add_argument( + "folder", nargs="?", type=str, help="Folder path containing imaging data" + ) + parser.add_argument( + "--metadata-path", + "-m", + type=str, + default=None, + const="", + nargs="?", + help="Path to metadata file for scan", + ) + parser.add_argument( + "--translation-script-path", + "-t", + default=None, + help="Path to a script written to extract and transform metadata from a spreadsheet to BIDS" + + " compliant text files (tsv and json)", + ) + parser.add_argument( + "--destination-path", + "-d", + type=str, + default=None, + help="Destination path to send converted imaging and metadata files to. If subject id and " + "session id is included in the path files created by dcm2niix4pet will be named as such. " + "e.g. sub-NDAR123/ses-ABCD/pet will yield fields named sub-NDAR123_ses-ABCD_*. If " + + "omitted defaults to using the path supplied to folder path. If destination path " + + "doesn't exist an attempt to create it will be made.", + required=False, + ) + parser.add_argument( + "--tempdir", + type=str, + default=None, + help="User-specified tempdir location (overrides default system tempfile default)", + required=False, + ) + parser.add_argument( + "--kwargs", + "-k", + nargs="*", + action=helper_functions.ParseKwargs, + default={}, + help="Include additional values in the nifti sidecar json or override values extracted from " + 'the supplied nifti. e.g. including `--kwargs TimeZero="12:12:12"` would override the ' + "calculated TimeZero. Any number of additional arguments can be supplied after --kwargs " + "e.g. `--kwargs BidsVariable1=1 BidsVariable2=2` etc etc." + "Note: the value portion of the argument (right side of the equal's sign) should always" + 'be surrounded by double quotes BidsVarQuoted="[0, 1 , 3]"', + ) + parser.add_argument( + "--silent", + "-s", + action="store_true", + default=False, + help="Hide missing metadata warnings and errors to stdout/stderr", + ) + parser.add_argument( + "--show-examples", + "-E", + "--HELP", + "-H", + help="Shows example usage of this cli.", + action="store_true", + ) + + parser.add_argument( + "--set-dcm2niix-path", + help="Provide a path to a dcm2niix install/exe, writes path to config " + f"file {Path.home()}/.pet2bidsconfig under the variable " + f"DCM2NIIX_PATH", + type=pathlib.Path, + ) + parser.add_argument( + "--set-default-metadata-json", + help="Provide a path to a default metadata file json file." + "This file will be used to fill in missing metadata not" + "contained within dicom headers or spreadsheet metadata." + "Sets given path to DEFAULT_METADATA_JSON var in " + f"{Path.home()}/.pet2bidsconfig", + ) + parser.add_argument( + "--trc", + "--tracer", + type=str, + default="", + help="Provide a tracer name to be used in the output file name", + ) + parser.add_argument( + "--run", + type=str, + default="", + help="Provide a run id to be used in the output file name", + ) + parser.add_argument( + "--rec", + type=str, + default="", + help="Provide a reconstruction method to be used in the output file name", + ) + parser.add_argument( + "--version", + "-v", + action="version", + version=f"{helper_functions.get_version()}", + ) return parser -example1 = textwrap.dedent(''' +example1 = textwrap.dedent( + """ Usage examples are below, the first being the most brutish way of making dcm2niix4pet to pass through the BIDS validator (with no errors, removing all warnings is left to the user as an exercise) see: @@ -1340,7 +1183,8 @@ def cli(): "SpecificRadioactivityUnits": "Bq/g", "InjectedMass": "n/a", "InjectedMassUnits": "n/a" - }''') + }""" +) def main(): @@ -1368,11 +1212,13 @@ def main(): sys.exit(0) if cli_args.set_dcm2niix_path: - helper_functions.modify_config_file('DCM2NIIX_PATH', cli_args.set_dcm2niix_path) + helper_functions.modify_config_file("DCM2NIIX_PATH", cli_args.set_dcm2niix_path) sys.exit(0) if cli_args.set_default_metadata_json: - helper_functions.modify_config_file('DEFAULT_METADATA_JSON', cli_args.set_default_metadata_json) + helper_functions.modify_config_file( + "DEFAULT_METADATA_JSON", cli_args.set_default_metadata_json + ) sys.exit(0) elif cli_args.folder: @@ -1381,16 +1227,28 @@ def main(): image_folder=helper_functions.expand_path(cli_args.folder), destination_path=helper_functions.expand_path(cli_args.destination_path), metadata_path=helper_functions.expand_path(cli_args.metadata_path), - metadata_translation_script=helper_functions.expand_path(cli_args.translation_script_path), + metadata_translation_script=helper_functions.expand_path( + cli_args.translation_script_path + ), additional_arguments=cli_args.kwargs, tempdir_location=cli_args.tempdir, - silent=cli_args.silent) + silent=cli_args.silent, + ) + + if cli_args.trc: + converter.tracer = "trc-" + cli_args.trc + if cli_args.run: + converter.run_id = "run-" + cli_args.run + if cli_args.rec: + converter.reconstruction_method = "rec-" + cli_args.rec converter.convert() else: - print("folder is a required argument for running dcm2niix, see -h for more detailed usage.") + print( + "folder is a required argument for running dcm2niix, see -h for more detailed usage." + ) sys.exit(1) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/pypet2bids/pypet2bids/dicom_convert.py b/pypet2bids/pypet2bids/dicom_convert.py index dfb16dcf..4fe9f6cb 100755 --- a/pypet2bids/pypet2bids/dicom_convert.py +++ b/pypet2bids/pypet2bids/dicom_convert.py @@ -8,6 +8,7 @@ :Authors: Anthony Galassi :Copyright: Open NeuroPET team """ + import os.path import importlib.util import subprocess @@ -36,12 +37,15 @@ class Convert: converted the imaging data from dicom into nifti. """ - def __init__(self, image_folder: str, - metadata_path: str = None, - destination_path: str = None, - subject_id: str = '', - session_id: str = '', - metadata_translation_script_path: str = None): + def __init__( + self, + image_folder: str, + metadata_path: str = None, + destination_path: str = None, + subject_id: str = "", + session_id: str = "", + metadata_translation_script_path: str = None, + ): self.image_folder = image_folder self.metadata_path = metadata_path @@ -51,7 +55,7 @@ def __init__(self, image_folder: str, self.metadata_dataframe = None # dataframe object of text file metadata self.metadata_translation_script_path = metadata_translation_script_path self.dicom_header_data = {} # extracted data from dicom header - self.nifti_json_data = {} # extracted data from dcm2niix generated json file + self.nifti_json_data = {} # extracted data from dcm2niix generated json file self.blood_json_data = {} # if no destination path is supplied plop nifti into the same folder as the dicom images @@ -63,23 +67,31 @@ def __init__(self, image_folder: str, self.destination_path = destination_path pass else: - print(f"No folder found at destination, creating folder(s) at {destination_path}") + print( + f"No folder found at destination, creating folder(s) at {destination_path}" + ) makedirs(destination_path) if self.check_for_dcm2niix() != 0: - raise Exception("dcm2niix error:\n" + - "The converter relies on dcm2niix.\n" + - "dcm2niix was not found in path, try installing or adding to path variable.") + raise Exception( + "dcm2niix error:\n" + + "The converter relies on dcm2niix.\n" + + "dcm2niix was not found in path, try installing or adding to path variable." + ) self.extract_dicom_header() # create strings for output files if self.session_id: - self.session_string = '_ses-' + self.session_id - elif self.session_id == 'autogeneratesessionid': + self.session_string = "_ses-" + self.session_id + elif self.session_id == "autogeneratesessionid": # if no session is supplied create a datestring from the dicom header - self.session_string = '_ses-' + self.dicom_header_data.SeriesDate + self.dicom_header_data.SeriesTime + self.session_string = ( + "_ses-" + + self.dicom_header_data.SeriesDate + + self.dicom_header_data.SeriesTime + ) else: - self.session_string = '' + self.session_string = "" # now for subject id if subject_id: @@ -87,26 +99,24 @@ def __init__(self, image_folder: str, else: self.subject_id = str(self.dicom_header_data.PatientID) # check for non-bids values - self.subject_id = re.sub(r"[^a-zA-Z0-9\d\s:]", '', self.subject_id) + self.subject_id = re.sub(r"[^a-zA-Z0-9\d\s:]", "", self.subject_id) - self.subject_string = 'sub-' + self.subject_id + self.subject_string = "sub-" + self.subject_id # no reason not to convert the image files immediately if dcm2niix is there self.run_dcm2niix() # extract all metadata - self.extract_nifti_json() # this will extract the data from the dcm2niix sidecar and store it in self.nifti_json_data + self.extract_nifti_json() # this will extract the data from the dcm2niix sidecar and store it in self.nifti_json_data if self.metadata_path: self.extract_metadata() # build output structures for metadata bespoke_data = self.bespoke() # assign output structures to class variables - self.future_json = bespoke_data['future_nifti_json'] - self.future_blood_tsv = bespoke_data['future_blood_tsv'] - self.future_blood_json = bespoke_data['future_blood_json'] - self.participant_info = bespoke_data['participants_info'] - - + self.future_json = bespoke_data["future_nifti_json"] + self.future_blood_tsv = bespoke_data["future_blood_tsv"] + self.future_blood_json = bespoke_data["future_blood_json"] + self.participant_info = bespoke_data["participants_info"] @staticmethod def check_for_dcm2niix(): @@ -114,7 +124,12 @@ def check_for_dcm2niix(): Just checks for dcm2niix using the system shell, returns 0 if dcm2niix is present. :return: status code of the command dcm2niix """ - check = subprocess.run("dcm2niix -h", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + check = subprocess.run( + "dcm2niix -h", + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) return check.returncode def extract_dicom_header(self, additional_fields=[]): @@ -163,7 +178,7 @@ def extract_nifti_json(self): if pet_json is None: raise Exception("Unable to find json file for nifti image") - with open(pet_json, 'r') as infile: + with open(pet_json, "r") as infile: self.nifti_json_data = json.load(infile) def extract_metadata(self): @@ -182,13 +197,10 @@ def open_meta_data(self, extension): :param extension: The extension of the file :return: a pandas dataframe representation of the spreadsheet/metadatafile """ - methods = { - 'excel': pd.read_excel, - 'csv': pd.read_csv - } + methods = {"excel": pd.read_excel, "csv": pd.read_csv} - if 'xls' in extension: - proper_method = 'excel' + if "xls" in extension: + proper_method = "excel" else: proper_method = extension @@ -206,19 +218,34 @@ def run_dcm2niix(self): with TemporaryDirectory() as tempdir: tempdir_pathlike = Path(tempdir) - convert = subprocess.run(f"dcm2niix -w 1 -z y -o {tempdir_pathlike} {self.image_folder}", shell=True, - capture_output=True) - if convert.returncode != 0 and bytes("Skipping existing file named", - 'utf-8') not in convert.stdout or convert.stderr: + convert = subprocess.run( + f"dcm2niix -w 1 -z y -o {tempdir_pathlike} {self.image_folder}", + shell=True, + capture_output=True, + ) + if ( + convert.returncode != 0 + and bytes("Skipping existing file named", "utf-8") not in convert.stdout + or convert.stderr + ): print(convert.stderr) raise Exception("Error during image conversion from dcm to nii!") # collect contents of the tempdir - files_created_by_dcm2niix = [os.path.join(tempdir_pathlike, file) for file in listdir(tempdir_pathlike)] + files_created_by_dcm2niix = [ + os.path.join(tempdir_pathlike, file) + for file in listdir(tempdir_pathlike) + ] # split files by json and nifti - niftis = [Path(nifti) for nifti in files_created_by_dcm2niix if 'nii.gz' in nifti] - sidecars = [Path(sidecar) for sidecar in files_created_by_dcm2niix if '.json' in sidecar] + niftis = [ + Path(nifti) for nifti in files_created_by_dcm2niix if "nii.gz" in nifti + ] + sidecars = [ + Path(sidecar) + for sidecar in files_created_by_dcm2niix + if ".json" in sidecar + ] # order lists niftis.sort() @@ -226,25 +253,40 @@ def run_dcm2niix(self): move_dictionary = {} # loop through and rename files, if there is more than one nifti or json per session add the run label - nifti_run_number, sidecar_run_number = '','' + nifti_run_number, sidecar_run_number = "", "" for index, nifti in enumerate(niftis): if len(niftis) > 1: nifti_run_number = str(index + 1) - nifti_run_number = '_' + nifti_run_number.zfill(len(nifti_run_number) + 1) - new_nifti_name = self.subject_string + self.session_string + nifti_run_number + '_pet.nii.gz' - move_dictionary[str(nifti)] = os.path.join(self.destination_path, new_nifti_name) + nifti_run_number = "_" + nifti_run_number.zfill( + len(nifti_run_number) + 1 + ) + new_nifti_name = ( + self.subject_string + + self.session_string + + nifti_run_number + + "_pet.nii.gz" + ) + move_dictionary[str(nifti)] = os.path.join( + self.destination_path, new_nifti_name + ) for index, sidecar in enumerate(sidecars): if len(sidecars) > 1: sidecar_run_number = str(index + 1) - sidecar_run_number = '_' + zfill(len(sidecar_run_number) + 1) - new_sidecar_name = self.subject_string + self.session_string + sidecar_run_number + '_pet.json' - move_dictionary[str(sidecar)] = os.path.join(self.destination_path, new_sidecar_name) - + sidecar_run_number = "_" + zfill(len(sidecar_run_number) + 1) + new_sidecar_name = ( + self.subject_string + + self.session_string + + sidecar_run_number + + "_pet.json" + ) + move_dictionary[str(sidecar)] = os.path.join( + self.destination_path, new_sidecar_name + ) # move files to actual destination for old_file_path, new_file_path in move_dictionary.items(): - subprocess.run(f'mv {old_file_path} {new_file_path}', shell=True) + subprocess.run(f"mv {old_file_path} {new_file_path}", shell=True) def bespoke(self): """ @@ -256,21 +298,29 @@ def bespoke(self): """ future_nifti_json = { - 'Manufacturer': self.nifti_json_data.get('Manufacturer'), - 'ManufacturersModelName': self.nifti_json_data.get('ManufacturersModelName'), - 'Units': 'Bq/mL', - 'TracerName': self.nifti_json_data.get('Radiopharmaceutical'), - 'TracerRadionuclide': self.nifti_json_data.get('RadionuclideTotalDose', 0) / 10 ** 6, - 'InjectedRadioactivityUnits': 'MBq', - 'FrameTimesStart': - [int(entry) for entry in ([0] + - list(cumsum(self.nifti_json_data['FrameDuration']))[ - 0:len(self.nifti_json_data['FrameDuration']) - 1])], - 'FrameDuration': self.nifti_json_data['FrameDuration'], - 'ReconMethodName': self.dicom_header_data.ReconstructionMethod, - 'ReconFilterType': self.dicom_header_data.ConvolutionKernel, - 'AttenuationCorrection': self.dicom_header_data.AttenuationCorrectionMethod, - 'DecayCorrectionFactor': self.nifti_json_data.get('DecayFactor', '') + "Manufacturer": self.nifti_json_data.get("Manufacturer"), + "ManufacturersModelName": self.nifti_json_data.get( + "ManufacturersModelName" + ), + "Units": "Bq/mL", + "TracerName": self.nifti_json_data.get("Radiopharmaceutical"), + "TracerRadionuclide": self.nifti_json_data.get("RadionuclideTotalDose", 0) + / 10**6, + "InjectedRadioactivityUnits": "MBq", + "FrameTimesStart": [ + int(entry) + for entry in ( + [0] + + list(cumsum(self.nifti_json_data["FrameDuration"]))[ + 0 : len(self.nifti_json_data["FrameDuration"]) - 1 + ] + ) + ], + "FrameDuration": self.nifti_json_data["FrameDuration"], + "ReconMethodName": self.dicom_header_data.ReconstructionMethod, + "ReconFilterType": self.dicom_header_data.ConvolutionKernel, + "AttenuationCorrection": self.dicom_header_data.AttenuationCorrectionMethod, + "DecayCorrectionFactor": self.nifti_json_data.get("DecayFactor", ""), } # initializing empty dictionaries to catch possible additional data from a metadata spreadsheet @@ -281,29 +331,32 @@ def bespoke(self): try: # this is where the goofiness happens, we allow the user to create their own custom script to manipulate # data from their particular spreadsheet wherever that file is located. - spec = importlib.util.spec_from_file_location("metadata_translation_script", - self.metadata_translation_script_path) + spec = importlib.util.spec_from_file_location( + "metadata_translation_script", self.metadata_translation_script_path + ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - text_file_data = module.translate_metadata(self.metadata_dataframe, self.dicom_header_data) + text_file_data = module.translate_metadata( + self.metadata_dataframe, self.dicom_header_data + ) except AttributeError as err: print(f"Unable to locate metadata_translation_script") - self.future_blood_tsv = text_file_data.get('blood_tsv',{}) - self.future_blood_json = text_file_data.get('blood_json',{}) - self.future_nifti_json = text_file_data.get('nifti_json', {}) + self.future_blood_tsv = text_file_data.get("blood_tsv", {}) + self.future_blood_json = text_file_data.get("blood_json", {}) + self.future_nifti_json = text_file_data.get("nifti_json", {}) participants_tsv = { - 'sub_id': [self.subject_id], - 'weight': [self.dicom_header_data.PatientWeight], - 'sex': [self.dicom_header_data.PatientSex] + "sub_id": [self.subject_id], + "weight": [self.dicom_header_data.PatientWeight], + "sex": [self.dicom_header_data.PatientSex], } return { - 'future_nifti_json': future_nifti_json, - 'future_blood_json': future_blood_json, - 'future_blood_tsv': future_blood_tsv, - 'participants_info': participants_tsv + "future_nifti_json": future_nifti_json, + "future_blood_json": future_blood_json, + "future_blood_tsv": future_blood_tsv, + "participants_info": participants_tsv, } def write_out_jsons(self, manual_path=None): @@ -318,16 +371,20 @@ def write_out_jsons(self, manual_path=None): if manual_path is None: # dry - identity_string = os.path.join(self.destination_path, self.subject_string + self.session_string) + identity_string = os.path.join( + self.destination_path, self.subject_string + self.session_string + ) else: - identity_string = os.path.join(manual_path, self.subject_string + self.session_string) + identity_string = os.path.join( + manual_path, self.subject_string + self.session_string + ) - with open(identity_string + '_pet.json', 'w') as outfile: + with open(identity_string + "_pet.json", "w") as outfile: self.nifti_json_data.update(self.future_nifti_json) json.dump(self.nifti_json_data, outfile, indent=4) # write out better json - with open(identity_string + '_recording-manual-blood.json', 'w') as outfile: + with open(identity_string + "_recording-manual-blood.json", "w") as outfile: self.blood_json_data.update(self.future_blood_json) json.dump(self.blood_json_data, outfile, indent=4) @@ -341,62 +398,84 @@ def write_out_blood_tsv(self, manual_path=None): """ if manual_path is None: # dry - identity_string = os.path.join(self.destination_path, self.subject_string + self.session_string) + identity_string = os.path.join( + self.destination_path, self.subject_string + self.session_string + ) else: - identity_string = os.path.join(manual_path, self.subject_string + self.session_string) + identity_string = os.path.join( + manual_path, self.subject_string + self.session_string + ) # make a pandas dataframe from blood data blood_data_df = pandas.DataFrame.from_dict(self.future_blood_tsv) - blood_data_df.to_csv(identity_string + '_recording-manual_blood.tsv', sep='\t', index=False) + blood_data_df.to_csv( + identity_string + "_recording-manual_blood.tsv", sep="\t", index=False + ) # make a small dataframe for the participants - #participants_df = pandas.DataFrame.from_dict(self.participant_info) - #participants_df.to_csv(os.path.join(self.destination_path, 'participants.tsv'), sep='\t', index=False) + # participants_df = pandas.DataFrame.from_dict(self.participant_info) + # participants_df.to_csv(os.path.join(self.destination_path, 'participants.tsv'), sep='\t', index=False) return identity_string # get around dark mode issues on OSX when viewing the Gooey generated gui on a darkmode enabled Mac, does not work well. -if platform.system() == 'Darwin': +if platform.system() == "Darwin": item_default = { - 'error_color': '#ea7878', - 'label_color': '#000000', - 'text_field_color': '#ffffff', - 'text_color': '#000000', - 'help_color': '#363636', - 'full_width': False, - 'validator': { - 'type': 'local', - 'test': 'lambda x: True', - 'message': '' + "error_color": "#ea7878", + "label_color": "#000000", + "text_field_color": "#ffffff", + "text_color": "#000000", + "help_color": "#363636", + "full_width": False, + "validator": {"type": "local", "test": "lambda x: True", "message": ""}, + "external_validator": { + "cmd": "", }, - 'external_validator': { - 'cmd': '', - } } else: item_default = None + def cli(): # simple converter takes command line arguments - parser = ArgumentParser() - parser.add_argument('folder', type=str, - help="Folder path containing imaging data") - parser.add_argument('-m', '--metadata-path', type=str, - help="Path to metadata file for scan") - parser.add_argument('-t', '--translation-script-path', - help="Path to a script written to extract and transform metadata from a spreadsheet to BIDS" + - " compliant text files (tsv and json)") - parser.add_argument('-d', '--destination-path', type=str, - help= "Destination path to send converted imaging and metadata files to. If " + - "omitted defaults to using the path supplied to folder path. If destination path " + - "doesn't exist an attempt to create it will be made.", required=False) - parser.add_argument('-i', '--subject-id', type=str, - help='user supplied subject id. If left blank will use PatientName from dicom header', - required=False) - parser.add_argument('-s', '--session_id', type=str, - help="User supplied session id. If left blank defaults to " + - "None/null and omits addition to output") + parser = ArgumentParser( + description="Converts PET imaging data from dicom to BIDS compliant nifti and metadata " + "files" + ) + parser.add_argument("folder", type=str, help="Folder path containing imaging data") + parser.add_argument( + "-m", "--metadata-path", type=str, help="Path to metadata file for scan" + ) + parser.add_argument( + "-t", + "--translation-script-path", + help="Path to a script written to extract and transform metadata from a spreadsheet to BIDS" + + " compliant text files (tsv and json)", + ) + parser.add_argument( + "-d", + "--destination-path", + type=str, + help="Destination path to send converted imaging and metadata files to. If " + + "omitted defaults to using the path supplied to folder path. If destination path " + + "doesn't exist an attempt to create it will be made.", + required=False, + ) + parser.add_argument( + "-i", + "--subject-id", + type=str, + help="user supplied subject id. If left blank will use PatientName from dicom header", + required=False, + ) + parser.add_argument( + "-s", + "--session_id", + type=str, + help="User supplied session id. If left blank defaults to " + + "None/null and omits addition to output", + ) args = parser.parse_args() @@ -409,7 +488,8 @@ def cli(): destination_path=args.destination_path, metadata_translation_script_path=args.translation_script_path, subject_id=args.subject_id, - session_id=args.session_id) + session_id=args.session_id, + ) # convert it all! converter.run_dcm2niix() diff --git a/pypet2bids/pypet2bids/ecat.py b/pypet2bids/pypet2bids/ecat.py index e1d744d3..a900dbd1 100644 --- a/pypet2bids/pypet2bids/ecat.py +++ b/pypet2bids/pypet2bids/ecat.py @@ -6,12 +6,14 @@ | *Anthony Galassi* | *Copyright Open NeuroPET team* """ + import datetime import re import nibabel import os import json import pathlib +import pandas as pd try: import helper_functions @@ -19,16 +21,25 @@ import read_ecat import ecat2nii import dcm2niix4pet + from update_json_pet_file import ( + get_metadata_from_spreadsheet, + check_meta_radio_inputs, + ) except ModuleNotFoundError: import pypet2bids.helper_functions as helper_functions import pypet2bids.sidecar as sidecar import pypet2bids.read_ecat as read_ecat import pypet2bids.ecat2nii as ecat2nii import pypet2bids.dcm2niix4pet as dcm2niix4pet + from pypet2bids.update_json_pet_file import ( + get_metadata_from_spreadsheet, + check_meta_radio_inputs, + ) from dateutil import parser -logger = helper_functions.logger('pypet2bids') +logger = helper_functions.logger("pypet2bids") + def parse_this_date(date_like_object) -> str: """ @@ -51,7 +62,15 @@ class Ecat: viewing in stdout. Additionally, this class can be used to convert an ECAT7.X image into a nifti image. """ - def __init__(self, ecat_file, nifti_file=None, decompress=True, collect_pixel_data=True): + def __init__( + self, + ecat_file, + nifti_file=None, + decompress=True, + collect_pixel_data=True, + metadata_path=None, + kwargs={}, + ): """ Initialization of this class requires only a path to an ecat file. @@ -64,23 +83,54 @@ def __init__(self, ecat_file, nifti_file=None, decompress=True, collect_pixel_da self.subheaders = [] # subheader information is placed here self.ecat_info = {} self.affine = {} # affine matrix/information is stored here. - self.frame_start_times = [] # frame_start_times, frame_durations, and decay_factors are all - self.frame_durations = [] # extracted from ecat subheaders. They're pretty important and get + self.frame_start_times = ( + [] + ) # frame_start_times, frame_durations, and decay_factors are all + self.frame_durations = ( + [] + ) # extracted from ecat subheaders. They're pretty important and get self.decay_factors = [] # stored here - self.sidecar_template = sidecar.sidecar_template_full # bids approved sidecar file with ALL bids fields - self.sidecar_template_short = sidecar.sidecar_template_short # bids approved sidecar with only required bids fields + self.sidecar_template = ( + sidecar.sidecar_template_full + ) # bids approved sidecar file with ALL bids fields + self.sidecar_template_short = ( + sidecar.sidecar_template_short + ) # bids approved sidecar with only required bids fields + self.sidecar_path = None self.directory_table = None + self.spreadsheet_metadata = { + "nifti_json": {}, + "blood_tsv": {}, + "blood_json": {}, + } + self.kwargs = kwargs + self.output_path = None + self.metadata_path = metadata_path + + # load config file + default_json_path = helper_functions.check_pet2bids_config( + "DEFAULT_METADATA_JSON" + ) + if default_json_path and pathlib.Path(default_json_path).exists(): + with open(default_json_path, "r") as json_file: + try: + self.spreadsheet_metadata.update(json.load(json_file)) + except json.decoder.JSONDecodeError: + logger.warning( + f"Unable to load default metadata json file at {default_json_path}, skipping." + ) + if os.path.isfile(ecat_file): - self.ecat_file = ecat_file + self.ecat_file = str(ecat_file) else: raise FileNotFoundError(ecat_file) - if '.gz' in self.ecat_file and decompress is True: - uncompressed_ecat_file = re.sub('.gz', '', self.ecat_file) + if ".gz" in self.ecat_file and decompress is True: + uncompressed_ecat_file = re.sub(".gz", "", self.ecat_file) helper_functions.decompress(self.ecat_file, uncompressed_ecat_file) self.ecat_file = uncompressed_ecat_file - if '.gz' in self.ecat_file and decompress is False: + if ".gz" in self.ecat_file and decompress is False: raise Exception("Nifti must be decompressed for reading of file headers") try: @@ -90,23 +140,28 @@ def __init__(self, ecat_file, nifti_file=None, decompress=True, collect_pixel_da raise err directory_byte_block = read_ecat.read_bytes( - path_to_bytes=self.ecat_file, - byte_start=512, - byte_stop=1024) + path_to_bytes=self.ecat_file, byte_start=512, byte_stop=1024 + ) - self.directory_table = read_ecat.get_directory_data(directory_byte_block, self.ecat_file) + self.directory_table = read_ecat.get_directory_data( + directory_byte_block, self.ecat_file + ) # extract ecat info self.extract_affine() if collect_pixel_data: - self.ecat_header, self.subheaders, self.data = read_ecat.read_ecat(self.ecat_file) + self.ecat_header, self.subheaders, self.data = read_ecat.read_ecat( + self.ecat_file + ) else: - self.ecat_header, self.subheaders, self.data = read_ecat.read_ecat(self.ecat_file, collect_pixel_data=False) + self.ecat_header, self.subheaders, self.data = read_ecat.read_ecat( + self.ecat_file, collect_pixel_data=False + ) # aggregate ecat info into ecat_info dictionary - self.ecat_info['header'] = self.ecat_header - self.ecat_info['subheaders'] = self.subheaders - self.ecat_info['affine'] = self.affine + self.ecat_info["header"] = self.ecat_header + self.ecat_info["subheaders"] = self.subheaders + self.ecat_info["affine"] = self.affine # swap file extensions and save output nifti with same name as original ecat if not nifti_file: @@ -114,6 +169,35 @@ def __init__(self, ecat_file, nifti_file=None, decompress=True, collect_pixel_da else: self.nifti_file = nifti_file + # que up metadata path for spreadsheet loading later + if self.metadata_path: + if ( + pathlib.Path(metadata_path).is_file() + and pathlib.Path(metadata_path).exists() + ): + self.metadata_path = metadata_path + elif metadata_path == "": + self.metadata_path = pathlib.Path(self.ecat_file).parent + else: + self.metadata_path = None + + if self.metadata_path: + load_spreadsheet_data = get_metadata_from_spreadsheet( + metadata_path=self.metadata_path, + image_folder=pathlib.Path(self.ecat_file).parent, + image_header_dict={}, + ) + + self.spreadsheet_metadata["nifti_json"].update( + load_spreadsheet_data["nifti_json"] + ) + self.spreadsheet_metadata["blood_tsv"].update( + load_spreadsheet_data["blood_tsv"] + ) + self.spreadsheet_metadata["blood_json"].update( + load_spreadsheet_data["blood_json"] + ) + def make_nifti(self, output_path=None): """ Outputs a nifti from the read in ECAT file. @@ -129,10 +213,15 @@ def make_nifti(self, output_path=None): output = self.nifti_file else: output = output_path - ecat2nii.ecat2nii(ecat_main_header=self.ecat_header, ecat_subheaders=self.subheaders, ecat_pixel_data=self.data, - nifti_file=output, affine=self.affine) - - if 'nii.gz' not in output: + ecat2nii.ecat2nii( + ecat_main_header=self.ecat_header, + ecat_subheaders=self.subheaders, + ecat_pixel_data=self.data, + nifti_file=output, + affine=self.affine, + ) + + if "nii.gz" not in output: output = helper_functions.compress(output) return output @@ -162,7 +251,7 @@ def show_directory_table(self): if column == self.directory_table.shape[1] - 1: print(self.directory_table[row][column]) else: - print(self.directory_table[row][column], end=',', sep='') + print(self.directory_table[row][column], end=",", sep="") def show_header(self): """ @@ -192,91 +281,155 @@ def populate_sidecar(self, **kwargs): """ # if it's an ecat it's Siemens - self.sidecar_template['Manufacturer'] = 'Siemens' + self.sidecar_template["Manufacturer"] = "Siemens" # Siemens model best guess - self.sidecar_template['ManufacturersModelName'] = self.ecat_header.get('SERIAL_NUMBER', None) - self.sidecar_template['TracerRadionuclide'] = self.ecat_header.get('ISOTOPE_NAME', None) - self.sidecar_template['PharmaceuticalName'] = self.ecat_header.get('RADIOPHARAMCEUTICAL', None) + self.sidecar_template["ManufacturersModelName"] = self.ecat_header.get( + "SERIAL_NUMBER", None + ) + self.sidecar_template["TracerRadionuclide"] = self.ecat_header.get( + "ISOTOPE_NAME", None + ) + self.sidecar_template["PharmaceuticalName"] = self.ecat_header.get( + "RADIOPHARMACEUTICAL", None + ) # collect frame time start and populate various subheader fields for subheader in self.subheaders: - self.sidecar_template['DecayCorrectionFactor'].append(subheader.get('DECAY_CORR_FCTR', None)) - self.sidecar_template['FrameTimesStart'].append(subheader.get('FRAME_START_TIME', None)) - self.sidecar_template['FrameDuration'].append(subheader.get('FRAME_DURATION', None)) - self.sidecar_template['ScaleFactor'].append(subheader.get('SCALE_FACTOR', None)) + self.sidecar_template["DecayCorrectionFactor"].append( + subheader.get("DECAY_CORR_FCTR", None) + ) + self.sidecar_template["FrameTimesStart"].append( + subheader.get("FRAME_START_TIME", None) + ) + self.sidecar_template["FrameDuration"].append( + subheader.get("FRAME_DURATION", None) + ) + self.sidecar_template["ScaleFactor"].append( + subheader.get("SCALE_FACTOR", None) + ) # note some of these values won't be in the subheaders for the standard matrix image # need to make sure to clean up arrays and fields filled w/ none during pruning - self.sidecar_template['ScatterFraction'].append(subheader.get('SCATTER_FRACTION', None)) - self.sidecar_template['PromptRate'].append(subheader.get('PROMPT_RATE', None)) - self.sidecar_template['RandomRate'].append(subheader.get('RANDOM_RATE', None)) - self.sidecar_template['SinglesRate'].append(subheader.get('SINGLES_RATE', None)) + if subheader.get("SCATTER_FRACTION", None): + self.sidecar_template["ScatterFraction"].append( + subheader.get("SCATTER_FRACTION") + ) + if subheader.get("PROMPT_RATE", None): + self.sidecar_template["PromptRate"].append(subheader.get("PROMPT_RATE")) + if subheader.get("RANDOM_RATE", None): + self.sidecar_template["RandomRate"].append(subheader.get("RANDOM_RATE")) + if subheader.get("SINGLES_RATE", None): + self.sidecar_template["SinglesRate"].append( + subheader.get("SINGLES_RATE") + ) # collect possible reconstruction method from subheader - recon_method = helper_functions.get_recon_method(self.subheaders[0].get('ANNOTATION')) + recon_method = helper_functions.get_recon_method( + self.subheaders[0].get("ANNOTATION") + ) if recon_method: self.sidecar_template.update(**recon_method) # collect and convert start times for acquisition/time zero? - scan_start_time = self.ecat_header.get('SCAN_START_TIME', None) + scan_start_time = self.ecat_header.get("SCAN_START_TIME", None) if scan_start_time: scan_start_time = parse_this_date(scan_start_time) - self.sidecar_template['AcquisitionTime'] = scan_start_time - self.sidecar_template['ScanStart'] = scan_start_time + self.sidecar_template["AcquisitionTime"] = scan_start_time + self.sidecar_template["ScanStart"] = scan_start_time # collect dose start time - dose_start_time = self.ecat_header.get('DOSE_START_TIME', None) - if dose_start_time: - parsed_dose_time = parse_this_date(dose_start_time) - self.sidecar_template['PharmaceuticalDoseTime'] = parsed_dose_time + dose_start_time = self.ecat_header.get("DOSE_START_TIME", None) + if dose_start_time is not None: + if ( + dose_start_time != 0 + and dose_start_time != "0" + and dose_start_time != 0.0 + ): + dose_start_time = parse_this_date(dose_start_time) + parsed_dose_time = parse_this_date(dose_start_time) + self.sidecar_template["PharmaceuticalDoseTime"] = parsed_dose_time + self.sidecar_template["InjectionStart"] = parsed_dose_time + else: + self.sidecar_template["PharmaceuticalDoseTime"] = int(dose_start_time) + self.sidecar_template["InjectionStart"] = int(dose_start_time) # if decay correction exists mark decay correction boolean as true if len(self.decay_factors) > 0: - self.sidecar_template['ImageDecayCorrected'] = "true" + self.sidecar_template["ImageDecayCorrected"] = "true" # calculate scaling factor sca = self.data.max() / 32767 - self.sidecar_template['DoseCalibrationFactor'] = sca * self.ecat_header.get('ECAT_CALIBRATION_FACTOR') - self.sidecar_template['Filename'] = os.path.basename(self.nifti_file) - self.sidecar_template['ImageSize'] = [ - self.subheaders[0]['X_DIMENSION'], - self.subheaders[0]['Y_DIMENSION'], - self.subheaders[0]['Z_DIMENSION'], - self.ecat_header['NUM_FRAMES'] + self.sidecar_template["DoseCalibrationFactor"] = sca * self.ecat_header.get( + "ECAT_CALIBRATION_FACTOR" + ) + self.sidecar_template["Filename"] = os.path.basename(self.nifti_file) + self.sidecar_template["ImageSize"] = [ + self.subheaders[0]["X_DIMENSION"], + self.subheaders[0]["Y_DIMENSION"], + self.subheaders[0]["Z_DIMENSION"], + self.ecat_header["NUM_FRAMES"], ] - self.sidecar_template['PixelDimensions'] = [ - self.subheaders[0]['X_PIXEL_SIZE'] * 10, - self.subheaders[0]['Y_PIXEL_SIZE'] * 10, - self.subheaders[0]['Z_PIXEL_SIZE'] * 10 + self.sidecar_template["PixelDimensions"] = [ + self.subheaders[0]["X_PIXEL_SIZE"] * 10, + self.subheaders[0]["Y_PIXEL_SIZE"] * 10, + self.subheaders[0]["Z_PIXEL_SIZE"] * 10, ] # add tag for conversion software - self.sidecar_template['ConversionSoftware'] = 'pypet2bids' - self.sidecar_template['ConversionSoftwareVersion'] = helper_functions.get_version() - + self.sidecar_template["ConversionSoftware"] = "pypet2bids" + self.sidecar_template["ConversionSoftwareVersion"] = ( + helper_functions.get_version() + ) + # update sidecar values from spreadsheet + if self.spreadsheet_metadata.get("nifti_json", None): + self.sidecar_template.update(self.spreadsheet_metadata["nifti_json"]) # include any additional values if kwargs: self.sidecar_template.update(**kwargs) - if not self.sidecar_template.get('TimeZero', None): - if not self.sidecar_template.get('AcquisitionTime', None): - logger.warn(f"Unable to determine TimeZero for {self.ecat_file}, you need will need to provide this" - f" for a valid BIDS sidecar.") + if not self.sidecar_template.get("TimeZero", None) and not kwargs.get( + "TimeZero", None + ): + if not self.sidecar_template.get( + "AcquisitionTime", None + ) and not kwargs.get("TimeZero", None): + logger.warn( + f"Unable to determine TimeZero for {self.ecat_file}, you need will need to provide this" + f" for a valid BIDS sidecar." + ) else: - self.sidecar_template['TimeZero'] = self.sidecar_template['AcquisitionTime'] - - # lastly infer radio data if we have it - meta_radio_inputs = dcm2niix4pet.check_meta_radio_inputs(self.sidecar_template) - self.sidecar_template.update(**meta_radio_inputs) + self.sidecar_template["TimeZero"] = self.sidecar_template[ + "AcquisitionTime" + ] + + # set scan start and pharmaceutical dose time relative to time zero. + times_make_relative = ["ScanStart", "PharmaceuticalDoseTime", "InjectionStart"] + time_zero_datetime = datetime.datetime.strptime( + self.sidecar_template.get("TimeZero"), "%H:%M:%S" + ) + for t in times_make_relative: + t_value = self.sidecar_template.get(t) + # sometimes we start with 0 or time in seconds, we first check for this + try: + int(t_value) + self.sidecar_template[t] = t_value + except ValueError: + t_datetime = datetime.datetime.strptime(t_value, "%H:%M:%S") + time_diff = t_datetime - time_zero_datetime + self.sidecar_template[t] = time_diff.total_seconds() # clear any nulls from json sidecar and replace with none's self.sidecar_template = helper_functions.replace_nones(self.sidecar_template) + # lastly infer radio data if we have it + meta_radio_inputs = check_meta_radio_inputs(self.sidecar_template) + self.sidecar_template.update(**meta_radio_inputs) + def prune_sidecar(self): """ Eliminate unpopulated fields in sidecar while leaving in mandatory fields even if they are unpopulated. @@ -287,7 +440,7 @@ def prune_sidecar(self): full_fields = list(self.sidecar_template) exclude_list = [] for field, value in self.sidecar_template.items(): - if value: + if value is not None and value != "": # check to make sure value isn't a list of null types # e.g. if value = [None, None, None] we don't want to include it. if type(value) is list: @@ -317,6 +470,37 @@ def show_sidecar(self, output_path=None): """ self.prune_sidecar() self.sidecar_template = helper_functions.replace_nones(self.sidecar_template) + + # this is mostly for ezBIDS, but it helps us to make better use of the series description that + # dcm2niix generates by default for PET imaging, here we mirror the dcm2niix output for ecats + collect_these_fields = { + "ProtocolName": "", + "SeriesDescription": "", + "TracerName": "trc", + "InjectedRadioactivity": "", + "InjectedRadioactivityUnits": "", + "ReconMethodName": "rec", + "TimeZero": "", + } + collection_of_fields = {} + for field, entity_string in collect_these_fields.items(): + if self.sidecar_template.get(field, None): + # if there's a shortened entity string for the field use that + if entity_string != "": + collection_of_fields[entity_string] = self.sidecar_template.get( + field + ) + else: + collection_of_fields[field] = self.sidecar_template.get(field) + + if helper_functions.collect_bids_part("ses", self.output_path) != "": + collection_of_fields["ses"] = helper_functions.collect_bids_part( + "ses", self.output_path + ) + + hash_string = helper_functions.hash_fields(**collection_of_fields) + self.sidecar_template["SeriesDescription"] = hash_string + if output_path: if not isinstance(output_path, pathlib.Path): output_path = pathlib.Path(output_path) @@ -324,13 +508,129 @@ def show_sidecar(self, output_path=None): if len(output_path.suffixes) > 1: temp_output_path = str(output_path) for suffix in output_path.suffixes: - temp_output_path = re.sub(suffix, '', temp_output_path) - output_path = pathlib.Path(temp_output_path).with_suffix('.json') - - with open(output_path, 'w') as outfile: - json.dump(helper_functions.replace_nones(self.sidecar_template), outfile, indent=4) + temp_output_path = re.sub(suffix, "", temp_output_path) + output_path = pathlib.Path(temp_output_path).with_suffix(".json") + + with open(output_path, "w") as outfile: + json.dump( + helper_functions.replace_nones(self.sidecar_template), + outfile, + indent=4, + ) else: - print(json.dumps(helper_functions.replace_nones(self.sidecar_template), indent=4)) + print( + json.dumps( + helper_functions.replace_nones(self.sidecar_template), indent=4 + ) + ) + + def write_out_blood_files( + self, new_file_name_with_entities=None, destination_folder=None + ): + recording_entity = "_recording-manual" + + if not new_file_name_with_entities: + new_file_name_with_entities = pathlib.Path(self.nifti_file) + if not destination_folder: + destination_folder = pathlib.Path(self.nifti_file).parent + + if "_pet" in new_file_name_with_entities.name: + if ( + new_file_name_with_entities.suffix == ".gz" + and len(new_file_name_with_entities.suffixes) > 1 + ): + new_file_name_with_entities = new_file_name_with_entities.with_suffix( + "" + ).with_suffix("") + + blood_file_name = new_file_name_with_entities.stem.replace( + "_pet", recording_entity + "_blood" + ) + else: + blood_file_name = ( + new_file_name_with_entities.stem + recording_entity + "_blood" + ) + + if self.spreadsheet_metadata.get("blood_tsv", {}) != {}: + blood_tsv_data = self.spreadsheet_metadata.get("blood_tsv") + if type(blood_tsv_data) is pd.DataFrame or type(blood_tsv_data) is dict: + if type(blood_tsv_data) is dict: + blood_tsv_data = pd.DataFrame(blood_tsv_data) + # write out blood_tsv using pandas csv write + blood_tsv_data.to_csv( + os.path.join(destination_folder, blood_file_name + ".tsv"), + sep="\t", + index=False, + ) + + elif type(blood_tsv_data) is str: + # write out with write + with open( + os.path.join(destination_folder, blood_file_name + ".tsv"), "w" + ) as outfile: + outfile.writelines(blood_tsv_data) + else: + raise ( + f"blood_tsv dictionary is incorrect type {type(blood_tsv_data)}, must be type: " + f"pandas.DataFrame" + ) + + # if there's blood data in the tsv then write out the sidecar file too + if ( + self.spreadsheet_metadata.get("blood_json", {}) != {} + and self.spreadsheet_metadata.get("blood_tsv", {}) != {} + ): + blood_json_data = self.spreadsheet_metadata.get("blood_json") + if type(blood_json_data) is dict: + # write out to file with json dump + pass + elif type(blood_json_data) is str: + # write out to file with json dumps + blood_json_data = json.loads(blood_json_data) + else: + raise ( + f"blood_json dictionary is incorrect type {type(blood_json_data)}, must be type: dict or str" + f"pandas.DataFrame" + ) + + with open( + os.path.join(destination_folder, blood_file_name + ".json"), "w" + ) as outfile: + json.dump(blood_json_data, outfile, indent=4) + + def update_pet_json(self, pet_json_path): + """given a json file (or a path ending in .json) update or create a PET json file with information collected + from an ecat file. + :param pet_json: a path to a json file + :type pet_json: str or pathlib.Path + :return: None + """ + + # open the json file if it exists + if isinstance(pet_json_path, str): + pet_json = pathlib.Path(pet_json_path) + if pet_json.exists(): + with open(pet_json_path, "r") as json_file: + try: + pet_json = json.load(json_file) + except json.decoder.JSONDecodeError: + logger.warning( + f"Unable to load json file at {pet_json_path}, skipping." + ) + + # update the template with values from the json file + self.sidecar_template.update(pet_json) + + if self.spreadsheet_metadata.get("nifti_json", None): + self.sidecar_template.update(self.spreadsheet_metadata["nifti_json"]) + + self.populate_sidecar(**self.kwargs) + self.prune_sidecar() + + # check metadata radio inputs + self.sidecar_template.update(check_meta_radio_inputs(self.sidecar_template)) + + self.show_sidecar(output_path=pet_json_path) def json_out(self): """ @@ -340,3 +640,16 @@ def json_out(self): """ temp_json = json.dumps(self.ecat_info, indent=4) print(temp_json) + + def convert(self): + """ + Convert ecat to nifti + :return: None + """ + self.output_path = pathlib.Path(self.make_nifti()) + self.sidecar_path = self.output_path.parent / self.output_path.stem + self.sidecar_path = self.sidecar_path.with_suffix(".json") + self.populate_sidecar(**self.kwargs) + self.prune_sidecar() + self.show_sidecar(output_path=self.sidecar_path) + self.write_out_blood_files() diff --git a/pypet2bids/pypet2bids/ecat2nii.py b/pypet2bids/pypet2bids/ecat2nii.py index bb50aeac..fa187088 100644 --- a/pypet2bids/pypet2bids/ecat2nii.py +++ b/pypet2bids/pypet2bids/ecat2nii.py @@ -5,11 +5,15 @@ | *Authors: Anthony Galassi* | *Copyright OpenNeuroPET team* """ + import datetime import nibabel import numpy import pathlib -from pypet2bids.read_ecat import read_ecat +from pypet2bids.read_ecat import ( + read_ecat, + code_dir, +) # we use code_dir for ecat debugging import os import pickle import logging @@ -20,23 +24,34 @@ import pypet2bids.helper_functions as helper_functions -#logger = helper_functions.logger('pypet2bids') - -logger = logging.getLogger('pypet2bids') - -def ecat2nii(ecat_main_header=None, - ecat_subheaders=None, - ecat_pixel_data=None, - ecat_file=None, - nifti_file: str = '', - sif_out=False, - affine=None, - save_binary=False, - **kwargs): +# debug variable +# save steps for debugging, for mor infor see ecat_testing/README.md +ecat_save_steps = os.environ.get("ECAT_SAVE_STEPS", 0) +if ecat_save_steps == "1": + # check to see if the code directory is available, if it's not create it and + # the steps dir to save outputs created if ecat_save_steps is set to 1 + steps_dir = code_dir.parent / "ecat_testing" / "steps" + if not steps_dir.is_dir(): + os.makedirs(code_dir.parent / "ecat_testing" / "steps", exist_ok=True) + +logger = logging.getLogger("pypet2bids") + + +def ecat2nii( + ecat_main_header=None, + ecat_subheaders=None, + ecat_pixel_data=None, + ecat_file=None, + nifti_file: str = "", + sif_out=False, + affine=None, + save_binary=False, + **kwargs, +): """ Converts an ECAT file into a nifti and a sidecar json, used in conjunction with :func:`pypet2bids.read_ecat.read_ecat` - + :param ecat_main_header: the main header of an ECAT file :param ecat_subheaders: the subheaders for each frame of the ECAT file :param ecat_pixel_data: the imaging/pixel data from the ECAT file @@ -65,32 +80,79 @@ def ecat2nii(ecat_main_header=None, nifti_file_w_out_extension = os.path.splitext(str(pathlib.Path(nifti_file).name))[0] # if already read nifti file skip re-reading - if ecat_main_header is None and ecat_subheaders is None and ecat_pixel_data is None and ecat_file: + if ( + ecat_main_header is None + and ecat_subheaders is None + and ecat_pixel_data is None + and ecat_file + ): # collect ecat_file main_header, sub_headers, data = read_ecat(ecat_file=ecat_file) - elif ecat_file is None and type(ecat_main_header) is dict and type(ecat_subheaders) is list and type( - ecat_pixel_data) is numpy.ndarray: - main_header, sub_headers, data = ecat_main_header, ecat_subheaders, ecat_pixel_data + elif ( + ecat_file is None + and type(ecat_main_header) is dict + and type(ecat_subheaders) is list + and type(ecat_pixel_data) is numpy.ndarray + ): + main_header, sub_headers, data = ( + ecat_main_header, + ecat_subheaders, + ecat_pixel_data, + ) else: - raise Exception("Must pass in filepath for ECAT file or " - "(ecat_main_header, ecat_subheaders, and ecat_pixel data " - f"got ecat_file={ecat_file}, type(ecat_main_header)={type(ecat_main_header)}, " - f"type(ecat_subheaders)={type(ecat_subheaders)}, " - f"type(ecat_pixel_data)={type(ecat_pixel_data)} instead.") + raise Exception( + "Must pass in filepath for ECAT file or " + "(ecat_main_header, ecat_subheaders, and ecat_pixel data " + f"got ecat_file={ecat_file}, type(ecat_main_header)={type(ecat_main_header)}, " + f"type(ecat_subheaders)={type(ecat_subheaders)}, " + f"type(ecat_pixel_data)={type(ecat_pixel_data)} instead." + ) + + # debug step #6 view data as passed to ecat2nii method + if ecat_save_steps == "1": + # collect only the first, middle, and last frames from pixel_data_matrix_4d as first, middle, and last + # frames are typically the most interesting + frames = [0, len(data) // 2, -1] + frames_to_record = [] + for f in frames: + frames_to_record.append(data[:, :, :, f]) + + # now collect a single 2d slice from the "middle" of the 3d frames in frames_to_record + slice_to_record = [] + for index, frame in enumerate(frames_to_record): + numpy.savetxt( + steps_dir / f"6_ecat2nii_python_{index}.tsv", + frames_to_record[index][:, :, frames_to_record[index].shape[2] // 2], + delimiter="\t", + fmt="%s", + ) + + helper_functions.first_middle_last_frames_to_text( + four_d_array_like_object=data, + output_folder=steps_dir, + step_name="6_ecat2nii_python", + ) + + # set the byte order and the pixel data type from the input array + pixel_data_type = data.dtype # check for TimeZero supplied via kwargs - if kwargs.get('TimeZero', None): - TimeZero = kwargs['TimeZero'] + if kwargs.get("TimeZero", None): + TimeZero = kwargs["TimeZero"] else: - logger.warn("Metadata TimeZero is missing -- set to ScanStart or empty to use the scanning time as " - "injection time") + logger.warning( + "Metadata TimeZero is missing -- set to ScanStart or empty to use the scanning time as " + "injection time" + ) # get image shape img_shape = data.shape - shape_from_headers = (sub_headers[0]['X_DIMENSION'], - sub_headers[0]['Y_DIMENSION'], - sub_headers[0]['Z_DIMENSION'], - main_header['NUM_FRAMES']) + shape_from_headers = ( + sub_headers[0]["X_DIMENSION"], + sub_headers[0]["Y_DIMENSION"], + sub_headers[0]["Z_DIMENSION"], + main_header["NUM_FRAMES"], + ) # make sure number of data elements matches frame number single_frame = False @@ -99,14 +161,19 @@ def ecat2nii(ecat_main_header=None, if img_shape != shape_from_headers and not single_frame: raise Exception( f"Mis-match between expected X,Y,Z, and Num. Frames dimensions ({shape_from_headers} obtained from headers" - f"and shape of imaging data ({img_shape}") + f"and shape of imaging data ({img_shape}" + ) # format data into acceptable shape for nibabel, by first creating empty matrix - img_temp = numpy.zeros(shape=(sub_headers[0]['X_DIMENSION'], - sub_headers[0]['Y_DIMENSION'], - sub_headers[0]['Z_DIMENSION'], - main_header['NUM_FRAMES']), - dtype=numpy.dtype('>f4')) + img_temp = numpy.zeros( + shape=( + sub_headers[0]["X_DIMENSION"], + sub_headers[0]["Y_DIMENSION"], + sub_headers[0]["Z_DIMENSION"], + main_header["NUM_FRAMES"], + ), + dtype=">f4", + ) # collect timing information start, delta = [], [] @@ -115,123 +182,170 @@ def ecat2nii(ecat_main_header=None, prompts, randoms = [], [] # load frame data into img temp - for index in reversed(range(img_shape[3])): # Don't throw stones working from existing matlab code + for index in reversed( + range(img_shape[3]) + ): # Don't throw stones working from existing matlab code print(f"Loading frame {index + 1}") - # save out our slice of data before flip to a text file to compare w/ matlab data - img_temp[:, :, :, index] = numpy.flip(numpy.flip(numpy.flip( - data[:, :, :, index].astype(numpy.dtype('>f4')) * sub_headers[index]['SCALE_FACTOR'], 1), 2), 0) - start.append(sub_headers[index]['FRAME_START_TIME'] * 60) # scale to per minute - delta.append(sub_headers[index]['FRAME_DURATION'] * 60) # scale to per minute - - if main_header.get('SW_VERSION', 0) >= 73: + img_temp[:, :, :, index] = numpy.flip( + numpy.flip( + numpy.flip( + data[:, :, :, index] * sub_headers[index]["SCALE_FACTOR"], 1 + ), + 2, + ), + 0, + ) + start.append(sub_headers[index]["FRAME_START_TIME"] * 60) # scale to per minute + delta.append(sub_headers[index]["FRAME_DURATION"] * 60) # scale to per minute + + if main_header.get("SW_VERSION", 0) >= 73: # scale both to per minute - prompts.append(sub_headers[index]['PROMPT_RATE'] * sub_headers[index]['FRAME_DURATION'] * 60) - randoms.append(sub_headers[index]['RANDOM_RATE'] * sub_headers[index]['FRAME_DURATION'] * 60) + prompts.append( + sub_headers[index]["PROMPT_RATE"] + * sub_headers[index]["FRAME_DURATION"] + * 60 + ) + randoms.append( + sub_headers[index]["RANDOM_RATE"] + * sub_headers[index]["FRAME_DURATION"] + * 60 + ) else: # this field is not available in ecat 7.2 prompts.append(0) randoms.append(0) - final_image = img_temp * main_header['ECAT_CALIBRATION_FACTOR'] + # debug step #7 view data after flipping into nifti space/orientation + if ecat_save_steps == "1": + helper_functions.first_middle_last_frames_to_text( + four_d_array_like_object=img_temp, + output_folder=steps_dir, + step_name="7_flip_ecat2nii_python", + ) + + # so the only real difference between the matlab code and the python code is that that we aren't manually + # scaling the date to 16 bit integers. + rg = img_temp.max() - img_temp.min() + if rg != 32767: + max_img = img_temp.max() + img_temp = img_temp / max_img * 32767 + sca = max_img / 32767 + min_img = img_temp.min() + if min_img < -32768: + img_temp = img_temp / (min_img * -32768) + sca = sca * (min_img * -32768) + if ecat_save_steps == "1": + with open(os.path.join(steps_dir, "8.5_sca.txt"), "w") as sca_file: + sca_file.write(f"Scaling factor: {sca}\n") + sca_file.write( + f"Scaling factor * ECAT Cal Factor: {sca * main_header['ECAT_CALIBRATION_FACTOR']}\n" + ) + + # scale image to 16 bit + final_image = img_temp.astype(numpy.single) + + # debug step 8 check after "rescaling" to 16 bit + if ecat_save_steps == "1": + helper_functions.first_middle_last_frames_to_text( + four_d_array_like_object=final_image, + output_folder=steps_dir, + step_name="8_rescale_to_16_ecat2nii_python", + ) + + ecat_cal_units = main_header[ + "CALIBRATION_UNITS" + ] # Header field designating whether data has already been calibrated + if ecat_cal_units == 1: # Calibrate if it hasn't been already + final_image = ( + numpy.round(final_image) * main_header["ECAT_CALIBRATION_FACTOR"] * sca + ) + # this debug step may not execute if we're not calibrating the scan, but that's okay + if ecat_save_steps == "1": + helper_functions.first_middle_last_frames_to_text( + four_d_array_like_object=final_image, + output_folder=steps_dir, + step_name="9_scal_cal_units_ecat2nii_python", + ) + else: # And don't calibrate if CALIBRATION_UNITS is anything else but 1 + final_image = numpy.round(final_image) * sca qoffset_x = -1 * ( - ((sub_headers[0]['X_DIMENSION'] * sub_headers[0]['X_PIXEL_SIZE'] * 10 / 2) - sub_headers[0][ - 'X_PIXEL_SIZE'] * 5)) + ( + (sub_headers[0]["X_DIMENSION"] * sub_headers[0]["X_PIXEL_SIZE"] * 10 / 2) + - sub_headers[0]["X_PIXEL_SIZE"] * 5 + ) + ) qoffset_y = -1 * ( - ((sub_headers[0]['Y_DIMENSION'] * sub_headers[0]['Y_PIXEL_SIZE'] * 10 / 2) - sub_headers[0][ - 'Y_PIXEL_SIZE'] * 5)) + ( + (sub_headers[0]["Y_DIMENSION"] * sub_headers[0]["Y_PIXEL_SIZE"] * 10 / 2) + - sub_headers[0]["Y_PIXEL_SIZE"] * 5 + ) + ) qoffset_z = -1 * ( - ((sub_headers[0]['Z_DIMENSION'] * sub_headers[0]['Z_PIXEL_SIZE'] * 10 / 2) - sub_headers[0][ - 'Z_PIXEL_SIZE'] * 5)) + ( + (sub_headers[0]["Z_DIMENSION"] * sub_headers[0]["Z_PIXEL_SIZE"] * 10 / 2) + - sub_headers[0]["Z_PIXEL_SIZE"] * 5 + ) + ) # build affine if it's not included in function call if not affine: - t = numpy.identity(4) - t[0, 0] = sub_headers[0]['X_PIXEL_SIZE'] * 10 - t[1, 1] = sub_headers[0]['Y_PIXEL_SIZE'] * 10 - t[2, 2] = sub_headers[0]['Z_PIXEL_SIZE'] * 10 - - t[3, 0] = qoffset_x - t[3, 1] = qoffset_y - t[3, 2] = qoffset_z - - # note this affine is the transform of of a nibabel ecat object's affine - affine = t + mat = ( + numpy.diag( + [ + sub_headers[0]["X_PIXEL_SIZE"], + sub_headers[0]["Y_PIXEL_SIZE"], + sub_headers[0]["Z_PIXEL_SIZE"], + ] + ) + * 10 + ) + affine = nibabel.affines.from_matvec(mat, [qoffset_x, qoffset_y, qoffset_z]) img_nii = nibabel.Nifti1Image(final_image, affine=affine) - # populating nifti header - if img_nii.header['sizeof_hdr'] != 348: - img_nii.header['sizeof_hdr'] = 348 - # img_nii.header['dim_info'] is populated on object creation - # img_nii.header['dim'] is populated on object creation - img_nii.header['intent_p1'] = 0 - img_nii.header['intent_p2'] = 0 - img_nii.header['intent_p3'] = 0 - # img_nii.header['datatype'] # created on invocation seems to be 16 or int16 - # img_nii.header['bitpix'] # also automatically created and inferred 32 as of testing w/ cimbi dataset - # img_nii.header['slice_type'] # defaults to 0 - # img_nii.header['pixdim'] # appears as 1d array of length 8 we rescale this - img_nii.header['pixdim'] = numpy.array( - [1, - sub_headers[0]['X_PIXEL_SIZE'] * 10, - sub_headers[0]['Y_PIXEL_SIZE'] * 10, - sub_headers[0]['Z_PIXEL_SIZE'] * 10, - 0, - 0, - 0, - 0]) - img_nii.header['vox_offset'] = 352 - - # TODO img_nii.header['scl_slope'] # this is a NaN array by default but apparently it should be the dose calibration - # factor img_nii.header['scl_inter'] # defaults to NaN array - img_nii.header['scl_slope'] = main_header['ECAT_CALIBRATION_FACTOR'] - img_nii.header['scl_inter'] = 0 - img_nii.header['slice_end'] = 0 - img_nii.header['slice_code'] = 0 - img_nii.header['xyzt_units'] = 10 - img_nii.header['cal_max'] = final_image.min() - img_nii.header['cal_min'] = final_image.max() - img_nii.header['slice_duration'] = 0 - img_nii.header['toffset'] = 0 - img_nii.header['descrip'] = "OpenNeuroPET ecat2nii.py conversion" - # img_nii.header['aux_file'] # ignoring as this is set to '' in matlab - img_nii.header['qform_code'] = 0 - img_nii.header['sform_code'] = 1 # 0: Arbitrary coordinates; - # 1: Scanner-based anatomical coordinates; - # 2: Coordinates aligned to another file's, or to anatomical "truth" (co-registration); - # 3: Coordinates aligned to Talairach-Tournoux Atlas; 4: MNI 152 normalized coordinates - - img_nii.header['quatern_b'] = 0 - img_nii.header['quatern_c'] = 0 - img_nii.header['quatern_d'] = 0 - # Please explain this - img_nii.header['qoffset_x'] = qoffset_x - img_nii.header['qoffset_y'] = qoffset_y - img_nii.header['qoffset_z'] = qoffset_z - img_nii.header['srow_x'] = numpy.array([sub_headers[0]['X_PIXEL_SIZE']*10, 0, 0, img_nii.header['qoffset_x']]) - img_nii.header['srow_y'] = numpy.array([0, sub_headers[0]['Y_PIXEL_SIZE']*10, 0, img_nii.header['qoffset_y']]) - img_nii.header['srow_z'] = numpy.array([0, 0, sub_headers[0]['Z_PIXEL_SIZE']*10, img_nii.header['qoffset_z']]) - - img_nii.header['intent_name'] = '' - img_nii.header['magic'] = 'n + 1 ' - - # nifti header items to include - img_nii.header.set_xyzt_units('mm', 'unknown') - - # save nifti + # debug step 10, check to see what's happened after we've converted our numpy array in to a nibabel object + if ecat_save_steps == "1": + helper_functions.first_middle_last_frames_to_text( + four_d_array_like_object=img_nii.dataobj, + output_folder=steps_dir, + step_name="10_save_nii_ecat2nii_python", + ) + + img_nii.header.set_slope_inter(slope=1, inter=0) + img_nii.header.set_xyzt_units("mm", "sec") + img_nii.header.set_qform(affine, code=1) + img_nii.header.set_sform(affine, code=1) + # No setter methods for these + img_nii.header["cal_max"] = final_image.max() + img_nii.header["cal_min"] = final_image.min() + img_nii.header["descrip"] = "OpenNeuroPET ecat2nii.py conversion" + nibabel.save(img_nii, nifti_file) + # run step 11 in debug + if ecat_save_steps == "1": + # load nifti file with nibabel + written_img_nii = nibabel.load(nifti_file) + + helper_functions.first_middle_last_frames_to_text( + four_d_array_like_object=written_img_nii.dataobj, + output_folder=steps_dir, + step_name="11_read_saved_nii_python", + ) + # used for testing veracity of nibabel read and write. if save_binary: - pickle.dump(img_nii, open(nifti_file + '.pickle', "wb")) + pickle.dump(img_nii, open(nifti_file + ".pickle", "wb")) # write out timing file if sif_out: - with open(os.path.join(output_folder, nifti_file_w_out_extension + '.sif'), 'w') as sif_file: - scantime = datetime.datetime.fromtimestamp(main_header['SCAN_START_TIME']) + with open( + os.path.join(output_folder, nifti_file_w_out_extension + ".sif"), "w" + ) as sif_file: + scantime = datetime.datetime.fromtimestamp(main_header["SCAN_START_TIME"]) scantime = scantime.astimezone().isoformat() sif_file.write(f"{scantime} {len(start)} 4 1\n") for index in reversed(range(len(start))): diff --git a/pypet2bids/pypet2bids/ecat_cli.py b/pypet2bids/pypet2bids/ecat_cli.py index 5e5a872d..5c6b831f 100644 --- a/pypet2bids/pypet2bids/ecat_cli.py +++ b/pypet2bids/pypet2bids/ecat_cli.py @@ -4,23 +4,26 @@ | *Authors: Anthony Galassi* | *Copyright OpenNeuroPET team* """ + import argparse import os import pathlib import sys import textwrap from os.path import join -from pypet2bids.ecat import Ecat +from importlib.metadata import version try: import helper_functions + import Ecat + from update_json_pet_file import check_json, check_meta_radio_inputs except ModuleNotFoundError: import pypet2bids.helper_functions as helper_functions + from pypet2bids.ecat import Ecat + from pypet2bids.update_json_pet_file import check_json, check_meta_radio_inputs -#from pypet2bids.helper_functions import load_vars_from_config, ParseKwargs - - -epilog = textwrap.dedent(''' +epilog = textwrap.dedent( + """ example usage: @@ -31,7 +34,8 @@ params For additional (highly verbose) example usage call this program with the --show-examples flag. -''') +""" +) def cli(): @@ -62,44 +66,136 @@ def cli(): :type --director_table: flag :param --show-examples: shows verbose example usage of this cli :type --show-examples: flag + :param --metadata-path: path to a spreadsheet containing PET metadata + :type --metadata-path: path :return: argparse.ArgumentParser.args for later use in executing conversions or ECAT methods """ - parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,epilog=epilog) - parser.add_argument("ecat", nargs='?', metavar="ecat_file", help="Ecat image to collect info from.") - parser.add_argument("--affine", "-a", help="Show affine matrix", action="store_true", default=False) - parser.add_argument("--convert", "-c", required=False, action='store_true', - help="If supplied will attempt conversion.") - parser.add_argument("--dump", "-d", help="Dump information in Header", action="store_true", default=False) - parser.add_argument("--json", "-j", action="store_true", default=False, help=""" - Output header and subheader info as JSON to stdout, overrides all other options""") - parser.add_argument("--nifti", "-n", metavar="file_name", help="Name of nifti output file", required=False, - default=None) - parser.add_argument("--subheader", '-s', help="Display subheaders", action="store_true", default=False) - parser.add_argument("--sidecar", action="store_true", help="Output a bids formatted sidecar for pairing with" - "a nifti.") - parser.add_argument('--kwargs', '-k', nargs='*', action=helper_functions.ParseKwargs, default={}, - help="Include additional values in the nifti sidecar json or override values extracted from " - "the supplied nifti. e.g. including `--kwargs TimeZero=\"12:12:12\"` would override the " - "calculated TimeZero. Any number of additional arguments can be supplied after --kwargs " - "e.g. `--kwargs BidsVariable1=1 BidsVariable2=2` etc etc." - "Note: the value portion of the argument (right side of the equal's sign) should always" - "be surrounded by double quotes BidsVarQuoted=\"[0, 1 , 3]\"") - parser.add_argument('--scannerparams', nargs='*', - help="Loads saved scanner params from a configuration file following " - "--scanner-params/-s if this option is used without an argument " - "this cli will look for any scanner parameters file in the " - "directory with the name *parameters.txt from which this cli is " - "called.") - parser.add_argument("--directory_table", '-t', help="Collect table/array of ECAT frame byte location map", - action="store_true", default=False) - parser.add_argument('--show-examples', '-E', '--HELP', '-H', help='Shows example usage of this cli.', - action='store_true') + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=epilog, + description="Extracts information from an ECAT file and/or converts ECAT imaging" + " and PET metadata files to BIDS compliant nifti, json, and tsv", + ) + update_or_convert = parser.add_mutually_exclusive_group() + parser.add_argument( + "ecat", nargs="?", metavar="ecat_file", help="Ecat image to collect info from." + ) + parser.add_argument( + "--affine", "-a", help="Show affine matrix", action="store_true", default=False + ) + update_or_convert.add_argument( + "--convert", + "-c", + required=False, + action="store_true", + help="If supplied will attempt conversion.", + ) + parser.add_argument( + "--dump", + "-d", + help="Dump information in Header", + action="store_true", + default=False, + ) + parser.add_argument( + "--json", + "-j", + action="store_true", + default=False, + help=""" + Output header and subheader info as JSON to stdout, overrides all other options""", + ) + parser.add_argument( + "--nifti", + "-n", + metavar="file_name", + help="Name of nifti output file", + required=False, + default=None, + ) + parser.add_argument( + "--subheader", + "-s", + help="Display subheaders", + action="store_true", + default=False, + ) + parser.add_argument( + "--sidecar", + action="store_true", + help="Output a bids formatted sidecar for pairing with" "a nifti.", + ) + parser.add_argument( + "--kwargs", + "-k", + nargs="*", + action=helper_functions.ParseKwargs, + default={}, + help="Include additional values in the nifti sidecar json or override values extracted from " + 'the supplied nifti. e.g. including `--kwargs TimeZero="12:12:12"` would override the ' + "calculated TimeZero. Any number of additional arguments can be supplied after --kwargs " + "e.g. `--kwargs BidsVariable1=1 BidsVariable2=2` etc etc." + "Note: the value portion of the argument (right side of the equal's sign) should always" + 'be surrounded by double quotes BidsVarQuoted="[0, 1 , 3]"', + ) + parser.add_argument( + "--scannerparams", + nargs="*", + help="Loads saved scanner params from a configuration file following " + "--scanner-params/-s if this option is used without an argument " + "this cli will look for any scanner parameters file in the " + "directory with the name *parameters.txt from which this cli is " + "called.", + ) + parser.add_argument( + "--directory_table", + "-t", + help="Collect table/array of ECAT frame byte location map", + action="store_true", + default=False, + ) + parser.add_argument( + "--show-examples", + "-E", + "--HELP", + "-H", + help="Shows example usage of this cli.", + action="store_true", + ) + parser.add_argument( + "--metadata-path", "-m", help="Path to a spreadsheet containing PET metadata." + ) + update_or_convert.add_argument( + "--update", + "-u", + type=str, + default="", + help="Update/create a json sidecar file from an ECAT given a path to that each " + "file,. e.g." + "ecatpet2bids ecatfile.v --update path/to/sidecar.json " + "additionally one can pass metadat to the sidecar via inclusion of the " + "--kwargs flag or" + "the --metadata-path flag. If both are included the --kwargs flag will " + "override any" + "overlapping values in the --metadata-path flag or found in the ECAT file \n" + "ecatpet2bids ecatfile.v --update path/to/sidecar.json --kwargs " + 'TimeZero="12:12:12"' + "ecatpet2bids ecatfile.v --update path/to/sidecar.json --metadata-path " + "path/to/metadata.xlsx", + ) + parser.add_argument( + "--version", + "-v", + action="version", + version=f"{helper_functions.get_version()}", + ) return parser -example1 = textwrap.dedent(''' +example1 = textwrap.dedent( + """ Usage examples are below, the first being the most brutish way of injecting BIDS required fields into the output from ecatpet2bids. Additional arguments/fields are passed via the kwargs flag @@ -144,7 +240,8 @@ def cli(): InjectionStart=0 InjectedRadioactivityUnits="Bq" ReconFilterType="['n/a']" -''') +""" +) def main(): @@ -166,7 +263,7 @@ def main(): sys.exit(0) collect_pixel_data = False - if cli_args.convert: + if cli_args.convert or cli_args.update: collect_pixel_data = True if cli_args.scannerparams is not None: # if no args are supplied to --scannerparams/-s @@ -179,9 +276,11 @@ def main(): break if scanner_txt is None: called_dir = os.getcwd() - error_string = f'No scanner file found in {called_dir}. Either create a parameters.txt file, omit ' \ - f'the --scannerparams argument, or specify a full path to a scanner.txt file after the '\ - f'--scannerparams argument.' + error_string = ( + f"No scanner file found in {called_dir}. Either create a parameters.txt file, omit " + f"the --scannerparams argument, or specify a full path to a scanner.txt file after the " + f"--scannerparams argument." + ) raise Exception(error_string) else: scanner_txt = cli_args.scannerparams[0] @@ -193,9 +292,13 @@ def main(): scanner_params.update(cli_args.kwargs) cli_args.kwargs.update(scanner_params) - ecat = Ecat(ecat_file=cli_args.ecat, - nifti_file=cli_args.nifti, - collect_pixel_data=collect_pixel_data) + ecat = Ecat( + ecat_file=cli_args.ecat, + nifti_file=cli_args.nifti, + collect_pixel_data=collect_pixel_data, + metadata_path=cli_args.metadata_path, + kwargs=cli_args.kwargs, + ) if cli_args.json: ecat.json_out() sys.exit(0) @@ -214,11 +317,58 @@ def main(): ecat.populate_sidecar(**cli_args.kwargs) ecat.show_sidecar() if cli_args.convert: - output_path = pathlib.Path(ecat.make_nifti()) - ecat.populate_sidecar(**cli_args.kwargs) - ecat.prune_sidecar() - sidecar_path = pathlib.Path(join(str(output_path.parent), output_path.stem + '.json')) - ecat.show_sidecar(output_path=sidecar_path) + ecat.convert() + if cli_args.update: + ecat.update_pet_json(cli_args.update) + + +def update_json_with_ecat_value_cli(): + """ + Updates a json sidecar with values extracted from an ecat file, optionally additional values can be included + via the -k --additional-arguments flag and/or a metadata spreadsheet can be supplied via the --metadata-path flag. + Command can be accessed after installation via `upadatepetjsonfromecat` + """ + json_update_cli = argparse.ArgumentParser( + description="Updates a json sidecar with values extracted from an ECAT." + ) + json_update_cli.add_argument( + "-j", "--json", help="Path to a json to update file.", required=True + ) + json_update_cli.add_argument( + "-e", "--ecat", help="Path to an ecat file.", required=True + ) + json_update_cli.add_argument( + "-m", "--metadata-path", help="Path to a spreadsheet containing PET metadata." + ) + json_update_cli.add_argument( + "-k", + "--additional-arguments", + nargs="*", + action=helper_functions.ParseKwargs, + default={}, + help="Include additional values in the sidecar json or override values extracted " + "from the supplied ECAT or metadata spreadsheet. " + 'e.g. including `--kwargs TimeZero="12:12:12"` would override the calculated ' + "TimeZero." + "Any number of additional arguments can be supplied after --kwargs e.g. `--kwargs" + "BidsVariable1=1 BidsVariable2=2` etc etc." + "Note: the value portion of the argument (right side of the equal's sign) should " + 'always be surrounded by double quotes BidsVarQuoted="[0, 1 , 3]"', + ) + + args = json_update_cli.parse_args() + + update_ecat = Ecat( + ecat_file=args.ecat, + nifti_file=None, + collect_pixel_data=True, + metadata_path=args.metadata_path, + kwargs=args.additional_arguments, + ) + update_ecat.update_pet_json(args.json) + + # lastly check the json + check_json(args.json, logger="check_json", silent=False) if __name__ == "__main__": diff --git a/pypet2bids/pypet2bids/ecat_header_update.py b/pypet2bids/pypet2bids/ecat_header_update.py new file mode 100644 index 00000000..2c54878e --- /dev/null +++ b/pypet2bids/pypet2bids/ecat_header_update.py @@ -0,0 +1,77 @@ +try: + import ecat + import write_ecat + import read_ecat + import helper_functions +except ImportError: + from pypet2bids.ecat import ecat + from pypet2bids.write_ecat import write_ecat + from pypet2bids.read_ecat import read_ecat + from pypet2bids import helper_functions + +# collect ecat header jsons +ecat_headers = read_ecat.ecat_header_maps.get("ecat_headers") + + +def update_ecat_header(ecat_file: str, new_values: dict): + """ + Update the header of an ECAT file with new values + :param ecat_file: path to the ECAT file + :param new_values: dictionary of new values to update + :param ecat_header: dictionary of the ECAT header + :return: None + """ + + # read ecat and determine version of ecat file + print(f"Reading ECAT file {ecat_file}") + infile = ecat.Ecat(ecat_file) + + # collect the appropriate header schema + sw_version = str(infile.ecat_header.get("SW_VERSION", 73)) + + infile_header_map = ecat_headers[sw_version]["mainheader"] + + # iterate through new values and update the header + for name, value in new_values.items(): + if infile.ecat_header.get(name): + if type(infile.ecat_header[name]) == type(value): + infile.ecat_header[name] = value + else: + print( + f"WARNING: {name} has type {type(infile.ecat_header[name])} " + f"and new value {value} has type {type(value)}" + ) + else: + print( + f"WARNING: {name} not found in header schema for ECAT {ecat_headers.ecat_header.get('SW_VERSION', 73)} " + f"not updating with value {value}" + ) + + # update the header of the ecat file in question + with open(infile.ecat_file, "r+b") as outfile: + write_ecat.write_header( + ecat_file=outfile, schema=infile_header_map, values=infile.ecat_header + ) + + +def cli(): + import argparse + + parser = argparse.ArgumentParser(description="Update the header of an ECAT file.") + parser.add_argument("ecat_file", type=str, help="path to the ECAT file") + parser.add_argument( + "new_values", + nargs="*", + action=helper_functions.ParseKwargs, + default={}, + help="new values to update the MAINHEADER of the ecat file, e.g. NUM_FRAMES=71 " + "or CALIBRATION_FACTOR=0.5. " + 'or STUDY_DESCRIPTION="very important work"' + "If the value is a string, it must be in quotes.", + ) + args = parser.parse_args() + update_ecat_header(ecat_file=args.ecat_file, new_values=args.new_values) + + +if __name__ == "__main__": + cli() diff --git a/pypet2bids/pypet2bids/ecat_headers.json b/pypet2bids/pypet2bids/ecat_headers.json index ba461876..1f626676 100644 --- a/pypet2bids/pypet2bids/ecat_headers.json +++ b/pypet2bids/pypet2bids/ecat_headers.json @@ -971,6 +971,1833 @@ } ] }, + "70": { + "mainheader": [ + { + "byte": 0, + "variable_name": "MAGIC_NUMBER", + "type": "Character*14", + "comment": "UNIX file type identification number (NOT PART OF THE MATRIX HEADER DATA)", + "struct": "14s" + }, + { + "byte": 14, + "variable_name": "ORIGINAL_FILE_NAME", + "type": "Character*32", + "comment": "Scan file\u2019s creation name", + "struct": "32s" + }, + { + "byte": 46, + "variable_name": "SW_VERSION", + "type": "Integer*2", + "comment": "Software version number", + "struct": "H" + }, + { + "byte": 48, + "variable_name": "SYSTEM TYPE", + "type": "Integer*2", + "comment": "Scanner model (i.e., 951, 951R, 953, 953B, 921, 922, 925, 961, 962, 966)", + "struct": "H" + }, + { + "byte": 50, + "variable_name": "FILE_TYPE", + "type": "Integer*2", + "comment": "Enumerated type (00=unknown, 01=Sinogram, 02=Image-16, 03=Attenuation Correction, 04=Normalization, 05=Polar Map, 06=Volume 8, 07=Volume 16, 08=Projection 8, 09=Projection 16, 10=Image 8, 11=3D Sinogram 16, 12=3D Sinogram 8, 13=3D Normalization, 14=3D Sinogram Fit)", + "struct": "H" + }, + { + "byte": 52, + "variable_name": "SERIAL_NUMBER", + "type": "Character*10", + "comment": "The serial number of the gantry", + "struct": "10s" + }, + { + "byte": 62, + "variable_name": "SCAN_START_TIME", + "type": "Integer*4", + "comment": "Date and time that acquisition was started (in secs from base time)", + "struct": "l" + }, + { + "byte": 66, + "variable_name": "ISOTOPE_NAME", + "type": "Character*8", + "comment": "Isotope", + "struct": "8s" + }, + { + "byte": 74, + "variable_name": "ISOTOPE_HALFLIFE", + "type": "Real*4", + "comment": "Half-life of isotope specified (in sec.)", + "struct": "f" + }, + { + "byte": 78, + "variable_name": "RADIOPHARMACEUTICAL", + "type": "Character*32", + "comment": "Free format ASCII", + "struct": "32s" + }, + { + "byte": 110, + "variable_name": "GANTRY_TILT", + "type": "Real*4", + "comment": "Angle (in degrees)", + "struct": "f" + }, + { + "byte": 114, + "variable_name": "GANTRY_ROTATION", + "type": "Real*4", + "comment": "Angle (in degrees)", + "struct": "f" + }, + { + "byte": 118, + "variable_name": "BED_ELEVATION", + "type": "Real*4", + "comment": "Bed height (in cm.) from lowest point", + "struct": "f" + }, + { + "byte": 122, + "variable_name": "INTRINSIC_TILT", + "type": "Real*4", + "comment": "Angle (in degrees),Angle that the first detector of Bucket 0 is offset from top center (in degrees)", + "struct": "f" + }, + { + "byte": 126, + "variable_name": "WOBBLE_SPEED", + "type": "Integer*2", + "comment": "Revolutions/minute (0 if not wobbled)", + "struct": "H" + }, + { + "byte": 128, + "variable_name": "TRANSM_SOURCE_TYPE", + "type": "Integer*2", + "comment": "Enumerated type (SRC_NONE, _RRING, _RING, _ROD, _RROD)", + "struct": "H" + }, + { + "byte": 130, + "variable_name": "DISTANCE_SCANNED", + "type": "Real*4", + "comment": "Total distance scanned (in cm)", + "struct": "f" + }, + { + "byte": 134, + "variable_name": "TRANSAXIAL_FOV", + "type": "Real*4", + "comment": "Diameter (in cm.) of transaxial view", + "struct": "f" + }, + { + "byte": 138, + "variable_name": "ANGULAR_COMPRESSION", + "type": "Integer*2", + "comment": "Enumerated Type (0=no mash,1=mash of 2, 2=mash of 4)", + "struct": "H" + }, + { + "byte": 140, + "variable_name": "COIN_SAMP_MODE", + "type": "Integer*2", + "comment": "Enumerated type (0=Net Trues, 1=Prompts and Delayed, 3= Prompts, Delayed, and Multiples)", + "struct": "H" + }, + { + "byte": 142, + "variable_name": "AXIAL_SAMP_MODE", + "type": "Integer*2", + "comment": "Enumerated type (0=Normal, 1=2X, 2=3X)", + "struct": "H" + }, + { + "byte": 144, + "variable_name": "ECAT_CALIBRATION_FACTOR", + "type": "Real*4", + "comment": "Quantification scale factor (to convert from ECAT counts to activity counts)", + "struct": "f" + }, + { + "byte": 148, + "variable_name": "CALIBRATION_UNITS", + "type": "Integer*2", + "comment": "Enumerated type (0=Uncalibrated, 1=Calibrated,)", + "struct": "H" + }, + { + "byte": 150, + "variable_name": "CALIBRATION_UNITS_LABE", + "type": "Integer*2", + "comment": "Enumerated type (BLOOD_FLOW, LMRGLU)", + "struct": "H" + }, + { + "byte": 152, + "variable_name": "COMPRESSION_CODE", + "type": "Integer*2", + "comment": "Enumerated type (COMP_NONE, (This is the only value))", + "struct": "H" + }, + { + "byte": 154, + "variable_name": "STUDY_TYPE", + "type": "Character*12", + "comment": "Study descriptor", + "struct": "12s" + }, + { + "byte": 166, + "variable_name": "PATIENT_ID", + "type": "Character*16", + "comment": "Patient identification descriptor", + "struct": "16s" + }, + { + "byte": 182, + "variable_name": "PATIENT_NAME", + "type": "Character*32", + "comment": "Patient name (free format ASCII)", + "struct": "32s" + }, + { + "byte": 214, + "variable_name": "PATIENT_SEX", + "type": "Character*1", + "comment": "Enumerated type (SEX_MALE, _FEMALE, _UNKNOWN)", + "struct": "1s" + }, + { + "byte": 215, + "variable_name": "PATIENT_DEXTERITY", + "type": "Character*1", + "comment": "Enumerated type (DEXT_RT, _LF, _UNKNOWN)", + "struct": "1s" + }, + { + "byte": 216, + "variable_name": "PATIENT_AGE", + "type": "Real*4", + "comment": "Patient age (years)", + "struct": "f" + }, + { + "byte": 220, + "variable_name": "PATIENT_HEIGHT", + "type": "Real*4", + "comment": "Patient height (cm)", + "struct": "f" + }, + { + "byte": 224, + "variable_name": "PATIENT_WEIGHT", + "type": "Real*4", + "comment": "Patient weight (kg)", + "struct": "f" + }, + { + "byte": 228, + "variable_name": "PATIENT_BIRTH_DATE", + "type": "Integer*4", + "comment": "Format is YYYYMMDD", + "struct": "I" + }, + { + "byte": 232, + "variable_name": "PHYSICIAN_NAME", + "type": "Character*32", + "comment": "Attending Physician name (free format)", + "struct": "32s" + }, + { + "byte": 264, + "variable_name": "OPERATOR_NAME", + "type": "Character*32", + "comment": "Operator name (free format)", + "struct": "32s" + }, + { + "byte": 296, + "variable_name": "STUDY_DESCRIPTION", + "type": "Character*32", + "comment": "Free format ASCII", + "struct": "32s" + }, + { + "byte": 328, + "variable_name": "ACQUISITION_TYPE", + "type": "Integer*2", + "comment": "Enumerated type (0=Undefined, 1=Blank, 2=Transmission, 3=Static emission, 4=Dynamic emission, 5=Gated emission, 6=Transmission rectilinear, 7=Emission rectilinear)", + "struct": "H" + }, + { + "byte": 330, + "variable_name": "PATIENT_ORIENTATION", + "type": "Integer*2", + "comment": "Enumerated Type (Bit 0 clear - Feet first, Bit 0 set - Head first, Bit 1-2 00 - Prone, Bit 1-2 01 - Supine, Bit 1-2 10 - Decubitus Right, Bit 1-2 11 - Decubitus Left)", + "struct": "H" + }, + { + "byte": 332, + "variable_name": "FACILITY_NAME", + "type": "Character*20", + "comment": "Free format ASCII", + "struct": "20s" + }, + { + "byte": 352, + "variable_name": "NUM_PLANES", + "type": "Integer*2", + "comment": "Number of planes of data collected", + "struct": "H" + }, + { + "byte": 354, + "variable_name": "NUM_FRAMES", + "type": "Integer*2", + "comment": "Number of frames of data collected OR Highest frame number (in partially reconstructed files)", + "struct": "H" + }, + { + "byte": 356, + "variable_name": "NUM_GATES", + "type": "Integer*2", + "comment": "Number of gates of data collected", + "struct": "H" + }, + { + "byte": 358, + "variable_name": "NUM_BED_POS", + "type": "Integer*2", + "comment": "Number of bed positions of data collected", + "struct": "H" + }, + { + "byte": 360, + "variable_name": "INIT_BED_POSITION", + "type": "Real*4", + "comment": "Absolute location of initial bed position (in cm.)", + "struct": "f" + }, + { + "byte": 364, + "variable_name": "BED_POSITION(15)", + "type": "Real*4", + "comment": "Absolute bed location (in cm.)", + "struct": "15f" + }, + { + "byte": 424, + "variable_name": "PLANE_SEPARATION", + "type": "Real*4", + "comment": "Physical distance between adjacent planes (in cm.)", + "struct": "f" + }, + { + "byte": 428, + "variable_name": "LWR_SCTR_THRES", + "type": "Integer*2", + "comment": "Lowest threshold setting for scatter (in KeV)", + "struct": "H" + }, + { + "byte": 430, + "variable_name": "LWR_TRUE_THRES", + "type": "Integer*2", + "comment": "Lower threshold setting for trues in (in KeV)", + "struct": "H" + }, + { + "byte": 432, + "variable_name": "UPR_TRUE_THRES", + "type": "Integer*2", + "comment": "Upper threshold setting for trues (in KeV)", + "struct": "H" + }, + { + "byte": 434, + "variable_name": "USER_PROCESS_CODE", + "type": "Character*10", + "comment": "Data processing code (defined by user)", + "struct": "10s" + }, + { + "byte": 444, + "variable_name": "ACQUISITION_MODE", + "type": "Integer*2", + "comment": "Enumerated Type (0=Normal, 1=Windowed, 2=Windowed & Nonwindowed, 3=Dual energy, 4=Upper energy, 5=Emission and Transmission)", + "struct": "H" + }, + { + "byte": 446, + "variable_name": "BIN_SIZE", + "type": "Real*4", + "comment": "Width of view sample (in cm)", + "struct": "f" + }, + { + "byte": 450, + "variable_name": "BRANCHING_FRACTION", + "type": "Real*4", + "comment": "Fraction of decay by positron emission", + "struct": "f" + }, + { + "byte": 454, + "variable_name": "DOSE_START_TIME", + "type": "Integer*4", + "comment": "Actual time radiopharmaceutical was injected or flow was started (in sec since base time)", + "struct": "l" + }, + { + "byte": 458, + "variable_name": "DOSAGE", + "type": "Real*4", + "comment": "Radiopharmaceutical dosage (in bequerels/cc) at time of injection", + "struct": "f" + }, + { + "byte": 462, + "variable_name": "WELL_COUNTER_CORR_FACTOR", + "type": "Real*4", + "comment": "TBD", + "struct": "f" + }, + { + "byte": 466, + "variable_name": "DATA_UNITS", + "type": "Character*32", + "struct": "32s" + }, + { + "byte": 498, + "variable_name": "SEPTA_STATE", + "type": "Integer*2", + "comment": "Septa position during scan (0=septa extended, 1=septa retracted)", + "struct": "H" + }, + { + "byte": 500, + "variable_name": "FILL(6)", + "type": "Integer*2", + "comment": "CTI Reserved space (12 bytes)", + "struct": "6H" + } + ], + "3": [ + { + "byte": 0, + "variable_name": "DATA_TYPE", + "type": "Integer*2", + "comment": "Enumerated type (DTYPE_BYTES, _I2, _I4, _VAXR4, _SUNFL, _SUNIN)", + "struct": "H" + }, + { + "byte": 2, + "variable_name": "NUM_DIMENSIONS", + "type": "Integer*2", + "comment": "Number of dimensions", + "struct": "H" + }, + { + "byte": 4, + "variable_name": "ATTENUATION_TYPE", + "type": "Integer*2", + "comment": "E. type (ATTEN_NONE, _MEAS, _CALC)", + "struct": "H" + }, + { + "byte": 6, + "variable_name": "NUM_R_ELEMENTS", + "type": "Integer*2", + "comment": "Total elements collected (x dimension)", + "struct": "H" + }, + { + "byte": 8, + "variable_name": "NUM_ANGLES", + "type": "Integer*2", + "comment": "Total views collected (y dimensions)", + "struct": "H" + }, + { + "byte": 10, + "variable_name": "NUM_Z_ELEMENTS", + "type": "Integer*2", + "comment": "Total elements collected (z dimension)", + "struct": "H" + }, + { + "byte": 12, + "variable_name": "RING_DIFFERENCE", + "type": "Integer*2", + "comment": "Maximum acceptance angle.", + "struct": "H" + }, + { + "byte": 14, + "variable_name": "X_RESOLUTION", + "type": "Real*4", + "comment": "Resolution in the x dimension (in cm)", + "struct": "f" + }, + { + "byte": 18, + "variable_name": "Y_RESOLUTION", + "type": "Real*4", + "comment": "Resolution in the y dimension (in cm)", + "struct": "f" + }, + { + "byte": 22, + "variable_name": "Z_RESOLUTION", + "type": "Real*4", + "comment": "Resolution in the z dimension (in cm)", + "struct": "f" + }, + { + "byte": 26, + "variable_name": "W_RESOLUTION", + "type": "Real*4", + "comment": "TBD", + "struct": "f" + }, + { + "byte": 30, + "variable_name": "SCALE_FACTOR", + "type": "Real*4", + "comment": "Attenuation Scale Factor", + "struct": "f" + }, + { + "byte": 34, + "variable_name": "X_OFFSET", + "type": "Real*4", + "comment": "Ellipse offset in x axis from center (in cm.)", + "struct": "f" + }, + { + "byte": 38, + "variable_name": "Y_OFFSET", + "type": "Real*4", + "comment": "Ellipse offset in y axis from center (in cm.)", + "struct": "f" + }, + { + "byte": 42, + "variable_name": "X_RADIUS", + "type": "Real*4", + "comment": "Ellipse radius in x axis (in cm.)", + "struct": "f" + }, + { + "byte": 46, + "variable_name": "Y_RADIUS", + "type": "Real*4", + "comment": "Ellipse radius in y axis (in cm.)", + "struct": "f" + }, + { + "byte": 50, + "variable_name": "TILT_ANGLE", + "type": "Real*4", + "comment": "Tilt angel of the ellipse (in degrees)", + "struct": "f" + }, + { + "byte": 54, + "variable_name": "ATTENUATION_COEFF", + "type": "Real*4", + "comment": "M u-absorption coefficient (in cm^-1)", + "struct": "f" + }, + { + "byte": 58, + "variable_name": "ATTENUATION_MIN", + "type": "Real*4", + "comment": "Minimum value in the attenuation data", + "struct": "f" + }, + { + "byte": 62, + "variable_name": "ATTENUATION_MAX", + "type": "Real*4", + "comment": "Maximum value in the attentuation data", + "struct": "f" + }, + { + "byte": 66, + "variable_name": "SKULL_THICKNESS", + "type": "Real*4", + "comment": "Skull thickness in cm", + "struct": "f" + }, + { + "byte": 70, + "variable_name": "NUM_ADDITIONAL_ATTN_COEFF", + "type": "Integer*2", + "comment": "Number of attenuation coefficients other than the Mu absorption coefficient above (max 8)", + "struct": "H" + }, + { + "byte": 72, + "variable_name": "ADDITIONAL_ATTEN_COEFF(8)", + "type": "Real*4", + "comment": "The additional attention coefficient values", + "struct": "8f" + }, + { + "byte": 104, + "variable_name": "EDGE_FINDING_THRESHOLD", + "type": "Real*4", + "comment": "The threshold value used by automatic edge-detection routine (fraction of maximum)", + "struct": "f" + }, + { + "byte": 108, + "variable_name": "STORAGE_ORDER", + "type": "Integer*2", + "comment": "Data storage order (RThetaZD, RZThetaD)", + "struct": "H" + }, + { + "byte": 110, + "variable_name": "SPAN", + "type": "Integer*2", + "comment": "Axial compression specifier (number of ring differences spanned by a segment)", + "struct": "H" + }, + { + "byte": 112, + "variable_name": "Z_ELEMENTS(64)", + "type": "Integer*2", + "comment": "Number of 'planes' in z direction for each ring difference segment", + "struct": "64H" + }, + { + "byte": 240, + "variable_name": "FILL(86)", + "type": "Integer*2", + "comment": "Unused (172 bytes)", + "struct": "86H" + }, + { + "byte": 412, + "variable_name": "FILL(50)", + "type": "Integer*2", + "comment": "User Reserved space (100 bytes) Note: use highest bytes first", + "struct": "50H" + } + ], + "7": [ + { + "byte": 0, + "variable_name": "DATA_TYPE", + "type": "Integer*2", + "comment": "Enumerated type (0=Unkonwn Matrix Data Type, 1=Byte Data, 2=VAX_Ix2, 3=VAX_Ix4, 4=VAX_Rx4, 5=IEEE Float, 6=Sun short, 7=Sun long)", + "struct": "H" + }, + { + "byte": 2, + "variable_name": "NUM_DIMENSIONS", + "type": "Integer*2", + "comment": "Number of dimensions", + "struct": "H" + }, + { + "byte": 4, + "variable_name": "X_DIMENSION", + "type": "Integer*2", + "comment": "Dimension along x axis", + "struct": "H" + }, + { + "byte": 6, + "variable_name": "Y_DIMENSION", + "type": "Integer*2", + "comment": "Dimension along y axis", + "struct": "H" + }, + { + "byte": 8, + "variable_name": "Z_DIMENSION", + "type": "Integer*2", + "comment": "Dimension along z axis", + "struct": "H" + }, + { + "byte": 10, + "variable_name": "X_OFFSET", + "type": "Real*4", + "comment": "Offset in x axis for recon target (in cm)", + "struct": "f" + }, + { + "byte": 14, + "variable_name": "Y_OFFSET", + "type": "Real*4", + "comment": "Offset in y axis for recon target (in cm)", + "struct": "f" + }, + { + "byte": 18, + "variable_name": "Z_OFFSET", + "type": "Real*4", + "comment": "Offset in z axis for recon target (in cm)", + "struct": "f" + }, + { + "byte": 22, + "variable_name": "RECON_ZOOM", + "type": "Real*4", + "comment": "Reconstruction magnification factor (zoom)", + "struct": "f" + }, + { + "byte": 26, + "variable_name": "SCALE_FACTOR", + "type": "Real*4", + "comment": "Quantification scale factor (in Quant_units)", + "struct": "f" + }, + { + "byte": 30, + "variable_name": "IMAGE_MIN", + "type": "Integer*2", + "comment": "Image minimum pixel value", + "struct": "h" + }, + { + "byte": 32, + "variable_name": "IMAGE_MAX", + "type": "Integer*2", + "comment": "Image maximum pixel value", + "struct": "H" + }, + { + "byte": 34, + "variable_name": "X_PIXEL_SIZE", + "type": "Real*4", + "comment": "X dimension pixel size (in cm.)", + "struct": "f" + }, + { + "byte": 38, + "variable_name": "Y_PIXEL_SIZE", + "type": "Real*4", + "comment": "Y dimension pixel size (in cm.)", + "struct": "f" + }, + { + "byte": 42, + "variable_name": "Z_PIXEL_SIZE", + "type": "Real*4", + "comment": "Z dimension pixel size (in cm.)", + "struct": "f" + }, + { + "byte": 46, + "variable_name": "FRAME_DURATION", + "type": "Integer*4", + "comment": "Total duration of current frame (in msec.)", + "struct": "l" + }, + { + "byte": 50, + "variable_name": "FRAME_START_TIME", + "type": "Integer*4", + "comment": "frame start time (offset from first frame, in msec)", + "struct": "l" + }, + { + "byte": 54, + "variable_name": "FILTER_CODE", + "type": "Integer*2", + "comment": "enumerated type (0=all pass, 1=ramp, 2=Butterworth, 3=Hanning, 4=Hamming, 5=Parzen, 6=shepp, 7=butterworth-order 2, 8=Gaussian, 9=Median, 10=Boxcar)", + "struct": "H" + }, + { + "byte": 56, + "variable_name": "X_RESOLUTION", + "type": "Real*4", + "comment": "resolution in the x dimension (in cm)", + "struct": "f" + }, + { + "byte": 60, + "variable_name": "Y_RESOLUTION", + "type": "Real*4", + "comment": "resolution in the y dimension (in cm)", + "struct": "f" + }, + { + "byte": 64, + "variable_name": "Z_RESOLUTION", + "type": "Real*4", + "comment": "resolution in the z dimension (in cm)", + "struct": "f" + }, + { + "byte": 68, + "variable_name": "NUM_R_ELEMENTS", + "type": "Real*4", + "comment": "number r elements from sinogram", + "struct": "f" + }, + { + "byte": 72, + "variable_name": "NUM_ANGLES", + "type": "Real*4", + "comment": "number of angles from sinogram", + "struct": "f" + }, + { + "byte": 76, + "variable_name": "Z_ROTATION_ANGLE", + "type": "Real*4", + "comment": "rotation in the xy plane (in degrees). Use right-hand coordinate system for rotation angle sign.", + "struct": "f" + }, + { + "byte": 80, + "variable_name": "DECAY_CORR_FCTR", + "type": "Real*4", + "comment": "isotope decay compensation applied to data", + "struct": "f" + }, + { + "byte": 84, + "variable_name": "PROCESSING_CODE", + "type": "Integer*4", + "comment": "bit mask (0=not processed, 1=normalized, 2=Measured Attenuation Correction, 4=Calculated attenuation correction, 8=x smoothing, 16=Y smoothing, 32=Z smoothing, 64=2d scatter correction, 128=3D scatter correction, 256=arc correction, 512=decay correction, 1024=Online compression)", + "struct": "l" + }, + { + "byte": 88, + "variable_name": "GATE_DURATION", + "type": "Integer*4", + "comment": "gate duration (in msec)", + "struct": "l" + }, + { + "byte": 92, + "variable_name": "R_WAVE_OFFSET", + "type": "Integer*4", + "comment": "r wave offset (for phase sliced studies, average, in msec)", + "struct": "l" + }, + { + "byte": 96, + "variable_name": "NUM_ACCEPTED_BEATS", + "type": "Integer*4", + "comment": "number of accepted beats for this gate", + "struct": "l" + }, + { + "byte": 100, + "variable_name": "FILTER_CUTOFF_FREQUENCY", + "type": "real*4", + "comment": "cutoff frequency", + "struct": "" + }, + { + "byte": 104, + "variable_name": "FILTER_RESOLUTION", + "type": "Real*4", + "comment": "do not use", + "struct": "f" + }, + { + "byte": 108, + "variable_name": "FILTER_RAMP_SLOPE", + "type": "Real*4", + "comment": "do not use", + "struct": "f" + }, + { + "byte": 112, + "variable_name": "FILTER_ORDER", + "type": "Integer*2", + "comment": "do not use", + "struct": "H" + }, + { + "byte": 114, + "variable_name": "FILTER_SCATTER_FRACTION", + "type": "Real*4", + "comment": "do not use", + "struct": "f" + }, + { + "byte": 118, + "variable_name": "FILTER_SCATTER_SLOPE", + "type": "Real*4", + "comment": "do not use", + "struct": "f" + }, + { + "byte": 122, + "variable_name": "ANNOTATION", + "type": "Character*40", + "comment": "free format ascii", + "struct": "40s" + }, + { + "byte": 162, + "variable_name": "MT_1_1", + "type": "Real*4", + "comment": "matrix transformation element (1,1).", + "struct": "f" + }, + { + "byte": 166, + "variable_name": "MT_1_2", + "type": "Real*4", + "comment": "matrix transformation element (1,2).", + "struct": "f" + }, + { + "byte": 170, + "variable_name": "MT_1_3", + "type": "Real*4", + "comment": "matrix transformation element (1,3).", + "struct": "f" + }, + { + "byte": 174, + "variable_name": "MT_2_1", + "type": "Real*4", + "comment": "matrix transformation element (2,1).", + "struct": "f" + }, + { + "byte": 178, + "variable_name": "MT_2_2", + "type": "Real*4", + "comment": "matrix transformation element (2,2).", + "struct": "f" + }, + { + "byte": 182, + "variable_name": "MT_2_3", + "type": "Real*4", + "comment": "matrix transformation element (2,3).", + "struct": "f" + }, + { + "byte": 186, + "variable_name": "MT_3_1", + "type": "Real*4", + "comment": "matrix transformation element (3,1).", + "struct": "f" + }, + { + "byte": 190, + "variable_name": "MT_3_2", + "type": "Real*4", + "comment": "matrix transformation element (3,2).", + "struct": "f" + }, + { + "byte": 194, + "variable_name": "MT_3_3", + "type": "Real*4", + "comment": "matrix transformation element (3,3).", + "struct": "f" + }, + { + "byte": 198, + "variable_name": "RFILTER_CUTOFF", + "type": "Real*4", + "struct": "f" + }, + { + "byte": 202, + "variable_name": "RFILTER_RESOLUTION", + "type": "Real*4", + "struct": "f" + }, + { + "byte": 206, + "variable_name": "RFILTER_CODE", + "type": "Integer*2", + "struct": "H" + }, + { + "byte": 208, + "variable_name": "RFILTER_ORDER", + "type": "Integer*2", + "struct": "H" + }, + { + "byte": 210, + "variable_name": "ZFILTER_CUTOFF", + "type": "Real*4", + "struct": "f" + }, + { + "byte": 214, + "variable_name": "ZFILTER_RESOLUTION", + "type": "Real*4", + "struct": "f" + }, + { + "byte": 218, + "variable_name": "ZFILTER_CODE", + "type": "Integer*2", + "struct": "H" + }, + { + "byte": 220, + "variable_name": "ZFILTER_ORDER", + "type": "Integer*2", + "struct": "H" + }, + { + "byte": 222, + "variable_name": "MT_1_4", + "type": "Real*4", + "comment": "Matrix transformation element (1,4)", + "struct": "f" + }, + { + "byte": 226, + "variable_name": "MT_2_4", + "type": "Real*4", + "comment": "Matrix transformation element (2,4)", + "struct": "f" + }, + { + "byte": 230, + "variable_name": "MT_3_4", + "type": "Real*4", + "comment": "Matrix transformation element (3,4)", + "struct": "f" + }, + { + "byte": 234, + "variable_name": "SCATTER_TYPE", + "type": "Integer*2", + "comment": "Enumerated type (0=None, 1=Deconvolution, 2=Simulated, 3=Dual Energy)", + "struct": "H" + }, + { + "byte": 236, + "variable_name": "RECON_TYPE", + "type": "Integer*2", + "comment": "Enumerated type (0=Filtered backprojection, 1=Forward projection 3D (PROMIS), 2=Ramp 3D, 3=FAVOR 3D, 4=SSRB, 5=Multi-slice rebinning, 6=FORE)", + "struct": "H" + }, + { + "byte": 238, + "variable_name": "RECON_VIEWS", + "type": "Integer*2", + "comment": "Number of views used to reconstruct the data", + "struct": "H" + }, + { + "byte": 240, + "variable_name": "FILL(87)", + "type": "Integer*2", + "comment": "CTI Reserved space (174 bytes)", + "struct": "87H" + }, + { + "byte": 414, + "variable_name": "FILL(48)", + "type": "Integer*2", + "comment": "User Reserved space (100 bytes) Note: Use highest bytes first", + "struct": "48H" + } + ], + "5": [ + { + "byte": 0, + "variable_name": "DATA_TYPE", + "type": "Integer*2", + "comment": "Enumerated type (DTYPE_BYTES, _I2,_I4)", + "struct": "H" + }, + { + "byte": 2, + "variable_name": "POLAR_MAP_TYPE", + "type": "Integer*2", + "comment": "Enumerated Type (Always 0 for now; denotes the version of the PM structure)", + "struct": "H" + }, + { + "byte": 4, + "variable_name": "NUM_RINGS", + "type": "Integer*2", + "comment": "Number of rings in this polar map", + "struct": "H" + }, + { + "byte": 6, + "variable_name": "SECTORS_PER_RING(32)", + "type": "Integer*2", + "comment": "Number of sectors in each ring for up to 32 rings (1, 9, 18, or 32 sectors normally)", + "struct": "32H" + }, + { + "byte": 70, + "variable_name": "RING_POSITION(32)", + "type": "Real*4", + "comment": "Fractional distance along the long axis from base to apex", + "struct": "32f" + }, + { + "byte": 198, + "variable_name": "RING_ANGLE(32)", + "type": "Integer*2", + "comment": "Ring angle relative to long axis(90 degrees along cylinder, decreasing to 0 at the apex)", + "struct": "32H" + }, + { + "byte": 262, + "variable_name": "START_ANGLE", + "type": "Integer*2", + "comment": "Start angle for rings (Always 258 degrees, defines Polar Map\u2019s 0)", + "struct": "H" + }, + { + "byte": 264, + "variable_name": "LONG_AXIS_LEFT(3)", + "type": "Integer*2", + "comment": "x, y, z location of long axis base end (in pixels)", + "struct": "3H" + }, + { + "byte": 270, + "variable_name": "LONG_AXIS_RIGHT(3)", + "type": "Integer*2", + "comment": "x, y, z location of long axis apex end (in pixels)", + "struct": "3H" + }, + { + "byte": 276, + "variable_name": "POSITION_DATA", + "type": "Integer*2", + "comment": "Enumerated type (0 - Not available, 1 - Present)", + "struct": "H" + }, + { + "byte": 278, + "variable_name": "IMAGE_MIN", + "type": "Integer*2", + "comment": "Minimum pixel value in this polar map", + "struct": "h" + }, + { + "byte": 280, + "variable_name": "IMAGE_MAX", + "type": "Integer*2", + "comment": "Maximum pixel value in this polar map", + "struct": "H" + }, + { + "byte": 282, + "variable_name": "SCALE_FACTOR", + "type": "Real*4", + "comment": "Scale factor to restore integer values to float values", + "struct": "f" + }, + { + "byte": 286, + "variable_name": "PIXEL_SIZE", + "type": "Real*4", + "comment": "Pixel size (in cubic cm, represents voxels)", + "struct": "f" + }, + { + "byte": 290, + "variable_name": "FRAME_DURATION", + "type": "Integer*4", + "comment": "Total duration of current frame (in msec)", + "struct": "l" + }, + { + "byte": 294, + "variable_name": "FRAME_START_TIME", + "type": "Integer*4", + "comment": "Frame start time (offset from first frame, in msec)", + "struct": "l" + }, + { + "byte": 298, + "variable_name": "PROCESSING_CODE", + "type": "Integer*2", + "comment": "Bit Encoded (1- Map type (0 = Sector Analysis, 1 = Volumetric), 2 - Threshold Applied, 3 - Summed Map, 4 - Subtracted Map, 5 - Product of two maps, 6 - Ratio of two maps, 7 - Bias, 8 - Multiplier, 9 - Transform, 10 - Polar Map calculational protocol used)", + "struct": "H" + }, + { + "byte": 300, + "variable_name": "QUANT_UNITS", + "type": "Integer*2", + "comment": "Enumerated Type (0 - Default (see main header), 1 - Normalized, 2 - Mean, 3 - Std. Deviation from Mean)", + "struct": "H" + }, + { + "byte": 302, + "variable_name": "ANNOTATION", + "type": "Character*40", + "comment": "label for polar map display", + "struct": "40s" + }, + { + "byte": 342, + "variable_name": "GATE_DURATION", + "type": "Integer*4", + "comment": "Gate duration (in msec)", + "struct": "l" + }, + { + "byte": 346, + "variable_name": "R_WAVE_OFFSET", + "type": "Integer*4", + "comment": "R wave offset (Average, in msec)", + "struct": "l" + }, + { + "byte": 350, + "variable_name": "NUM_ACCEPTED_BEATS", + "type": "Integer*4", + "comment": "Number of accepted beats for this gate", + "struct": "l" + }, + { + "byte": 354, + "variable_name": "POLAR_MAP_PROTOCOL", + "type": "Character*20", + "comment": "Polar Map protocol used to generate this polar map", + "struct": "20s" + }, + { + "byte": 374, + "variable_name": "DATABASE_NAME", + "type": "Character*30", + "comment": "Database name used for polar map comparison", + "struct": "30s" + }, + { + "byte": 404, + "variable_name": "FILL(27)", + "type": "Integer*2", + "comment": "Reserved for future CTI use (54 bytes)", + "struct": "27H" + }, + { + "byte": 464, + "variable_name": "FILL(27)", + "type": "Integer*2", + "comment": "User reserved space (54 bytes) Note: Use highest bytes first", + "struct": "27H" + } + ], + "11": [ + { + "byte": 0, + "variable_name": "DATA_TYPE", + "type": "Integer*2", + "comment": "Enumerated type (ByteData, SunShortt)", + "struct": "H" + }, + { + "byte": 2, + "variable_name": "NUM_DIMENSIONS", + "type": "Integer*2", + "comment": "Number of Dimensions", + "struct": "H" + }, + { + "byte": 4, + "variable_name": "NUM_R_ELEMENTS", + "type": "Integer*2", + "comment": "Total views collected (\u03b8 dimension)", + "struct": "H" + }, + { + "byte": 6, + "variable_name": "NUM_ANGLES", + "type": "Integer*2", + "comment": "Total views collected (\u03b8 dimension)", + "struct": "H" + }, + { + "byte": 8, + "variable_name": "CORRECTIONS_APPLIED", + "type": "Integer*2", + "comment": "Designates processing applied to scan data (Bit encoded, Bit 0 - Norm, Bit 1 - Atten, Bit 2 - Smooth)", + "struct": "H" + }, + { + "byte": 10, + "variable_name": "NUM_Z_ELEMENTS(64)", + "type": "Integer*2", + "comment": "Number of elements in z dimension for each ring difference segment in 3D scans", + "struct": "64H" + }, + { + "byte": 138, + "variable_name": "RING_DIFFERENCE", + "type": "Integer*2", + "comment": "Max ring difference (d dimension) in this frame", + "struct": "H" + }, + { + "byte": 140, + "variable_name": "STORAGE_ORDER", + "type": "Integer*2", + "comment": "Data storage order (r\u03b8zd or rz\u03b8d)", + "struct": "H" + }, + { + "byte": 142, + "variable_name": "AXIAL_COMPRESSION", + "type": "Integer*2", + "comment": "Axial compression code or factor, generally referred to as SPAN", + "struct": "H" + }, + { + "byte": 144, + "variable_name": "X_RESOLUTION", + "type": "Real*4", + "comment": "Resolution in the r dimension (in cm)", + "struct": "f" + }, + { + "byte": 148, + "variable_name": "V_RESOLUTION", + "type": "Real*4", + "comment": "Resolution in the \u03b8 dimension (in radians)", + "struct": "f" + }, + { + "byte": 152, + "variable_name": "Z_RESOLUTION", + "type": "Real*4", + "comment": "Resolution in the z dimension (in cm)", + "struct": "f" + }, + { + "byte": 156, + "variable_name": "W_RESOLUTION", + "type": "Real*4", + "comment": "Not Used", + "struct": "f" + }, + { + "byte": 160, + "variable_name": "FILL(6)", + "type": "Integer*2", + "comment": "RESERVED for gating", + "struct": "6H" + }, + { + "byte": 172, + "variable_name": "GATE_DURATION", + "type": "Integer*4", + "comment": "Gating segment length (msec, Average time if phased gates are used)", + "struct": "l" + }, + { + "byte": 176, + "variable_name": "R_WAVE_OFFSET", + "type": "Integer*4", + "comment": "Time from start of first gate (Average, in msec.)", + "struct": "l" + }, + { + "byte": 180, + "variable_name": "NUM_ACCEPTED_BEATS", + "type": "Integer*4", + "comment": "Number of accepted beats for this gate", + "struct": "l" + }, + { + "byte": 184, + "variable_name": "SCALE_FACTOR", + "type": "Real*4", + "comment": "If data type is integer, this factor is used to convert to float values", + "struct": "f" + }, + { + "byte": 188, + "variable_name": "SCAN_MIN", + "type": "Integer*2", + "comment": "Minimum value in sinogram if data is in integer form (not currently filled in)", + "struct": "H" + }, + { + "byte": 190, + "variable_name": "SCAN_MAX", + "type": "Integer*2", + "comment": "Maximum value in sinogram if data is in integer form (not currently filled in)", + "struct": "H" + }, + { + "byte": 192, + "variable_name": "PROMPTS", + "type": "Integer*4", + "comment": "Total prompts collected in this frame/gate", + "struct": "l" + }, + { + "byte": 196, + "variable_name": "DELAYED", + "type": "Integer*4", + "comment": "Total delays collected in this frame/gate", + "struct": "l" + }, + { + "byte": 200, + "variable_name": "MULTIPLES", + "type": "Integer*4", + "comment": "Total multiples collected in this frame/gate (notused)", + "struct": "l" + }, + { + "byte": 204, + "variable_name": "NET_TRUES", + "type": "Integer*4", + "comment": "Total net trues (prompts\u2013-randoms)", + "struct": "l" + }, + { + "byte": 208, + "variable_name": "TOT_AVG_COR", + "type": "Real*4", + "comment": "Mean value of loss-corrected singles", + "struct": "f" + }, + { + "byte": 212, + "variable_name": "TOT_AVG_UNCOR", + "type": "Real*4", + "comment": "Mean value of singles (not loss corrected)", + "struct": "f" + }, + { + "byte": 216, + "variable_name": "TOTAL_COIN_RATE", + "type": "Integer*4", + "comment": "Measured coincidence rate (from IPCP)", + "struct": "l" + }, + { + "byte": 220, + "variable_name": "FRAME_START_TIME", + "type": "Integer*4", + "comment": "Time offset from first frame time (in msec.)", + "struct": "l" + }, + { + "byte": 224, + "variable_name": "FRAME_DURATION", + "type": "Integer*4", + "comment": "Total duration of current frame (in msec.)", + "struct": "l" + }, + { + "byte": 228, + "variable_name": "DEADTIME_CORRECTION_FACTOR", + "type": "Real*4", + "comment": "Dead-time correction factor applied to the sinogram", + "struct": "f" + }, + { + "byte": 232, + "variable_name": "FILL(90)", + "type": "Integer*2", + "comment": "CTI Reserved space (180 bytes)", + "struct": "90H" + }, + { + "byte": 412, + "variable_name": "FILL(50)", + "type": "Integer*2", + "comment": "User Reserved space (100 bytes) Note: Use highest bytes first", + "struct": "50H" + }, + { + "byte": 512, + "variable_name": "UNCOR_SINGLES(128)", + "type": "Real*4", + "comment": "Total uncorrected singles from each bucket", + "struct": "128f" + } + ], + "13": [ + { + "byte": 0, + "variable_name": "DATA_TYPE", + "type": "Integer*2", + "comment": "Enumerated type (IeeeFloat)", + "struct": "H" + }, + { + "byte": 2, + "variable_name": "NUM_R_ELEMENTS", + "type": "Integer*2", + "comment": "Total elements collected (y dimension)", + "struct": "H" + }, + { + "byte": 4, + "variable_name": "NUM_TRANSAXIAL_CRYSTALS", + "type": "Integer*2", + "comment": "Number of transaxial crystals per block", + "struct": "H" + }, + { + "byte": 6, + "variable_name": "NUM_CRYSTAL_RINGS", + "type": "Integer*2", + "comment": "Number of crystal rings", + "struct": "H" + }, + { + "byte": 8, + "variable_name": "CRYSTALS_PER_RING", + "type": "Integer*2", + "comment": "Number of crystals per ring", + "struct": "H" + }, + { + "byte": 10, + "variable_name": "NUM_GEO_CORR_PLANES", + "type": "Integer*2", + "comment": "Number of rows in the Plane Geometric Correction array", + "struct": "H" + }, + { + "byte": 12, + "variable_name": "ULD", + "type": "Integer*2", + "comment": "Upper energy limit", + "struct": "H" + }, + { + "byte": 14, + "variable_name": "LLD", + "type": "Integer*2", + "comment": "Lower energy limit", + "struct": "H" + }, + { + "byte": 16, + "variable_name": "SCATTER_ENERGY", + "type": "Integer*2", + "comment": "Scatter energy threshold", + "struct": "H" + }, + { + "byte": 18, + "variable_name": "NORM_QUALITY_FACTOR", + "type": "Real*4", + "comment": "Used by Daily Check to determine the quality of the scanner", + "struct": "f" + }, + { + "byte": 22, + "variable_name": "NORM_QUALITY_FACTOR_CODE", + "type": "Integer*2", + "comment": "Enumerated Type (TBD)", + "struct": "H" + }, + { + "byte": 24, + "variable_name": "RING_DTCOR1(32)", + "type": "Real*4", + "comment": "First \u201cper ring\u201d dead time correction coefficient", + "struct": "32f" + }, + { + "byte": 152, + "variable_name": "RING_DTCOR2(32)", + "type": "Real*4", + "comment": "Second \u201cper ring\u201d dead time correction coefficient", + "struct": "32f" + }, + { + "byte": 280, + "variable_name": "CRYSTAL_DTCOR(8)", + "type": "Real*4", + "comment": "Dead time correction factors for transaxial crystals", + "struct": "8f" + }, + { + "byte": 312, + "variable_name": "SPAN", + "type": "Integer*2", + "comment": "Axial compression specifier (number of ring differences included in each segment)", + "struct": "H" + }, + { + "byte": 314, + "variable_name": "MAX_RING_DIFF", + "type": "Integer*2", + "comment": "Maximum ring difference acquired", + "struct": "H" + }, + { + "byte": 316, + "variable_name": "FILL(48)", + "type": "Integer*2", + "comment": "CTI Reserved space (96 bytes)", + "struct": "48H" + }, + { + "byte": 412, + "variable_name": "FILL(50)", + "type": "Integer*2", + "comment": "User Reserved space (100 bytes) Note: Use highest bytes first", + "struct": "50H" + } + ], + "subheader_imported_6.5_matrix_scan_files": [ + { + "byte": 0, + "variable_name": "DATA_TYPE", + "type": "Integer*2", + "comment": "Enumerated type (DTYPE_BYTES, _I2, _I4, _VAXR4, _SUNFL, _SUNIN)", + "struct": "H" + }, + { + "byte": 2, + "variable_name": "NUM_DIMENSIONS", + "type": "Integer*2", + "comment": "Number of Dimensions", + "struct": "H" + }, + { + "byte": 4, + "variable_name": "NUM_R_ELEMENTS", + "type": "Integer*2", + "comment": "Total elements collected (x dimension)", + "struct": "H" + }, + { + "byte": 6, + "variable_name": "NUM_ANGLES", + "type": "Integer*2", + "comment": "Total views collected (y dimension)", + "struct": "H" + }, + { + "byte": 8, + "variable_name": "CORRECTIONS_APPLIED", + "type": "Integer*2", + "comment": "Designates processing applied to scan data (Bit encoded, Bit 0 - Norm, Bit 1 - Atten, Bit 2 - Smooth)", + "struct": "H" + }, + { + "byte": 10, + "variable_name": "NUM_Z_ELEMENTS", + "type": "Integer*2", + "comment": "Total elements collected (z dimension) For 3D scans", + "struct": "H" + }, + { + "byte": 12, + "variable_name": "RING_DIFFERENCE", + "type": "Integer*2", + "comment": "Maximum acceptance angle", + "struct": "H" + }, + { + "byte": 14, + "variable_name": "X_RESOLUTION", + "type": "Real*4", + "comment": "Resolution in the x dimension (in cm)", + "struct": "f" + }, + { + "byte": 18, + "variable_name": "Y_RESOLUTION", + "type": "Real*4", + "comment": "Resolution in the y dimension (in cm)", + "struct": "f" + }, + { + "byte": 22, + "variable_name": "Z_RESOLUTION", + "type": "Real*4", + "comment": "Resolution in the z dimension (in cm)", + "struct": "f" + }, + { + "byte": 26, + "variable_name": "W_RESOLUTION", + "type": "Real*4", + "comment": "TBD", + "struct": "f" + }, + { + "byte": 30, + "variable_name": "FILL(6)", + "type": "Integer*2", + "comment": "RESERVED for gating", + "struct": "6H" + }, + { + "byte": 42, + "variable_name": "GATE_DURATION", + "type": "Integer*4", + "comment": "Gating segment length (msec, Average time if phased gates are used)", + "struct": "l" + }, + { + "byte": 46, + "variable_name": "R_WAVE_OFFSET", + "type": "Integer*4", + "comment": "Time from start of first gate (Average, in msec.)", + "struct": "l" + }, + { + "byte": 50, + "variable_name": "NUM_ACCEPTED_BEATS", + "type": "Integer*4", + "comment": "Number of accepted beats for this gate", + "struct": "l" + }, + { + "byte": 54, + "variable_name": "SCALE_FACTOR", + "type": "Real*4", + "comment": "If data type=integer, use this factor, convert to float values", + "struct": "f" + }, + { + "byte": 58, + "variable_name": "SCAN_MIN", + "type": "Integer*2", + "comment": "Minimum value in sinogram if data is in integer form", + "struct": "H" + }, + { + "byte": 60, + "variable_name": "SCAN_MAX", + "type": "Integer*2", + "comment": "Maximum value in sinogram if data is in integer form", + "struct": "H" + }, + { + "byte": 62, + "variable_name": "PROMPTS", + "type": "Integer*4", + "comment": "Total prompts collected in this frame/gate", + "struct": "l" + }, + { + "byte": 66, + "variable_name": "DELAYED", + "type": "Integer*4", + "comment": "Total delays collected in thes frame/gate", + "struct": "l" + }, + { + "byte": 70, + "variable_name": "MULTIPLES", + "type": "Integer*4", + "comment": "Total multiples collected in the frame/gate", + "struct": "l" + }, + { + "byte": 74, + "variable_name": "NET_TRUES", + "type": "Integer*4", + "comment": "Total net trues (prompts--randoms)", + "struct": "l" + }, + { + "byte": 78, + "variable_name": "COR_SINGLES(16)", + "type": "Real*4", + "comment": "Total singles with loss correction factoring", + "struct": "16f" + }, + { + "byte": 142, + "variable_name": "UNCOR_SINGLES(16)", + "type": "Real*4", + "comment": "Total singles without loss correction factoring", + "struct": "16f" + }, + { + "byte": 206, + "variable_name": "TOT_AVG_COR", + "type": "Real*4", + "comment": "Mean value of loss-corrected singles", + "struct": "f" + }, + { + "byte": 210, + "variable_name": "TOT_AVG_UNCOR", + "type": "Real*4", + "comment": "Mean value of singles (not loss corrected)", + "struct": "f" + }, + { + "byte": 214, + "variable_name": "TOTAL_COIN_RATE", + "type": "Integer*4", + "comment": "Measured coincidence rate (from IPCP)", + "struct": "l" + }, + { + "byte": 218, + "variable_name": "FRAME_START_TIME", + "type": "Integer*4", + "comment": "Time offset from first frame time (in msec.)", + "struct": "l" + }, + { + "byte": 222, + "variable_name": "FRAME_DURATION", + "type": "Integer*4", + "comment": "Total duration of current frame (in msec.)", + "struct": "l" + }, + { + "byte": 226, + "variable_name": "DEADTIME_CORRECTION_FACTOR", + "type": "Real*4", + "comment": "Dead-time correction factor applied to the sinogram", + "struct": "f" + }, + { + "byte": 230, + "variable_name": "PHYSICAL_PLANES(8)", + "type": "Integer*2", + "comment": "Physical planes that make up this logical plane", + "struct": "8H" + }, + { + "byte": 246, + "variable_name": "FILL(83)", + "type": "Integer*2", + "comment": "CTI Reserved space (166 bytes)", + "struct": "83H" + }, + { + "byte": 412, + "variable_name": "FILL(50)", + "type": "Integer*2", + "comment": "User Reserved space (100 bytes) Note: use highest bytes first", + "struct": "50H" + } + ] + }, "72": { "mainheader": [ { @@ -2798,6 +4625,7 @@ } ] }, + "73": { "mainheader": [ { diff --git a/pypet2bids/pypet2bids/golden_ecat.py b/pypet2bids/pypet2bids/golden_ecat.py index 6af0560a..160bd3e2 100644 --- a/pypet2bids/pypet2bids/golden_ecat.py +++ b/pypet2bids/pypet2bids/golden_ecat.py @@ -1,13 +1,13 @@ import numpy import dotenv -from pypet2bids.write_ecat import * +from pypet2bids.write_ecat import write_ecat from pypet2bids.read_ecat import read_ecat, ecat_header_maps import os from math import e from pathlib import Path from scipy.io import savemat -''' +""" This script exists solely to create an ecat with pixel values ranging from 0 to 32767 with a minimum number of frames. This evenly pixel spaced ecat that has a small number of frames will be used to evaluate the accuracy and precision of a number of tools that convert or otherwise manipulate ecat data by supplying a ecat image and a corresponding text @@ -33,22 +33,27 @@ Anthony Galassi - 2022 ---------------------------------------------- Copyright Open NeuroPET team -''' +""" + + def main(): script_path = Path(__file__).absolute() - ecat_validation_folder = os.path.join(script_path.parent.parent.parent, 'ecat_validation/') + ecat_validation_folder = os.path.join( + script_path.parent.parent.parent, "ecat_validation/" + ) # Path to a reference/skeleton ecat as well as output paths for saving the created ecats to are stored in # environment variables or a .env file dotenv.load_dotenv(dotenv.find_dotenv()) # collect path to golden ecat file - int_golden_ecat_path = os.environ['GOLDEN_ECAT_INTEGER'] + int_golden_ecat_path = os.environ["GOLDEN_ECAT_INTEGER"] int_golden_ecat_path_stem = Path(int_golden_ecat_path).stem # collect skeleton header and subheaders. - skeleton_main_header, skeleton_subheader, _ = read_ecat(os.environ['GOLDEN_ECAT_TEMPLATE_ECAT'], - collect_pixel_data=False) + skeleton_main_header, skeleton_subheader, _ = read_ecat( + os.environ["GOLDEN_ECAT_TEMPLATE_ECAT"], collect_pixel_data=False + ) number_of_frames = 4 one_dimension = 16 # generate known 'pixel' data e.g. 4 frames with volume = 4x4x4 @@ -60,64 +65,75 @@ def main(): image_max = 32767 spacing = round((image_max - image_min) / number_of_array_elements) - #integer_pixel_data = numpy.arange(image_min, image_max, dtype=numpy.ushort, step=spacing) + # integer_pixel_data = numpy.arange(image_min, image_max, dtype=numpy.ushort, step=spacing) integer_pixel_data = numpy.arange(image_min, image_max, dtype=">H", step=spacing) # save data for analysis in matlab matlab_struct = {} # reshape pixel data into 3-d arrays - pixels_to_collect = one_dimension ** 3 + pixels_to_collect = one_dimension**3 frames = [] temp_three_d_arrays = numpy.array_split(integer_pixel_data, number_of_frames) for i in range(number_of_frames): - frames.append(temp_three_d_arrays[i].reshape(one_dimension, one_dimension, one_dimension)) + frames.append( + temp_three_d_arrays[i].reshape(one_dimension, one_dimension, one_dimension) + ) matlab_struct[f"frame_{i + 1}_pixel_data"] = frames[i] # edit the header to suit the new file header_to_write = skeleton_main_header - header_to_write['NUM_FRAMES'] = 4 - header_to_write['ORIGINAL_FILE_NAME'] = 'GoldenECATInteger' - header_to_write['STUDY_TYPE'] = 'Golden' - header_to_write['PATIENT_ID'] = 'PerfectPatient' - header_to_write['PATIENT_NAME'] = 'Majesty' - header_to_write['FACILITY_NAME'] = 'Virtual' - header_to_write['NUM_PLANES'] = one_dimension - header_to_write['ECAT_CALIBRATION_FACTOR'] = 1.0 + header_to_write["NUM_FRAMES"] = 4 + header_to_write["ORIGINAL_FILE_NAME"] = "GoldenECATInteger" + header_to_write["STUDY_TYPE"] = "Golden" + header_to_write["PATIENT_ID"] = "PerfectPatient" + header_to_write["PATIENT_NAME"] = "Majesty" + header_to_write["FACILITY_NAME"] = "Virtual" + header_to_write["NUM_PLANES"] = one_dimension + header_to_write["ECAT_CALIBRATION_FACTOR"] = 1.0 subheaders_to_write = skeleton_subheader[0:number_of_frames] for subheader in subheaders_to_write: - subheader['X_DIMENSION'] = one_dimension # pixel data is 3-d this is 1/3 root of the 3d array. - subheader['Y_DIMENSION'] = one_dimension - subheader['Z_DIMENSION'] = one_dimension - subheader['IMAGE_MIN'] = integer_pixel_data.min() - subheader['IMAGE_MAX'] = integer_pixel_data.max() - subheader['ANNOTATION'] = 'This patient is very small.' - subheader['DATA_TYPE'] = 6 - subheader['SCALE_FACTOR'] = 1 - - matlab_struct['subheaders'] = subheaders_to_write - matlab_struct['mainheader'] = header_to_write - - write_ecat(ecat_file=int_golden_ecat_path, - mainheader_schema=ecat_header_maps['ecat_headers']['73']['mainheader'], - mainheader_values=header_to_write, - subheaders_values=subheaders_to_write, - subheader_schema=ecat_header_maps['ecat_headers']['73']['7'], - number_of_frames=number_of_frames, - pixel_x_dimension=one_dimension, - pixel_y_dimension=one_dimension, - pixel_z_dimension=one_dimension, - pixel_byte_size=2, - pixel_data=frames - ) - - savemat(os.path.join(ecat_validation_folder, (int_golden_ecat_path_stem + '.mat')) - , matlab_struct) + subheader["X_DIMENSION"] = ( + one_dimension # pixel data is 3-d this is 1/3 root of the 3d array. + ) + subheader["Y_DIMENSION"] = one_dimension + subheader["Z_DIMENSION"] = one_dimension + subheader["IMAGE_MIN"] = integer_pixel_data.min() + subheader["IMAGE_MAX"] = integer_pixel_data.max() + subheader["ANNOTATION"] = "This patient is very small." + subheader["DATA_TYPE"] = 6 + subheader["SCALE_FACTOR"] = 1 + + matlab_struct["subheaders"] = subheaders_to_write + matlab_struct["mainheader"] = header_to_write + + write_ecat( + ecat_file=int_golden_ecat_path, + mainheader_schema=ecat_header_maps["ecat_headers"]["73"]["mainheader"], + mainheader_values=header_to_write, + subheaders_values=subheaders_to_write, + subheader_schema=ecat_header_maps["ecat_headers"]["73"]["7"], + number_of_frames=number_of_frames, + pixel_x_dimension=one_dimension, + pixel_y_dimension=one_dimension, + pixel_z_dimension=one_dimension, + pixel_byte_size=2, + pixel_data=frames, + ) + + savemat( + os.path.join(ecat_validation_folder, (int_golden_ecat_path_stem + ".mat")), + matlab_struct, + ) # now read it just to make sure what was created is a real file and not error riddled. - int_golden_ecat_main_header, int_golden_ecat_subheaders, int_golden_ecat_pixel_data = read_ecat(int_golden_ecat_path) + ( + int_golden_ecat_main_header, + int_golden_ecat_subheaders, + int_golden_ecat_pixel_data, + ) = read_ecat(int_golden_ecat_path) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/pypet2bids/pypet2bids/helper_functions.py b/pypet2bids/pypet2bids/helper_functions.py index 1b5b13ec..a9e284de 100644 --- a/pypet2bids/pypet2bids/helper_functions.py +++ b/pypet2bids/pypet2bids/helper_functions.py @@ -14,12 +14,14 @@ | *Authors: Anthony Galassi* | *Copyright OpenNeuroPET team* """ + import os import gzip import re import shutil import typing import json +import hashlib import warnings import logging import dotenv @@ -40,18 +42,27 @@ parent_dir = pathlib.Path(__file__).parent.resolve() project_dir = parent_dir.parent.parent -if 'PET2BIDS' not in project_dir.parts: +if "PET2BIDS" not in project_dir.parts: project_dir = parent_dir -metadata_dir = os.path.join(project_dir, 'metadata') -pet_metadata_json = os.path.join(metadata_dir, 'PET_metadata.json') +metadata_dir = os.path.join(project_dir, "metadata") + +# check to see where the schema is at +pet_metadata_json = os.path.join(metadata_dir, "PET_metadata.json") permalink_pet_metadata_json = "https://github.com/openneuropet/PET2BIDS/blob/76d95cf65fa8a14f55a4405df3fdec705e2147cf/metadata/PET_metadata.json" -pet_reconstruction_metadata_json = os.path.join(metadata_dir, 'PET_reconstruction_methods.json') +pet_reconstruction_metadata_json = os.path.join( + metadata_dir, "PET_reconstruction_methods.json" +) # load bids schema -bids_schema_path = os.path.join(metadata_dir, 'schema.json') -schema = json.load(open(bids_schema_path, 'r')) +bids_schema_path = os.path.join(metadata_dir, "schema.json") +schema = json.load(open(bids_schema_path, "r")) +# putting these paths here as they are reused in dcm2niix4pet.py, update_json_pet_file.py, and ecat.py +module_folder = Path(__file__).parent.resolve() +python_folder = module_folder.parent +pet2bids_folder = python_folder.parent +metadata_folder = os.path.join(pet2bids_folder, "metadata") loggers = {} @@ -80,11 +91,13 @@ def logger(name): return logger -def load_pet_bids_requirements_json(pet_bids_req_json: Union[str, pathlib.Path] = pet_metadata_json) -> dict: +def load_pet_bids_requirements_json( + pet_bids_req_json: Union[str, pathlib.Path] = pet_metadata_json +) -> dict: if type(pet_bids_req_json) is str: pet_bids_req_json = pathlib.Path(pet_bids_req_json) if pet_bids_req_json.is_file(): - with open(pet_bids_req_json, 'r') as infile: + with open(pet_bids_req_json, "r") as infile: reqs = json.load(infile) return reqs else: @@ -112,18 +125,28 @@ def flatten_series(series): def collect_spreadsheets(folder_path: pathlib.Path): spreadsheet_files = [] - all_files = [folder_path / pathlib.Path(file) for file in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, file))] + all_files = [ + folder_path / pathlib.Path(file) + for file in os.listdir(folder_path) + if os.path.isfile(os.path.join(folder_path, file)) + ] for file in all_files: - if file.suffix == '.xlsx' or file.suffix == '.csv' or file.suffix == '.xls' or file.suffix == '.tsv': + if ( + file.suffix == ".xlsx" + or file.suffix == ".csv" + or file.suffix == ".xls" + or file.suffix == ".tsv" + ): spreadsheet_files.append(file) return spreadsheet_files def single_spreadsheet_reader( - path_to_spreadsheet: Union[str, pathlib.Path], - pet2bids_metadata_json: Union[str, pathlib.Path] = pet_metadata_json, - dicom_metadata={}, - **kwargs) -> dict: + path_to_spreadsheet: Union[str, pathlib.Path], + pet2bids_metadata_json: Union[str, pathlib.Path] = pet_metadata_json, + dicom_metadata={}, + **kwargs, +) -> dict: metadata = {} @@ -140,18 +163,22 @@ def single_spreadsheet_reader( pet2bids_metadata_json = pathlib.Path(pet2bids_metadata_json) if pet2bids_metadata_json.is_file(): - with open(pet_metadata_json, 'r') as infile: + with open(pet_metadata_json, "r") as infile: metadata_fields = json.load(infile) else: - raise FileNotFoundError(f"Required metadata file not found at {pet_metadata_json}, check to see if this file exists;" - f"\nelse pass path to file formatted to this {permalink_pet_metadata_json} via " - f"pet2bids_metadata_json argument in simplest_spreadsheet_reader call.") + raise FileNotFoundError( + f"Required metadata file not found at {pet_metadata_json}, check to see if this file exists;" + f"\nelse pass path to file formatted to this {permalink_pet_metadata_json} via " + f"pet2bids_metadata_json argument in simplest_spreadsheet_reader call." + ) else: - raise FileNotFoundError(f"pet2bids_metadata_json input required for function call, you provided {pet2bids_metadata_json}") + raise FileNotFoundError( + f"pet2bids_metadata_json input required for function call, you provided {pet2bids_metadata_json}" + ) spreadsheet_dataframe = open_meta_data(path_to_spreadsheet) - log = logging.getLogger('pypet2bids') + log = logging.getLogger("pypet2bids") # collect mandatory fields for field_level in metadata_fields.keys(): @@ -159,8 +186,15 @@ def single_spreadsheet_reader( series = spreadsheet_dataframe.get(field, Series(dtype=numpy.float64)) if not series.empty: metadata[field] = flatten_series(series) - elif series.empty and field_level == 'mandatory' and not dicom_metadata.get(field, None) and field not in kwargs: - log.warning(f"{field} not found in metadata spreadsheet: {path_to_spreadsheet}, {field} is required by BIDS") + elif ( + series.empty + and field_level == "mandatory" + and not dicom_metadata.get(field, None) + and field not in kwargs + ): + log.warning( + f"{field} not found in metadata spreadsheet: {path_to_spreadsheet}, {field} is required by BIDS" + ) # lastly apply any kwargs to the metadata metadata.update(**kwargs) @@ -168,23 +202,25 @@ def single_spreadsheet_reader( # more lastly, check to see if values are of the correct datatype (e.g. string, number, boolean) for field, value in metadata.items(): # check schema for field - field_schema_properties = schema['objects']['metadata'].get(field, None) + field_schema_properties = schema["objects"]["metadata"].get(field, None) if field_schema_properties: # check to see if value is of the correct type - if field_schema_properties.get('type') == 'number': + if field_schema_properties.get("type") == "number": if not is_numeric(str(value)): log.warning(f"{field} is not numeric, it's value is {value}") - if field_schema_properties.get('type') == 'boolean': + if field_schema_properties.get("type") == "boolean": if type(value) is not bool: try: - check_bool = int(value)/1 + check_bool = int(value) / 1 if check_bool == 0 or check_bool == 1: metadata[field] = bool(value) else: - log.warning(f"{field} is not boolean, it's value is {value}") + log.warning( + f"{field} is not boolean, it's value is {value}" + ) except ValueError: pass - elif field_schema_properties.get('type') == 'string': + elif field_schema_properties.get("type") == "string": if type(value) is not str: log.warning(f"{field} is not string, it's value is {value}") else: @@ -205,8 +241,8 @@ def compress(file_like_object, output_path: str = None): if file_like_object.exists() and not output_path: old_suffix = file_like_object.suffix - if '.gz' not in old_suffix: - output_path = file_like_object.with_suffix(old_suffix + '.gz') + if ".gz" not in old_suffix: + output_path = file_like_object.with_suffix(old_suffix + ".gz") else: output_path = file_like_object @@ -215,10 +251,10 @@ def compress(file_like_object, output_path: str = None): else: pass - with open(file_like_object, 'rb') as infile: + with open(file_like_object, "rb") as infile: input_data = infile.read() - output = gzip.GzipFile(output_path, 'wb') + output = gzip.GzipFile(output_path, "wb") output.write(input_data) output.close() @@ -237,14 +273,14 @@ def decompress(file_like_object, output_path: str = None): the input file and writes to that amended path :return: output_path on successful decompression """ - if not output_path and '.gz' in file_like_object: - output_path = re.sub('.gz', '', file_like_object) + if not output_path and ".gz" in file_like_object: + output_path = re.sub(".gz", "", file_like_object) compressed_file = gzip.GzipFile(file_like_object) compressed_input = compressed_file.read() compressed_file.close() - with open(output_path, 'wb') as outfile: + with open(output_path, "wb") as outfile: outfile.write(compressed_input) return output_path @@ -280,19 +316,19 @@ def get_version(): try: # if this is bundled as a package look next to this file for the pyproject.toml - toml_path = os.path.join(scripts_dir, 'pyproject.toml') - with open(toml_path, 'r') as infile: + toml_path = os.path.join(scripts_dir, "pyproject.toml") + with open(toml_path, "r") as infile: tomlfile = toml.load(infile) except FileNotFoundError: # when in development the toml file with the version is 2 directories above (e.g. where it should actually live) toml_dir = scripts_dir.parent - toml_path = os.path.join(toml_dir, 'pyproject.toml') - with open(toml_path, 'r') as infile: + toml_path = os.path.join(toml_dir, "pyproject.toml") + with open(toml_path, "r") as infile: tomlfile = toml.load(infile) - attrs = tomlfile.get('tool', {}) - poetry = attrs.get('poetry', {}) - version = poetry.get('version', '') + attrs = tomlfile.get("tool", {}) + poetry = attrs.get("poetry", {}) + version = poetry.get("version", "") return version @@ -309,7 +345,7 @@ def is_numeric(check_this_object: str) -> bool: class ParseKwargs(argparse.Action): """ - Class that is used to extract key pair arguments passed to an argparse.ArgumentParser objet via the command line. + Class that is used to extract key pair arguments passed to an argparse.ArgumentParser object via the command line. Accepts key value pairs in the form of 'key=value' and then passes these arguments onto the arg parser as kwargs. This class is used during the construction of the ArgumentParser class via the add_argument method. e.g.:\n `ArgumentParser.add_argument('--kwargs', '-k', nargs='*', action=helper_functions.ParseKwargs, default={})` @@ -319,7 +355,7 @@ def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, dict()) for value in values: try: - key, value = value.split('=') + key, value = value.split("=") getattr(namespace, self.dest)[key] = very_tolerant_literal_eval(value) except ValueError: raise Exception(f"Unable to unpack {value}") @@ -337,15 +373,15 @@ def very_tolerant_literal_eval(value): try: value = ast.literal_eval(value) except (SyntaxError, ValueError): - if str(value).lower() == 'none': + if str(value).lower() == "none": value = None - elif str(value).lower() == 'true': + elif str(value).lower() == "true": value = True - elif str(value).lower() == 'false': + elif str(value).lower() == "false": value = False - elif str(value)[0] == '[' and str(value)[-1] == ']': - array_contents = str(value).replace('[', '').replace(']', '') - array_contents = array_contents.split(',') + elif str(value)[0] == "[" and str(value)[-1] == "]": + array_contents = str(value).replace("[", "").replace("]", "") + array_contents = array_contents.split(",") array_contents = [str.strip(content) for content in array_contents] # evaluate array contents one by one value = [very_tolerant_literal_eval(v) for v in array_contents] @@ -354,7 +390,9 @@ def very_tolerant_literal_eval(value): return value -def open_meta_data(metadata_path: Union[str, pathlib.Path], separator=None) -> pandas.DataFrame: +def open_meta_data( + metadata_path: Union[str, pathlib.Path], separator=None +) -> pandas.DataFrame: """ Opens a text metadata file with the pandas method most appropriate for doing so based on the metadata file's extension. @@ -364,7 +402,7 @@ def open_meta_data(metadata_path: Union[str, pathlib.Path], separator=None) -> p :type separator: str :return: a pandas dataframe representation of the spreadsheet/metadatafile """ - log = logger('pypet2bids') + log = logger("pypet2bids") if type(metadata_path) is str: metadata_path = pathlib.Path(metadata_path) @@ -377,32 +415,29 @@ def open_meta_data(metadata_path: Union[str, pathlib.Path], separator=None) -> p # collect suffix from metadata and use the approriate pandas method to read the data extension = metadata_path.suffix - methods = { - 'excel': read_excel, - 'csv': read_csv, - 'txt': read_csv - } + methods = {"excel": read_excel, "csv": read_csv, "tsv": read_csv, "txt": read_csv} - if 'xls' in extension or 'bld' in extension: - proper_method = 'excel' + if "xls" in extension or "bld" in extension: + proper_method = "excel" else: - proper_method = extension.replace('.', '') - with open(metadata_path, 'r') as infile: + proper_method = extension.replace(".", "") + with open(metadata_path, "r") as infile: first_line = infile.read() # check for separators in line - separators = ['\t', ','] + separators = ["\t", ","] for sep in separators: if sep in first_line: separators_present.append(sep) try: - warnings.filterwarnings('ignore', message='ParserWarning: Falling*') + warnings.filterwarnings("ignore", message="ParserWarning: Falling*") use_me_to_read = methods.get(proper_method, None) - if proper_method != 'excel': - if '\t' in separators_present: - separator = '\t' + + if proper_method != "excel": + if "\t" in separators_present: + separator = "\t" else: - separator = ',' + separator = "," metadata_dataframe = use_me_to_read(metadata_path, sep=separator) else: metadata_dataframe = use_me_to_read(metadata_path, sheet_name=None) @@ -412,22 +447,28 @@ def open_meta_data(metadata_path: Union[str, pathlib.Path], separator=None) -> p if len(multiple_sheets) >= 1: for index, sheet_name in enumerate(multiple_sheets): for column in metadata_dataframe[sheet_name].columns: - metadata_dataframe[first_sheet][column] = metadata_dataframe[sheet_name][column] + metadata_dataframe[first_sheet][column] = metadata_dataframe[ + sheet_name + ][column] metadata_dataframe = metadata_dataframe[first_sheet] except (IOError, ValueError) as err: try: - metadata_dataframe = pandas.read_csv(metadata_path, sep=separator, engine='python') + metadata_dataframe = pandas.read_csv( + metadata_path, sep=separator, engine="python" + ) except IOError: - log.error(f"Tried falling back to reading {metadata_path} with pandas.read_csv, still unable to parse") + log.error( + f"Tried falling back to reading {metadata_path} with pandas.read_csv, still unable to parse" + ) raise err(f"Problem opening {metadata_path}") return metadata_dataframe def translate_metadata(metadata_path, metadata_translation_script_path, **kwargs): - log = logger('pypet2bids') + log = logger("pypet2bids") # load metadata metadata_dataframe = open_meta_data(metadata_path) @@ -435,8 +476,9 @@ def translate_metadata(metadata_path, metadata_translation_script_path, **kwargs try: # this is where the goofiness happens, we allow the user to create their own custom script to manipulate # data from their particular spreadsheet wherever that file is located. - spec = importlib.util.spec_from_file_location("metadata_translation_script_path", - metadata_translation_script_path) + spec = importlib.util.spec_from_file_location( + "metadata_translation_script_path", metadata_translation_script_path + ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # note the translation must have a method named translate metadata in order to work @@ -467,9 +509,16 @@ def import_and_write_out_module(module: str, destination: str): return os.path.join(destination, os.path.basename(path_to_module)) -def write_out_module(module: str = 'pypet2bids.metadata_spreadsheet_example_reader'): - parser = argparse.ArgumentParser() - parser.add_argument('template_path', type=str, help="Path to write out template for a translation script.") +def write_out_module(module: str = "pypet2bids.metadata_spreadsheet_example_reader"): + parser = argparse.ArgumentParser( + description="[DEPRECATED!!] Write out a template for a python script used for " + "bespoke metadata." + ) + parser.add_argument( + "template_path", + type=str, + help="Path to write out template for a translation script.", + ) args = parser.parse_args() import_and_write_out_module(module=module, destination=args.template_path) @@ -486,12 +535,12 @@ def expand_path(path_like: str) -> str: :rtype: string """ if path_like: - if path_like[0] == '~': + if path_like[0] == "~": return str(os.path.expanduser(path_like)) else: - return (os.path.abspath(path_like)) + return os.path.abspath(path_like) else: - return '' + return "" def collect_bids_part(bids_part: str, path_like: Union[str, pathlib.Path]) -> str: @@ -510,10 +559,10 @@ def collect_bids_part(bids_part: str, path_like: Union[str, pathlib.Path]) -> st :return: the collected bids part :rtype: string """ - log = logger('pypet2bids') + log = logger("pypet2bids") # get os of system - if os.name == 'posix': + if os.name == "posix": not_windows = True else: not_windows = False @@ -524,14 +573,16 @@ def collect_bids_part(bids_part: str, path_like: Union[str, pathlib.Path]) -> st # this shouldn't happen, but we check if someone passed a windows path to a posix machine # should we check for the inverse? No, not until someone complains about this loudly enough. for part in parts: - if '\\' in part and not_windows: + if "\\" in part and not_windows: # explicitly use windows path splitting parts = pathlib.PureWindowsPath(path_like).parts - log.warning(f"Detected \\ in BIDS like path {path_like}, but os is {os.name}, doing best to parse.") + log.warning( + f"Detected \\ in BIDS like path {path_like}, but os is {os.name}, doing best to parse." + ) break # create search string - search_string = bids_part + '-(.*)' + search_string = bids_part + "-(.*)" # collect bids_part for part in parts: found_part = re.search(search_string, part) @@ -539,26 +590,27 @@ def collect_bids_part(bids_part: str, path_like: Union[str, pathlib.Path]) -> st collected_part = found_part[0] break else: - collected_part = '' + collected_part = "" - if '_' in collected_part: - parts = collected_part.split('_') + if "_" in collected_part: + parts = collected_part.split("_") for part in parts: found_part = re.search(search_string, part) if found_part: collected_part = found_part[0] break else: - collected_part = '' + collected_part = "" return collected_part def get_coordinates_containing( - containing: typing.Union[str, int, float], - dataframe: pandas.DataFrame, - exact: bool = False, - single=False) -> typing.Union[list, tuple]: + containing: typing.Union[str, int, float], + dataframe: pandas.DataFrame, + exact: bool = False, + single=False, +) -> typing.Union[list, tuple]: """ Collects the co-ordinates (row and column) containing an input value, that value could be a string, integer, or float. When searching for integers or floats it is most likely best to use set the exact argument to True; the same @@ -579,9 +631,9 @@ def get_coordinates_containing( If you are confident in your input data you would most likely call this method this way: >>> coordinates = get_coordinates_containing( - >>> 'sub-NDAR1', - >>> pandas.DataFrame({'subject_id': ['sub-NDAR1', 'sub-NDAR2'], 'some_values': [1, 2]}), - >>> single=True) + >>> 'sub-NDAR1', + >>> pandas.DataFrame({'subject_id': ['sub-NDAR1', 'sub-NDAR2'], 'some_values': [1, 2]}), + >>> single=True) >>> coordinates >>> (0, 1) @@ -589,14 +641,15 @@ def get_coordinates_containing( :type containing: string, integer, or float :param dataframe: A pandas dataframe read in from a spreadsheet :type dataframe: pandas.datafarame - :param exact: Boolean proscribing an exact match to containing; default is to locate a string or value that holds the - input containing + :param exact: Boolean proscribing an exact match to containing; default is to locate a string or value that holds + the input containing :type exact: bool :param single: return only the first co-ordinate, use only if the string/contains your searching for is unique and - you have high confidence in your data + you have high confidence in your data :type single: bool :return: A co-ordinate in the form of (row, column) or a list containing a set of co-ordinates [(row, column), ...] :rtype: tuple or list of tuples + """ percent_tolerance = 0.001 @@ -607,7 +660,11 @@ def get_coordinates_containing( if exact: if value == containing: coordinates.append((row_index, column_index)) - elif not exact and not isinstance(value, str) and not isinstance(containing, str): + elif ( + not exact + and not isinstance(value, str) + and not isinstance(containing, str) + ): if numpy.isclose(value, containing, rtol=percent_tolerance): coordinates.append((row_index, column_index)) elif not exact and type(value) is str: @@ -619,7 +676,9 @@ def get_coordinates_containing( return coordinates -def transform_row_to_dict(row: typing.Union[int, str, pandas.Series], dataframe: pandas.DataFrame = None) -> dict: +def transform_row_to_dict( + row: typing.Union[int, str, pandas.Series], dataframe: pandas.DataFrame = None +) -> dict: """ Parses a row of a dataframe (or a series from a dataframe) into a dictionary, special care is taken to transform array like data contained in a single cell to a list of numbers or strings. @@ -663,20 +722,40 @@ def get_recon_method(ReconstructionMethodString: str) -> dict: ReconMethodParameterLabels, and ReconMethodParameterValues """ - contents = ReconstructionMethodString.replace(' ', '') + contents = ReconstructionMethodString.replace(" ", "") subsets = None iterations = None # determine order of recon iterations and subsets, this is not a surefire way to determine this... iter_sub_combos = { - 'iter_first': [r'\d\di\ds', r'\d\di\d\ds', r'\di\ds', r'\di\d\ds', - r'i\d\ds\d', r'i\d\ds\d\d', r'i\ds\d', r'i\ds\d\d'], - 'sub_first': [r'\d\ds\di', r'\d\ds\d\di', r'\ds\di', r'\ds\d\di', - r's\d\di\d', r's\d\di\d\d', r's\di\d', r's\di\d\d'], + "iter_first": [ + r"\d\di\ds", + r"\d\di\d\ds", + r"\di\ds", + r"\di\d\ds", + r"i\d\ds\d", + r"i\d\ds\d\d", + r"i\ds\d", + r"i\ds\d\d", + ], + "sub_first": [ + r"\d\ds\di", + r"\d\ds\d\di", + r"\ds\di", + r"\ds\d\di", + r"s\d\di\d", + r"s\d\di\d\d", + r"s\di\d", + r"s\di\d\d", + ], } - iter_sub_combos['iter_first'] = [re.compile(regex) for regex in iter_sub_combos['iter_first']] - iter_sub_combos['sub_first'] = [re.compile(regex) for regex in iter_sub_combos['sub_first']] + iter_sub_combos["iter_first"] = [ + re.compile(regex) for regex in iter_sub_combos["iter_first"] + ] + iter_sub_combos["sub_first"] = [ + re.compile(regex) for regex in iter_sub_combos["sub_first"] + ] order = None possible_iter_sub_strings = [] iteration_subset_string = None @@ -699,12 +778,12 @@ def get_recon_method(ReconstructionMethodString: str) -> dict: # after we've captured the subsets and iterations we next need to separate them out from each other if iteration_subset_string and order: # remove all chars replace with spaces - just_digits = re.sub(r'[a-zA-Z]', " ", iteration_subset_string) + just_digits = re.sub(r"[a-zA-Z]", " ", iteration_subset_string) just_digits = just_digits.strip() # split up subsets and iterations just_digits = just_digits.split(" ") # assign digits to either subsets or iterations based on order information obtained earlier - if order == 'iter_first' and len(just_digits) == 2: + if order == "iter_first" and len(just_digits) == 2: iterations = int(just_digits[0]) subsets = int(just_digits[1]) elif len(just_digits) == 2: @@ -721,34 +800,38 @@ def get_recon_method(ReconstructionMethodString: str) -> dict: ReconMethodName = contents # cleaning up weird chars at end or start of name - ReconMethodName = re.sub(r'[^a-zA-Z0-9]$', "", ReconMethodName) - ReconMethodName = re.sub(r'^[^a-zA-Z0-9]', "", ReconMethodName) + ReconMethodName = re.sub(r"[^a-zA-Z0-9]$", "", ReconMethodName) + ReconMethodName = re.sub(r"^[^a-zA-Z0-9]", "", ReconMethodName) expanded_name = "" # get the dimension as it's often somewhere in the name dimension = "" - search_criteria = r'[1-4][D-d]' + search_criteria = r"[1-4][D-d]" if re.search(search_criteria, ReconMethodName): dimension = re.search(search_criteria, ReconMethodName)[0] # doing some more manipulation of the recon method name to expand it from not so helpful acronyms - possible_names = load_pet_bids_requirements_json(pet_reconstruction_metadata_json)['reconstruction_names'] + possible_names = load_pet_bids_requirements_json(pet_reconstruction_metadata_json)[ + "reconstruction_names" + ] # we want to sort the possible names by longest first that we don't break up an acronym prematurely - sorted_df = pandas.DataFrame(possible_names).sort_values(by='value', key=lambda x: x.str.len(), ascending=False) + sorted_df = pandas.DataFrame(possible_names).sort_values( + by="value", key=lambda x: x.str.len(), ascending=False + ) possible_names = [] for row in sorted_df.iterrows(): - possible_names.append({'value': row[1]['value'], 'name': row[1]['name']}) + possible_names.append({"value": row[1]["value"], "name": row[1]["name"]}) for name in possible_names: - if name['value'] in ReconMethodName: - expanded_name += name['name'] + " " - ReconMethodName = re.sub(name['value'], "", ReconMethodName) + if name["value"] in ReconMethodName: + expanded_name += name["name"] + " " + ReconMethodName = re.sub(name["value"], "", ReconMethodName) if expanded_name != "": - ReconMethodName = dimension + " "*len(dimension) + expanded_name.rstrip() + ReconMethodName = dimension + " " * len(dimension) + expanded_name.rstrip() ReconMethodName = " ".join(ReconMethodName.split()) if ReconMethodName in ["Filtered Back Projection", "3D Reprojection"]: @@ -767,16 +850,16 @@ def get_recon_method(ReconstructionMethodString: str) -> dict: "ReconMethodParameterValues": ReconMethodParameterValues, } - if None in reconstruction_dict['ReconMethodParameterValues']: - reconstruction_dict.pop('ReconMethodParameterValues') - reconstruction_dict.pop('ReconMethodParameterUnits') - for i in range(len(reconstruction_dict['ReconMethodParameterLabels'])): - reconstruction_dict['ReconMethodParameterLabels'][i] = "none" + if None in reconstruction_dict["ReconMethodParameterValues"]: + reconstruction_dict.pop("ReconMethodParameterValues") + reconstruction_dict.pop("ReconMethodParameterUnits") + for i in range(len(reconstruction_dict["ReconMethodParameterLabels"])): + reconstruction_dict["ReconMethodParameterLabels"][i] = "none" return reconstruction_dict -def modify_config_file(var: str, value: Union[pathlib.Path, str]): +def modify_config_file(var: str, value: Union[pathlib.Path, str], config_path=None): """ Given a variable name and a value updates the config file with those inputs. Namely used (on Windows, but not limited to) to point to a dcm2niix executable (dcm2niix.exe) @@ -785,32 +868,40 @@ def modify_config_file(var: str, value: Union[pathlib.Path, str]): :type var: str :param value: variable value, most often a path to another file :type value: Union[pathlib.Path, str] + :param config_path: path to the config file, if not provided this function will look for a file at the user's home + :type config_path: Union[pathlib.Path, str] :return: None :rtype: None """ # load dcm2niix file - config_file = pathlib.Path.home() - config_file = config_file / ".pet2bidsconfig" + if not config_path: + config_file = pathlib.Path.home() / ".pet2bidsconfig" + else: + config_file = pathlib.Path(config_path) + if config_file.exists(): # open the file and read in all the lines - temp_file = pathlib.Path.home() / '.pet2bidsconfig.temp' - with open(config_file, 'r') as infile, open(temp_file, 'w') as outfile: + temp_file = config_file.with_suffix(".temp") + with open(config_file, "r") as infile, open(temp_file, "w") as outfile: updated_file = False for line in infile: - if var + '=' in line: + if var + "=" in line: outfile.write(f"{var}={value}\n") updated_file = True else: outfile.write(line) if not updated_file: outfile.write(f"{var}={value}\n") - if system().lower() == 'windows': + if system().lower() == "windows": config_file.unlink(missing_ok=True) temp_file.replace(config_file) else: # create the file - with open(config_file, 'w') as outfile: - outfile.write(f'{var}={value}\n') + try: + with open(config_file, "w") as outfile: + outfile.write(f"{var}={value}\n") + except FileNotFoundError as err: + logging.error(f"Unable to write to {config_file}\n{err}") def check_units(entity_key: str, entity_value: str, accepted_units: Union[list, str]): @@ -837,7 +928,9 @@ def check_units(entity_key: str, entity_value: str, accepted_units: Union[list, if allowed: pass else: - if type(accepted_units) is str or (type(accepted_units) is list and len(accepted_units) == 1): + if type(accepted_units) is str or ( + type(accepted_units) is list and len(accepted_units) == 1 + ): warning = f"{entity_key} must have units as {accepted_units}, ignoring given units {entity_value}" elif type(accepted_units) is list and len(accepted_units) > 1: warning = f"{entity_key} must have units as on of {accepted_units}, ignoring given units {entity_value}" @@ -847,7 +940,9 @@ def check_units(entity_key: str, entity_value: str, accepted_units: Union[list, return allowed -def ad_hoc_checks(metadata: dict, modify_input=False, items_that_should_be_checked=None): +def ad_hoc_checks( + metadata: dict, modify_input=False, items_that_should_be_checked=None +): """ Check to run on PET BIDS metadata to evaluate whether input is acceptable or not, this function will most likely be refactored to use the schema instead of relying on hardcoded checks as listed in items_that_should_be_checked @@ -879,17 +974,19 @@ def ad_hoc_checks(metadata: dict, modify_input=False, items_that_should_be_check check_input_entity = metadata.get(entity, None) if check_input_entity: # this will raise a warning if the units aren't acceptable - units_are_good = check_units(entity_key=entity, entity_value=check_input_entity, accepted_units=units) + units_are_good = check_units( + entity_key=entity, entity_value=check_input_entity, accepted_units=units + ) # this will remove an entity from metadata form dictionary if it's not good if modify_input and not units_are_good: metadata.pop(entity) return metadata - + def sanitize_bad_path(bad_path: Union[str, pathlib.Path]) -> Union[str, pathlib.Path]: - if ' ' in str(bad_path): + if " " in str(bad_path): return f'"{bad_path}"'.rstrip().strip() else: return bad_path @@ -904,11 +1001,13 @@ def drop_row(dataframe: pandas.DataFrame, index: int): def replace_nones(dictionary): json_string = json.dumps(dictionary) # sub nulls - json_fixed = re.sub('null', '"none"', json_string) + json_fixed = re.sub("null", '"none"', json_string) return json.loads(json_fixed) -def check_pet2bids_config(variable: str = 'DCM2NIIX_PATH'): +def check_pet2bids_config( + variable: str = "DCM2NIIX_PATH", config_path=Path.home() / ".pet2bidsconfig" +): """ Checks the config file at users home /.pet2bidsconfig for the variable passed in, defaults to checking for DCM2NIIX_PATH. However, we can use it for anything we like, @@ -917,38 +1016,69 @@ def check_pet2bids_config(variable: str = 'DCM2NIIX_PATH'): :param variable: a string variable name to check for in the config file :type variable: string + :param config_path: path to the config file, defaults to $HOME/.pet2bidsconfig + :type config_path: string or pathlib.Path :return: the value of the variable if it exists in the config file :rtype: str """ - log = logger('pypet2bids') + log = logger("pypet2bids") # check to see if path to dcm2niix is in .env file dcm2niix_path = None - home_dir = Path.home() - pypet2bids_config = home_dir / '.pet2bidsconfig' + variable_value = None + pypet2bids_config = Path(config_path) if pypet2bids_config.exists(): dotenv.load_dotenv(pypet2bids_config) variable_value = os.getenv(variable) - if variable == 'DCM2NIIX_PATH': - if variable_value: - # check dcm2niix path exists - dcm2niix_path = Path(variable_value) - check = subprocess.run( - f"{dcm2niix_path} -h", - shell=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL - ) - if check.returncode == 0: - return dcm2niix_path - else: - log.error(f"Unable to locate dcm2niix executable at {dcm2niix_path.__str__()}") - return None - if variable != 'DCM2NIIX_PATH': - return variable_value + # else we check our environment variables for the variable + elif os.getenv(variable) and not pypet2bids_config.exists(): + variable_value = os.getenv(variable) + log.warning( + f"Found {variable} in environment variables as {variable_value}, but no .pet2bidsconfig file found at {pypet2bids_config}" + ) + if variable == "DCM2NIIX_PATH": + # check to see if dcm2niix is on the path at all + check_on_path = subprocess.run( + "dcm2niix -h", + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + if variable_value: + # check dcm2niix path exists + dcm2niix_path = Path(variable_value) + check = subprocess.run( + f"{dcm2niix_path} -h", + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if check.returncode == 0: + return dcm2niix_path + elif ( + not variable_value + and pypet2bids_config.exists() + and check_on_path.returncode == 0 + ): + # do nothing + # log.info( + # f"DCM2NIIX_PATH not found in {pypet2bids_config}, but dcm2niix is on $PATH." + # ) + return None + else: + log.error( + f"Unable to locate dcm2niix executable at {dcm2niix_path.__str__()}" + f" Set DCM2NIIX_PATH variable in {pypet2bids_config} or export DCM2NIIX_PATH=/path/to/dcm2niix into your environment variables." + ) + return None + if variable != "DCM2NIIX_PATH": + return variable_value - else: - log.warning(f"Config file not found at {pypet2bids_config}, .pet2bidsconfig file must exist and " - f"have variable: {variable} and {variable} must be set.") + if not variable_value and not pypet2bids_config.exists(): + log.warning( + f"Config file not found at {pypet2bids_config}, .pet2bidsconfig file must exist and " + f"have variable: {variable} and {variable} must be set." + ) class CustomFormatter(logging.Formatter): @@ -962,17 +1092,60 @@ class CustomFormatter(logging.Formatter): red = "\x1b[31;20m" bold_red = "\x1b[31;1m" reset = "\x1b[0m" - format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)" + format = ( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)" + ) FORMATS = { logging.DEBUG: grey + format + reset, logging.INFO: grey + format + reset, logging.WARNING: yellow + format + reset, logging.ERROR: red + format + reset, - logging.CRITICAL: bold_red + format + reset + logging.CRITICAL: bold_red + format + reset, } def format(self, record): log_fmt = self.FORMATS.get(record.levelno) formatter = logging.Formatter(log_fmt) return formatter.format(record) + + +def hash_fields(**fields): + hash_return_string = "" + hash_string = "" + if fields.get("ProtocolName", None): + hash_return_string += f"{fields.get('ProtocolName')}_" + keys_we_want = ["ses", "rec", "trc"] + for key, value in fields.items(): + # sanitize values + regex = r"[^a-zA-Z0-9]" + value = re.sub(regex, "", str(value)) + hash_string += f"{value}_" + if key in keys_we_want: + hash_return_string += f"{value}_" + + hash_hex = hashlib.md5(hash_string.encode("utf-8")).hexdigest() + + return f"{hash_return_string}{hash_hex}" + + +def first_middle_last_frames_to_text( + four_d_array_like_object, output_folder, step_name="_step_name_" +): + frames = [ + 0, + four_d_array_like_object.shape[-1] // 2, + four_d_array_like_object.shape[-1] - 1, + ] + frames_to_record = [] + for f in frames: + frames_to_record.append(four_d_array_like_object[:, :, :, f]) + + # now collect a single 2d slice from the "middle" of the 3d frames in frames_to_record + for index, frame in enumerate(frames_to_record): + numpy.savetxt( + output_folder / f"{step_name}_frame_{frames[index]}.tsv", + frames_to_record[index][:, :, frames_to_record[index].shape[2] // 2], + delimiter="\t", + fmt="%s", + ) diff --git a/pypet2bids/pypet2bids/is_pet.py b/pypet2bids/pypet2bids/is_pet.py index 1ca9e02c..08dbfb46 100644 --- a/pypet2bids/pypet2bids/is_pet.py +++ b/pypet2bids/pypet2bids/is_pet.py @@ -27,20 +27,26 @@ def spread_sheet_check_for_pet(sourcefile: Union[str, Path], **kwargs): # load BIDS PET requirements try: - with open(helper_functions.pet_metadata_json, 'r') as pet_field_requirements_json: + with open( + helper_functions.pet_metadata_json, "r" + ) as pet_field_requirements_json: pet_field_requirements = json.load(pet_field_requirements_json) except (FileNotFoundError, json.JSONDecodeError) as error: - print(f"Unable to load list of required, recommended, and optional PET BIDS fields from" - f" {helper_functions.pet_metadata_json}, will not be able to determine if sourcefile contains" - f" PET BIDS specific metadata") + print( + f"Unable to load list of required, recommended, and optional PET BIDS fields from" + f" {helper_functions.pet_metadata_json}, will not be able to determine if sourcefile contains" + f" PET BIDS specific metadata" + ) pet_field_requirements = {} - mandatory_fields = pet_field_requirements.get('mandatory', []) - recommended_fields = pet_field_requirements.get('recommended', []) - optional_fields = pet_field_requirements.get('optional', []) - blood_recording_fields = pet_field_requirements.get('blood_recording_fields', []) + mandatory_fields = pet_field_requirements.get("mandatory", []) + recommended_fields = pet_field_requirements.get("recommended", []) + optional_fields = pet_field_requirements.get("optional", []) + blood_recording_fields = pet_field_requirements.get("blood_recording_fields", []) - intersection = set(mandatory_fields + recommended_fields + optional_fields + blood_recording_fields) & set(data.keys()) + intersection = set( + mandatory_fields + recommended_fields + optional_fields + blood_recording_fields + ) & set(data.keys()) if len(intersection) > 0: return True @@ -58,12 +64,15 @@ def read_files_in_parallel(file_paths: list, function, n_jobs=-2, **kwargs): :return: list of results """ # TODO replace dependency on joblib with threading module - results = Parallel(n_jobs=n_jobs)(delayed(function)(file_path, **kwargs) for file_path in file_paths) + results = Parallel(n_jobs=n_jobs)( + delayed(function)(file_path, **kwargs) for file_path in file_paths + ) return results class DummyFile(object): - def write(self, x): pass + def write(self, x): + pass @contextlib.contextmanager @@ -98,35 +107,44 @@ def pet_file(file_path: Path, return_only_path=False) -> Union[bool, str]: if not file_path.exists(): raise FileNotFoundError(file_path) - file_type = '' + file_type = "" # get suffix of file suffix = file_path.suffix + + # if there are suffixes, join them together + if len(file_path.suffixes) > 1: + suffix = "".join(file_path.suffixes) + # suppress all stdout from other functions with nostdout(): - if not file_type and (suffix.lower() in ['.dcm', '.ima', '.img', ''] or 'mr' in str(file_path.name).lower() or bool(re.search(r"\d", suffix.lower()))): + if not file_type and ( + suffix.lower() in [".dcm", ".ima", ".img", ""] + or "mr" in str(file_path.name).lower() + or bool(re.search(r"\d", suffix.lower())) + ): try: read_file = pydicom.dcmread(file_path) - if read_file.Modality == 'PT': - file_type = 'DICOM' + if read_file.Modality == "PT": + file_type = "DICOM" else: # do nothing, we only want dicoms with the correct modality pass - except pydicom.errors.InvalidDicomError: + except (pydicom.errors.InvalidDicomError, AttributeError): pass - if not file_type and suffix.lower() in ['.v', '.gz']: + if not file_type and suffix.lower() in [".v", ".v.gz"]: try: read_file = ecat.Ecat(str(file_path)) - file_type = 'ECAT' + file_type = "ECAT" except nibabel.filebasedimages.ImageFileError: pass - if not file_type and suffix.lower() in ['.xlsx', '.tsv', '.csv', '.xls']: + if not file_type and suffix.lower() in [".xlsx", ".tsv", ".csv", ".xls"]: try: read_file = spread_sheet_check_for_pet(file_path) if read_file: # if it looks like a pet file - file_type = 'SPREADSHEET' + file_type = "SPREADSHEET" except (IOError, ValueError): pass @@ -142,7 +160,7 @@ def pet_file(file_path: Path, return_only_path=False) -> Union[bool, str]: return False, file_type -def pet_folder(folder_path: Path) -> Union[str, list, bool]: +def pet_folder(folder_path: Path, skim=False, njobs=2) -> Union[str, list, bool]: if not folder_path.exists(): raise FileNotFoundError(folder_path) if not folder_path.is_dir(): @@ -152,10 +170,43 @@ def pet_folder(folder_path: Path) -> Union[str, list, bool]: # collect list of all files for root, folders, files in os.walk(folder_path): for f in files: - all_files.append(Path(os.path.join(root, f))) + f_path = Path(os.path.join(root, f)) + if f_path.exists(): + all_files.append(Path(os.path.join(root, f))) + + # we aren't going to want to inspect every single file, instead we're going skim through the list of files + # by first selecting only the first file of each folder that has a given suffix (especially for dicom files) + if skim: + from pprint import pprint + + # collect all the dicom images + dicoms, spreadsheets, ecats = {}, {}, {} + for f in all_files: + if f.suffix.lower() in [".dcm", ".ima", ".img", ""]: + parent = dicoms.get(str(f.parent), {str(f.parent): {f.suffix: f}}) + dicoms[str(f.parent)] = parent + if f.suffix.lower() in [".xlsx", ".tsv", ".csv", ".xls"]: + parent = spreadsheets.get(str(f.parent), {str(f.parent): {f.suffix: f}}) + spreadsheets[str(f.parent)] = parent + if f.suffix.lower() in [".v"] or f.suffixes == [".v", ".gz"]: + parent = ecats.get( + str(f.parent), {str(f.parent): {"".join(f.suffixes): f}} + ) + ecats[str(f.parent)] = parent + # now flatten all the dictionaries to only include the file parts + smaller_list = [] + for file_dict in [dicoms, spreadsheets, ecats]: + for k, v in file_dict.items(): + for k2, v2 in v.items(): + for k3, v3 in v2.items(): + smaller_list.append(v3) + + all_files = smaller_list # check if any files are pet files - files = read_files_in_parallel(all_files, pet_file, n_jobs=1, return_only_path=True) + files = read_files_in_parallel( + all_files, pet_file, n_jobs=njobs, return_only_path=True + ) files = [Path(f) for f in files if f is not None] # check through list of pet files and statuses for True values in parallel folders = set([f.parent for f in files]) @@ -164,13 +215,48 @@ def pet_folder(folder_path: Path) -> Union[str, list, bool]: def main(): + """ + This command line utility exists almost entirely for ezBIDS. It's use there is to ensure that dcm2niix is not run + on folders containing PET images. Instead, the PET images are converted using the dcm2niix4pet from this library. + :return: Path to PET folder or file + :rtype: str + """ - parser = argparse.ArgumentParser() - parser.add_argument('filepath', type=Path, help="File path to check whether file is PET image or bloodfile. " - "If a folder is given, all files in the folder will be checked and " - "any folders containing PET files will be returned.") - parser.add_argument('-p', '--path-only', action='store_true', default=False, - help="Omit type of pet file; only return file path if file is PET file") + parser = argparse.ArgumentParser( + description="Check if a file is a PET image or bloodfile. If a folder is given, " + "all files in the folder will be checked and any folders " + "containing PET files will be returned." + ) + parser.add_argument( + "filepath", + type=Path, + help="File path to check whether file is PET image or bloodfile. " + "If a folder is given, all files in the folder will be checked and " + "any folders containing PET files will be returned.", + ) + parser.add_argument( + "-p", + "--path-only", + action="store_true", + default=False, + help="Omit type of pet file; only return file path if file is PET file", + ) + parser.add_argument( + "-s", + "--skim", + action="store_true", + default=False, + help="Only check files that " + "are suspected to be PET files. Defaults to checking every file found in a folder." + "When selected checks only a single file in folder ending in an extension that may be a PET FILE.", + ) + parser.add_argument( + "-n", + "--njobs", + type=int, + default=2, + help="Number of jobs to run in parallel when examining folders, defaults to 2.", + ) args = parser.parse_args() if args.filepath.is_file(): @@ -184,7 +270,9 @@ def main(): sys.exit(1) elif args.filepath.is_dir(): - pet_folders = pet_folder(args.filepath.resolve()) + pet_folders = pet_folder( + args.filepath.resolve(), skim=args.skim, njobs=args.njobs + ) if len(pet_folders) > 0: for f in pet_folders: print(f"{f}") @@ -194,5 +282,5 @@ def main(): sys.exit(1) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/pypet2bids/pypet2bids/metadata_spreadsheet_example_reader.py b/pypet2bids/pypet2bids/metadata_spreadsheet_example_reader.py index d298c355..30799283 100644 --- a/pypet2bids/pypet2bids/metadata_spreadsheet_example_reader.py +++ b/pypet2bids/pypet2bids/metadata_spreadsheet_example_reader.py @@ -1,24 +1,21 @@ -import pandas +from os.path import join import warnings +import pandas import pathlib -from typing import Union -from os.path import join -import json + try: - from helper_functions import * + import helper_functions except ModuleNotFoundError: - from pypet2bids.helper_functions import * + import pypet2bids.helper_functions as helper_functions -#from pypet2bids.helper_functions import open_meta_data, flatten_series parent_dir = pathlib.Path(__file__).parent.resolve() project_dir = parent_dir.parent -metadata_dir = join(project_dir, 'metadata') -pet_metadata_json = join(metadata_dir, 'PET_metadata.json') +metadata_dir = join(project_dir, "metadata") +pet_metadata_json = join(metadata_dir, "PET_metadata.json") permalink_pet_metadata_json = "https://github.com/openneuropet/PET2BIDS/blob/76d95cf65fa8a14f55a4405df3fdec705e2147cf/metadata/PET_metadata.json" """ -This function does? :format: :param: @@ -29,6 +26,7 @@ Copyright Open NeuroPET team """ + def translate_metadata(metadata_dataframe, image_path=NotImplemented): """ This method exists as an example/template for an individual to customize and use on their own PET metadata contained @@ -46,40 +44,40 @@ def translate_metadata(metadata_dataframe, image_path=NotImplemented): :return: """ nifti_json = { - 'Manufacturer': '', - 'ManufacturersModelName': '', - 'Units': '', - 'TracerName': '', - 'TracerRadionuclide': '', - 'InjectedRadioactivity': 0, - 'InjectedRadioactivityUnits': '', - 'InjectedMass': 0, - 'InjectedMassUnits': '', - 'SpecificRadioactivity': 0, - 'SpecificRadioactivityUnits': '', - 'ModeOfAdministration': '', - 'TimeZero': 0, - 'ScanStart': 0, - 'InjectionStart': 0, - 'FrameTimesStart': [], - 'FrameDuration': [], - 'AcquisitionMode': '', - 'ImageDecayCorrected': '', - 'ImageDecayCorrectionTime': 0, - 'ReconMethodName': '', - 'ReconMethodParameterLabels': [], - 'ReconMethodParameterUnits': [], - 'ReconMethodParameterValues': [], - 'ReconFilterType': '', - 'ReconFilterSize': 0, - 'AttenuationCorrection': '', - 'InstitutionName': '', - 'InstitutionalDepartmentName': '' + "Manufacturer": "", + "ManufacturersModelName": "", + "Units": "", + "TracerName": "", + "TracerRadionuclide": "", + "InjectedRadioactivity": 0, + "InjectedRadioactivityUnits": "", + "InjectedMass": 0, + "InjectedMassUnits": "", + "SpecificRadioactivity": 0, + "SpecificRadioactivityUnits": "", + "ModeOfAdministration": "", + "TimeZero": 0, + "ScanStart": 0, + "InjectionStart": 0, + "FrameTimesStart": [], + "FrameDuration": [], + "AcquisitionMode": "", + "ImageDecayCorrected": "", + "ImageDecayCorrectionTime": 0, + "ReconMethodName": "", + "ReconMethodParameterLabels": [], + "ReconMethodParameterUnits": [], + "ReconMethodParameterValues": [], + "ReconFilterType": "", + "ReconFilterSize": 0, + "AttenuationCorrection": "", + "InstitutionName": "", + "InstitutionalDepartmentName": "", } for key in nifti_json.keys(): try: - nifti_json[key] = flatten_series(metadata_dataframe[key]) + nifti_json[key] = helper_functions.flatten_series(metadata_dataframe[key]) except KeyError: warnings.warn(f"{key} not found in metadata extracted from spreadsheet") @@ -90,31 +88,31 @@ def translate_metadata(metadata_dataframe, image_path=NotImplemented): "DispersionCorrected": False, "time": { "Description": "Time in relation to time zero defined in _pet.json", - "Units": "s" + "Units": "s", }, "plasma_radioactivity": { "Description": "Radioactivity in plasma samples, measured by eye balling it.", - "Units": "kBq/mL" + "Units": "kBq/mL", }, "whole_blood_radioactivity": { "Description": "Radioactivity in whole blood samples, measured by divining rod.", - "Units": "kBq/mL" - } + "Units": "kBq/mL", + }, } blood_tsv = { "time": [], "plasma_radioactivity": [], - "whole_blood_radioactivity": [] + "whole_blood_radioactivity": [], } for key in blood_tsv.keys(): try: - blood_tsv[key] = flatten_series(metadata_dataframe[key]) + blood_tsv[key] = helper_functions.flatten_series(metadata_dataframe[key]) except KeyError: warnings.warn(f"{key} not found in metadata extracted from spreadsheet") # now transform the key value pairs in blood_tsv into a pandas dataframe object. blood_tsv = pandas.DataFrame.from_dict(blood_tsv) - return {'nifti_json': nifti_json, 'blood_json': blood_json, 'blood_tsv': blood_tsv} + return {"nifti_json": nifti_json, "blood_json": blood_json, "blood_tsv": blood_tsv} diff --git a/pypet2bids/pypet2bids/multiple_spreadsheets.py b/pypet2bids/pypet2bids/multiple_spreadsheets.py index 129198b7..3b7e78c4 100644 --- a/pypet2bids/pypet2bids/multiple_spreadsheets.py +++ b/pypet2bids/pypet2bids/multiple_spreadsheets.py @@ -1,46 +1,62 @@ from json_maj.main import JsonMAJ +import pathlib +import numpy +import typing +import os +import argparse + try: - from pypet2bids.helper_functions import * + import pypet2bids.helper_functions as helper_functions except ModuleNotFoundError: - from helper_functions import * + import helper_functions def read_multi_subject_spreadsheets( - general_metadata_spreadsheet: pathlib.Path, - multiple_subject_spreadsheet: pathlib.Path, - **kwargs) -> dict: + general_metadata_spreadsheet: pathlib.Path, + multiple_subject_spreadsheet: pathlib.Path, + **kwargs, +) -> dict: """ Reads in two spreadsheets as formatted in PET2BIDS/spreadsheet_conversion/many_subject_sheets, generic (scanner or subject independent data is supplied via the first argument and subject specific data is supplied via the second argument. :param general_metadata_spreadsheet: path to a metadata spreadsheet containing bids fields as columns - with values below + with values below :type general_metadata_spreadsheet: file path :param multiple_subject_spreadsheet: path to multi subject spreadsheet containing a subject id, participant id, - subject, or participant column consisting of paths to subject folders/files. + subject, or participant column consisting of paths to subject folders/files. :type multiple_subject_spreadsheet: file path :param kwargs: additional key pair arguments to pass on, these get applied generally - just like the first spreadsheet. e.g. TimeZero=12:12:12, SpecificRadioactivity=3 + just like the first spreadsheet. e.g. TimeZero=12:12:12, SpecificRadioactivity=3 :type kwargs: string or dict :return: dictionary of subject data extracted from each spreadsheet along with any additional - kwargs supplied + kwargs supplied :rtype: dict - + Anthony Galassi - ----------------------------- Copyright Open NeuroPET team - """ - required_fields = load_pet_bids_requirements_json() + """ - if general_metadata_spreadsheet.is_file() and multiple_subject_spreadsheet.is_file(): - general_metadata = single_spreadsheet_reader(general_metadata_spreadsheet) - multiple_subject_metadata = open_meta_data(multiple_subject_spreadsheet) + required_fields = helper_functions.load_pet_bids_requirements_json() + + if ( + general_metadata_spreadsheet.is_file() + and multiple_subject_spreadsheet.is_file() + ): + general_metadata = helper_functions.single_spreadsheet_reader( + general_metadata_spreadsheet + ) + multiple_subject_metadata = helper_functions.open_meta_data( + multiple_subject_spreadsheet + ) multiple_subject_metadata - column_set = set(list(general_metadata.keys()) + list(multiple_subject_metadata.columns)) + column_set = set( + list(general_metadata.keys()) + list(multiple_subject_metadata.columns) + ) column_set = list(column_set) @@ -48,26 +64,39 @@ def read_multi_subject_spreadsheets( for k in required_fields.keys(): for field in required_fields[k]: field_exists = field in column_set - if k == 'mandatory' and not field_exists: - logging.warning(f"Input spreadsheet(s) {general_metadata_spreadsheet} and " - f"{multiple_subject_spreadsheet} are missing required column {field}") - elif k == 'recommended' and not field_exists: - logging.info(f"Input spreadsheet(s) {general_metadata_spreadsheet} and " - f"{multiple_subject_spreadsheet} are missing recommended column {field}") - elif k == 'optional': - logging.info(f"Input spreadsheet(s) {general_metadata_spreadsheet} and " - f"{multiple_subject_spreadsheet} are missing optional column {field}") + if k == "mandatory" and not field_exists: + helper_functions.logging.warning( + f"Input spreadsheet(s) {general_metadata_spreadsheet} and " + f"{multiple_subject_spreadsheet} are missing required column {field}" + ) + elif k == "recommended" and not field_exists: + helper_functions.logging.info( + f"Input spreadsheet(s) {general_metadata_spreadsheet} and " + f"{multiple_subject_spreadsheet} are missing recommended column {field}" + ) + elif k == "optional": + helper_functions.logging.info( + f"Input spreadsheet(s) {general_metadata_spreadsheet} and " + f"{multiple_subject_spreadsheet} are missing optional column {field}" + ) # check to see if there's a subject column in the multi subject data - accepted_column_names = ['participant_id', 'participant', 'subject', 'subject_id'] + accepted_column_names = [ + "participant_id", + "participant", + "subject", + "subject_id", + ] found_column_names = [] for acceptable in accepted_column_names: if acceptable in multiple_subject_metadata.columns: found_column_names.append(acceptable) if len(found_column_names) != 1: - error_message = f"multi-subject_spreadsheet: {multiple_subject_spreadsheet} must contain only one column " \ - f"of the following names: " + error_message = ( + f"multi-subject_spreadsheet: {multiple_subject_spreadsheet} must contain only one column " + f"of the following names: " + ) for name in accepted_column_names: error_message += name + " " error_message += f"\nContains these instead {found_column_names}." @@ -78,17 +107,21 @@ def read_multi_subject_spreadsheets( # collect all subject id's subject_metadata = {} for subject in multiple_subject_metadata.get(subject_column, None): - subject_row = get_coordinates_containing(subject, multiple_subject_metadata, single=True)[0] - subject_id = collect_bids_part('sub', subject) - session_id = collect_bids_part('ses', subject) + subject_row = helper_functions.get_coordinates_containing( + subject, multiple_subject_metadata, single=True + )[0] + subject_id = helper_functions.collect_bids_part("sub", subject) + session_id = helper_functions.collect_bids_part("ses", subject) if subject_id: subject_metadata[subject_id] = general_metadata if session_id: - subject_metadata[subject_id]['session_id'] = session_id + subject_metadata[subject_id]["session_id"] = session_id if kwargs: subject_metadata[subject_id].update(**kwargs) - subject_data_from_row = transform_row_to_dict(subject_row, multiple_subject_metadata) + subject_data_from_row = helper_functions.transform_row_to_dict( + subject_row, multiple_subject_metadata + ) for k, v in subject_data_from_row.items(): if v and k != subject_column and v is not numpy.nan: subject_metadata[subject_id][k] = v @@ -96,18 +129,23 @@ def read_multi_subject_spreadsheets( return subject_metadata else: - missing_files = \ - [missing for missing in (general_metadata_spreadsheet, multiple_subject_spreadsheet) - if not missing.is_file()] + missing_files = [ + missing + for missing in (general_metadata_spreadsheet, multiple_subject_spreadsheet) + if not missing.is_file() + ] error_message = f"Missing " for m in missing_files: - error_message += f'{m=}'.split('=')[0] - error_message += f' {m}' + error_message += f"{m=}".split("=")[0] + error_message += f" {m}" raise Exception(error_message) -def write_multi_subject_spreadsheets(subjects: dict, output_path: typing.Union[str, pathlib.Path], - create_bids_tree: bool = False) -> None: +def write_multi_subject_spreadsheets( + subjects: dict, + output_path: typing.Union[str, pathlib.Path], + create_bids_tree: bool = False, +) -> None: """ Writes out a dictionary of subjects to a series of json files, if files exist updates them with new values obtained from spreadsheets. @@ -127,22 +165,28 @@ def write_multi_subject_spreadsheets(subjects: dict, output_path: typing.Union[s if create_bids_tree: for subject, fields in subjects.items(): json_out_path = os.path.join(output_path, f"{subject}") - if fields.get('session_id', None): - json_out_file_name = subject + "_" + f"{fields.get('session_id')}_pet.json" - json_out_path = os.path.join(json_out_path, fields.get('session_id'), 'pet') + if fields.get("session_id", None): + json_out_file_name = ( + subject + "_" + f"{fields.get('session_id')}_pet.json" + ) + json_out_path = os.path.join( + json_out_path, fields.get("session_id"), "pet" + ) else: json_out_file_name = subject + "_pet.json" - json_out_path = os.path.join(json_out_path, 'pet') + json_out_path = os.path.join(json_out_path, "pet") pathlib.Path(json_out_path).mkdir(parents=True, exist_ok=True) json_out_file_name = os.path.join(json_out_path, json_out_file_name) - json_out = JsonMAJ(json_path=json_out_file_name, update_values=fields).update() + json_out = JsonMAJ( + json_path=json_out_file_name, update_values=fields + ).update() else: for subject, fields in subjects.items(): json_out_path = os.path.join(output_path, f"{subject}") - if fields.get('session_id', None): + if fields.get("session_id", None): json_out_path += f"{fields.get('session_id')}_pet.json" else: - json_out_path += '_pet.json' + json_out_path += "_pet.json" json_out = JsonMAJ(json_path=json_out_path, update_values=fields).update() @@ -154,26 +198,40 @@ def cli(): :return: a dictionary version of the subject sidecar json's that get written out :rtype: dict """ - parser = argparse.ArgumentParser() - parser.add_argument("--general-spreadsheet", '-g', type=pathlib.Path, - help="Path to a spreadsheet with data applicable to mulitiple subjects") - parser.add_argument("--many-subjects-spreadsheet", '-m', type=pathlib.Path, - help="Path to spreadsheet containing multiple subjects") + parser = argparse.ArgumentParser( + description="allows reading and writing of 2 spreadsheets following the format " + "specified in PET2BIDS/spreadsheet_conversion/many_subjects_sheet" + ) + parser.add_argument( + "--general-spreadsheet", + "-g", + type=pathlib.Path, + help="Path to a spreadsheet with data applicable to mulitiple subjects", + ) + parser.add_argument( + "--many-subjects-spreadsheet", + "-m", + type=pathlib.Path, + help="Path to spreadsheet containing multiple subjects", + ) parser.add_argument("--output-path", "-o", type=pathlib.Path) - parser.add_argument("--bids-tree", "-b", action='store_true') + parser.add_argument("--bids-tree", "-b", action="store_true") args = parser.parse_args() subjects = read_multi_subject_spreadsheets( general_metadata_spreadsheet=args.general_spreadsheet, - multiple_subject_spreadsheet=args.many_subjects_spreadsheet) + multiple_subject_spreadsheet=args.many_subjects_spreadsheet, + ) if args.output_path: output_path = args.output_path else: output_path = os.getcwd() - write_multi_subject_spreadsheets(output_path=output_path, subjects=subjects, create_bids_tree=args.bids_tree) + write_multi_subject_spreadsheets( + output_path=output_path, subjects=subjects, create_bids_tree=args.bids_tree + ) return subjects -if __name__ == '__main__': +if __name__ == "__main__": x = cli() diff --git a/pypet2bids/pypet2bids/read_ecat.py b/pypet2bids/pypet2bids/read_ecat.py index 42bdff1a..ecae98f3 100644 --- a/pypet2bids/pypet2bids/read_ecat.py +++ b/pypet2bids/pypet2bids/read_ecat.py @@ -1,5 +1,5 @@ """ -This module contains methods used to read ecat files (*.v), the primary method pulled/imported from this module +This module contains methods used to read ecat files (\*.v), the primary method pulled/imported from this module is read_ecat which returns the contents of a singular ecat file divided into three parts: - main header @@ -11,7 +11,9 @@ :Authors: Anthony Galassi :Copyright: Open NeuroPET team + """ + import json import os.path import struct @@ -20,20 +22,32 @@ import pathlib import re import numpy -from pypet2bids.helper_functions import decompress +from pypet2bids.helper_functions import decompress, first_middle_last_frames_to_text parent_dir = pathlib.Path(__file__).parent.resolve() code_dir = parent_dir.parent data_dir = code_dir.parent +# debug variable +# save steps for debugging for more info see ecat_testing/README.md +ecat_save_steps = os.environ.get("ECAT_SAVE_STEPS", 0) +if ecat_save_steps == "1": + # check to see if the code directory is available, if it's not create it and + # the steps dir to save outputs created if ecat_save_steps is set to 1 + steps_dir = code_dir.parent / "ecat_testing" / "steps" + if not steps_dir.is_dir(): + os.makedirs(code_dir.parent / "ecat_testing" / "steps", exist_ok=True) + # collect ecat header maps, this program will not work without these as ECAT data varies in the byte location of its # data depending on the version of ECAT it was formatted with. try: - with open(join(parent_dir, 'ecat_headers.json'), 'r') as infile: + with open(join(parent_dir, "ecat_headers.json"), "r") as infile: ecat_header_maps = json.load(infile) except FileNotFoundError: - raise Exception("Unable to load header definitions and map from ecat_headers.json. Aborting.") + raise Exception( + "Unable to load header definitions and map from ecat_headers.json. Aborting." + ) # noinspection PyShadowingNames @@ -48,7 +62,7 @@ def get_ecat_bytes(path_to_ecat: str): """ # check if file exists if path.isfile(path_to_ecat): - with open(path_to_ecat, 'rb') as infile: + with open(path_to_ecat, "rb") as infile: ecat_bytes = infile.read() else: raise Exception(f"No such file found at {path_to_ecat}") @@ -69,7 +83,7 @@ def read_bytes(path_to_bytes: str, byte_start: int, byte_stop: int = -1): raise Exception(f"{path_to_bytes} is not a valid file.") # open that file - bytes_to_read = open(path_to_bytes, 'rb') + bytes_to_read = open(path_to_bytes, "rb") # move to start byte bytes_to_read.seek(byte_start, 0) @@ -82,19 +96,20 @@ def read_bytes(path_to_bytes: str, byte_start: int, byte_stop: int = -1): return sought_bytes -def collect_specific_bytes(bytes_object: bytes, start_position: int = 0, width: int = 0): +def collect_specific_bytes( + bytes_object: bytes, start_position: int = 0, width: int = 0 +): """ Collects specific bytes within a bytes object. :param bytes_object: an opened bytes object :param start_position: the position to start to read at :param width: how far to read from the start position - :param relative_to: position relative to 0 -> start of file/object, 1 -> current position of seek head, 2 -> end of file/object :return: the bytes starting at position """ # navigate to byte position - content = bytes_object[start_position: start_position + width] + content = bytes_object[start_position : start_position + width] return {"content": content, "new_position": start_position + width} @@ -107,17 +122,19 @@ def get_buffer_size(data_type: str, variable_name: str): :param variable_name: :return: the number of bytes to expand a buffer to """ - first_split = variable_name.split('(') + first_split = variable_name.split("(") if len(first_split) == 2: fill_scalar = int(first_split[1][:-1]) else: fill_scalar = 1 - scalar = int(re.findall(r'\d+', data_type)[0]) * fill_scalar + scalar = int(re.findall(r"\d+", data_type)[0]) * fill_scalar return scalar -def get_header_data(header_data_map: dict, ecat_file: str = '', byte_offset: int = 0, clean=True): +def get_header_data( + header_data_map: dict, ecat_file: str = "", byte_offset: int = 0, clean=True +): """ ECAT header data is contained in json files translated from ECAT documentation provided via the Turku PET Inst. For machine and human readability the original Siemens PDF/Scanned ECAT file documentation has been rewritten into @@ -137,14 +154,18 @@ def get_header_data(header_data_map: dict, ecat_file: str = '', byte_offset: int header = {} for value in header_data_map: - byte_position, variable_name, struct_fmt = value['byte'], value['variable_name'], '>' + value['struct'] + byte_position, variable_name, struct_fmt = ( + value["byte"], + value["variable_name"], + ">" + value["struct"], + ) byte_width = struct.calcsize(struct_fmt) relative_byte_position = byte_position + byte_offset raw_bytes = read_bytes(ecat_file, relative_byte_position, byte_width) header[variable_name] = struct.unpack(struct_fmt, raw_bytes) - if clean and 'fill' not in variable_name.lower(): + if clean and "fill" not in variable_name.lower(): header[variable_name] = filter_bytes(header[variable_name], struct_fmt) - if 'comment' in value and 'msec' in value['comment']: + if "comment" in value and "msec" in value["comment"]: # for entries that are in msec, convert to sec for PET BIDS json header[variable_name] /= 1000 read_head_position = relative_byte_position + byte_width @@ -166,8 +187,8 @@ def filter_bytes(unfiltered: bytes, struct_fmt: str): unfiltered = unfiltered[0] elif len(unfiltered) > 1: unfiltered = list(unfiltered) - if 's' in struct_fmt: - filtered = str(bytes(filter(None, unfiltered)), 'UTF-8') + if "s" in struct_fmt: + filtered = str(bytes(filter(None, unfiltered)), "UTF-8") else: filtered = unfiltered return filtered @@ -195,23 +216,27 @@ def get_directory_data(byte_block: bytes, ecat_file: str, return_raw: bool = Fal # observed to signal the end of an additional 512 byte block/buffer when the number of frames # exceeds 31 - read_byte_array = numpy.frombuffer(byte_block, dtype=numpy.dtype('>i4'), count=-1) + read_byte_array = numpy.frombuffer( + byte_block, dtype=numpy.dtype(">i4"), count=-1 + ) # reshape 1d array into 2d, a 4 row by 32 column table is expected reshaped = numpy.transpose(numpy.reshape(read_byte_array, (-1, 4))) raw.append(reshaped) # chop off columns after 32, rows after 32 appear to be noise - reshaped = reshaped[:, 0:read_byte_array[3] + 1] + reshaped = reshaped[:, 0 : read_byte_array[3] + 1] # get directory size/number of frames in dir from 1st column 4th row of the array in the buffer directory_size = reshaped[3, 0] if directory_size == 0: break # on the first pass do this if directory is None: - directory = reshaped[:, 1:directory_size + 1] + directory = reshaped[:, 1 : directory_size + 1] else: - directory = numpy.append(directory, reshaped[:, 1:directory_size + 1], axis=1) + directory = numpy.append( + directory, reshaped[:, 1 : directory_size + 1], axis=1 + ) # determine if this is the last directory by examining the 2nd row of the first column of the buffer next_directory_position = reshaped[1, 0] if next_directory_position == 2: @@ -220,7 +245,7 @@ def get_directory_data(byte_block: bytes, ecat_file: str, return_raw: bool = Fal byte_block = read_bytes( path_to_bytes=ecat_file, byte_start=(next_directory_position - 1) * 512, - byte_stop=512 + byte_stop=512, ) # sort the directory contents as they're sometimes out of order @@ -232,41 +257,53 @@ def get_directory_data(byte_block: bytes, ecat_file: str, return_raw: bool = Fal return sorted_directory -def read_ecat(ecat_file: str, calibrated: bool = False, collect_pixel_data: bool = True): +def read_ecat( + ecat_file: str, calibrated: bool = False, collect_pixel_data: bool = True +): """ Reads in an ecat file and collects the main header data, subheader data, and imagining data. :param ecat_file: path to an ecat file, does not handle compression currently :param calibrated: if True, will scale the raw imaging data by the SCALE_FACTOR in the subheader and :param collect_pixel_data: By default collects the entire ecat, can be passed false to only return headers - CALIBRATION_FACTOR in the main header + CALIBRATION_FACTOR in the main header :return: main_header, a list of subheaders for each frame, the imagining data from the subheaders + """ if ".gz" in ecat_file: ecat_file = decompress(ecat_file) # try to determine what type of ecat this is possible_ecat_headers = {} - for entry, dictionary in ecat_header_maps['ecat_headers'].items(): - possible_ecat_headers[entry] = dictionary['mainheader'] + for entry, dictionary in ecat_header_maps["ecat_headers"].items(): + possible_ecat_headers[entry] = dictionary["mainheader"] + + # set byte order + byte_order = ">" confirmed_version = None for version, dictionary in possible_ecat_headers.items(): try: possible_header, _ = get_header_data(dictionary, ecat_file) - if version == str(possible_header['SW_VERSION']): + if version == str(possible_header["SW_VERSION"]): confirmed_version = version break except UnicodeDecodeError: continue if not confirmed_version: - raise Exception(f"Unable to determine ECAT File Type from these types {possible_ecat_headers.keys()}") + raise Exception( + f"Unable to determine ECAT File Type from these types {possible_ecat_headers.keys()}" + ) - ecat_main_header = ecat_header_maps['ecat_headers'][confirmed_version]['mainheader'] + ecat_main_header = ecat_header_maps["ecat_headers"][confirmed_version]["mainheader"] main_header, read_to = get_header_data(ecat_main_header, ecat_file) + + if ecat_save_steps == "1": + with open(steps_dir / "1_read_mh_ecat_python.json", "w") as outfile: + json.dump(main_header, outfile) """ Some notes about the file directory/sorted directory: @@ -291,14 +328,13 @@ def read_ecat(ecat_file: str, calibrated: bool = False, collect_pixel_data: bool # Collecting First Part of File Directory/Index this Directory lies directly after the main header byte block next_block = read_bytes( - path_to_bytes=ecat_file, - byte_start=read_to, - byte_stop=read_to + 512) + path_to_bytes=ecat_file, byte_start=read_to, byte_stop=read_to + 512 + ) directory = get_directory_data(next_block, ecat_file) # determine subheader type by checking main header - subheader_type_number = main_header['FILE_TYPE'] + subheader_type_number = main_header["FILE_TYPE"] """ ECAT 7.2 Only @@ -339,7 +375,9 @@ def read_ecat(ecat_file: str, calibrated: bool = False, collect_pixel_data: bool """ # collect the bytes map file for the designated subheader, note some are not supported. - subheader_map = ecat_header_maps['ecat_headers'][confirmed_version][str(subheader_type_number)] + subheader_map = ecat_header_maps["ecat_headers"][confirmed_version][ + str(subheader_type_number) + ] if not subheader_map: raise Exception(f"Unsupported data type: {subheader_type_number}") @@ -358,41 +396,67 @@ def read_ecat(ecat_file: str, calibrated: bool = False, collect_pixel_data: bool frame_start_byte_position = 512 * (frame_start - 1) # sure why not # read subheader - subheader, byte_position = get_header_data(subheader_map, - ecat_file, - byte_offset=frame_start_byte_position) + subheader, byte_position = get_header_data( + subheader_map, ecat_file, byte_offset=frame_start_byte_position + ) if collect_pixel_data: # collect pixel data from file - pixel_data = read_bytes(path_to_bytes=ecat_file, - byte_start=512 * frame_start, - byte_stop=512 * frame_stop) + pixel_data = read_bytes( + path_to_bytes=ecat_file, + byte_start=512 * frame_start, + byte_stop=512 * frame_stop, + ) # calculate size of matrix for pixel data, may vary depending on image type (polar, 3d, etc.) if subheader_type_number == 7: - image_size = [subheader['X_DIMENSION'], subheader['Y_DIMENSION'], subheader['Z_DIMENSION']] + image_size = [ + subheader["X_DIMENSION"], + subheader["Y_DIMENSION"], + subheader["Z_DIMENSION"], + ] # check subheader for pixel datatype - dt_val = subheader['DATA_TYPE'] + dt_val = subheader["DATA_TYPE"] if dt_val == 5: - formatting = '>f4' + formatting = ">f4" pixel_data_type = numpy.dtype(formatting) elif dt_val == 6: - pixel_data_type = '>H' + # >H is unsigned short e.g. >u2, reverting to int16 e.g. i2 to align with commit 9beee53 + pixel_data_type = ">i2" else: raise ValueError( - f"Unable to determine pixel data type from value: {dt_val} extracted from {subheader}") + f"Unable to determine pixel data type from value: {dt_val} extracted from {subheader}" + ) # read it into a one dimensional matrix - pixel_data_matrix_3d = numpy.frombuffer(pixel_data, - dtype=pixel_data_type, - count=image_size[0] * image_size[1] * image_size[2]).reshape( - *image_size, order='F') + pixel_data_matrix_3d = numpy.frombuffer( + pixel_data, + dtype=pixel_data_type, + count=image_size[0] * image_size[1] * image_size[2], + ).reshape(*image_size, order="F") + # pixel_data_matrix_3d.newbyteorder(byte_order) + + if ecat_save_steps == "1": + with open( + steps_dir / f"4_determine_data_type_python.json", "w" + ) as outfile: + json.dump( + {"DATA_TYPE": dt_val, "formatting": pixel_data_type}, + outfile, + ) + else: - raise Exception(f"Unable to determine frame image size, unsupported image type {subheader_type_number}") + raise Exception( + f"Unable to determine frame image size, unsupported image type {subheader_type_number}" + ) # we assume the user doesn't want to do multiplication to adjust for calibration here if calibrated: - calibration_factor = subheader['SCALE_FACTOR'] * main_header['ECAT_CALIBRATION_FACTOR'] - calibrated_pixel_data_matrix_3d = calibration_factor * pixel_data_matrix_3d + calibration_factor = ( + subheader["SCALE_FACTOR"] * main_header["ECAT_CALIBRATION_FACTOR"] + ) + calibrated_pixel_data_matrix_3d = ( + calibration_factor * pixel_data_matrix_3d + ) data.append(calibrated_pixel_data_matrix_3d) else: data.append(pixel_data_matrix_3d) @@ -401,12 +465,44 @@ def read_ecat(ecat_file: str, calibrated: bool = False, collect_pixel_data: bool subheaders.append(subheader) + if ecat_save_steps == "1": + with open(steps_dir / f"2_read_sh_ecat_python.json", "w") as outfile: + json.dump({"subheaders": subheaders}, outfile) + if collect_pixel_data: # return 4d array instead of list of 3d arrays - pixel_data_matrix_4d = numpy.zeros(tuple(image_size + [len(data)]), dtype=numpy.dtype(pixel_data_type)) + pixel_data_matrix_4d = numpy.zeros( + tuple(image_size + [len(data)]), dtype=numpy.dtype(pixel_data_type) + ) for index, frame in enumerate(data): pixel_data_matrix_4d[:, :, :, index] = frame + if ecat_save_steps == "1": + + # write out the endianess and datatype of the pixel data matrix + step_3_dict = { + "datatype": pixel_data_matrix_4d.dtype.name, + "endianness": pixel_data_matrix_4d.dtype.byteorder, + } + + with open(steps_dir / f"3_determine_data_type_python.json", "w") as outfile: + json.dump(step_3_dict, outfile) + + # record out step 4 + first_middle_last_frames_to_text( + four_d_array_like_object=pixel_data_matrix_4d, + output_folder=steps_dir, + step_name="4_read_img_ecat_python", + ) + + # record out step 5 + if calibrated: + first_middle_last_frames_to_text( + four_d_array_like_object=calibrated_pixel_data_matrix_3d, + output_folder=steps_dir, + step_name="5_scale_img_ecat_python", + ) + else: pixel_data_matrix_4d = None diff --git a/pypet2bids/pypet2bids/sidecar.py b/pypet2bids/pypet2bids/sidecar.py index bc5806e9..ac68dc5c 100644 --- a/pypet2bids/pypet2bids/sidecar.py +++ b/pypet2bids/pypet2bids/sidecar.py @@ -1,16 +1,18 @@ """ This is a lazy way to avoid opening a json, simply import this file to collect your BIDS sidecar templates instead. This -is not a function or a `true` module. It's just two python dictionaries with keys and empty value pairs. +is not a function or a true module. This is just two python dictionaries with keys and empty value pairs. -:param sidecar_template_full: a dictionary containing every field specified in the BIDS standard for PET imaging data -:param sidecar_template_short: a dictionary containing only the required fields in the BIDS standard for PET - imaging data -:return: sidecar_template_full, sidecar_template_short +**Parameters** -|*Anthony Galassi* -|*Copyright OpenNeuroPET team* -""" +* sidecar_template_full: a dict containing every PET BIDS field +* sidecar_template_short: a dict containing only required PET BIDS fields + +**Returns** sidecar_template_full, sidecar_template_short +*Anthony Galassi* +*Copyright OpenNeuroPET team* + +""" sidecar_template_full = { "Manufacturer": "", @@ -75,7 +77,7 @@ "DecayCorrectionFactor": [], "PromptRate": [], "RandomRate": [], - "SinglesRate": [] + "SinglesRate": [], } sidecar_template_short = { @@ -105,5 +107,5 @@ "ReconMethodParameterValues": [], "ReconFilterType": [], "ReconFilterSize": [], - "AttenuationCorrection": "" + "AttenuationCorrection": "", } diff --git a/pypet2bids/pypet2bids/single_spreadsheet.py b/pypet2bids/pypet2bids/single_spreadsheet.py index 2c44e716..47d9f80a 100644 --- a/pypet2bids/pypet2bids/single_spreadsheet.py +++ b/pypet2bids/pypet2bids/single_spreadsheet.py @@ -10,12 +10,13 @@ except ModuleNotFoundError: import pypet2bids.helper_functions as helper_functions -#from pypet2bids.helper_functions import single_spreadsheet_reader, \ +# from pypet2bids.helper_functions import single_spreadsheet_reader, \ # collect_bids_part, open_meta_data, load_pet_bids_requirements_json, ParseKwargs + def read_single_subject_spreadsheets( - general_metadata_spreadsheet: pathlib.Path, - **kwargs) -> dict: + general_metadata_spreadsheet: pathlib.Path, **kwargs +) -> dict: """ Reads in spreadsheet as formatted in PET2BIDS/spreadsheet_conversion/single_subject_sheet, extra arguments are supplied in key pair form via kwargs as is convention. @@ -30,7 +31,7 @@ def read_single_subject_spreadsheets( :return: dictionary of subject data extracted from each spreadsheet along with any additional kwargs supplied :rtype: dict - + Anthony Galassi ----------------------------- Copyright Open NeuroPET team @@ -38,64 +39,87 @@ def read_single_subject_spreadsheets( required_fields = helper_functions.load_pet_bids_requirements_json() - subject_id = kwargs.get('subject_id', None) - session_id = kwargs.get('session_id', None) + subject_id = kwargs.get("subject_id", None) + session_id = kwargs.get("session_id", None) if general_metadata_spreadsheet.is_file(): - general_metadata = helper_functions.single_spreadsheet_reader(general_metadata_spreadsheet) + general_metadata = helper_functions.single_spreadsheet_reader( + general_metadata_spreadsheet + ) if not subject_id: - subject_id = helper_functions.collect_bids_part('sub', str(general_metadata_spreadsheet)) + subject_id = helper_functions.collect_bids_part( + "sub", str(general_metadata_spreadsheet) + ) if subject_id: - general_metadata['subject_id'] = subject_id + general_metadata["subject_id"] = subject_id else: - general_metadata['subject_id'] = subject_id + general_metadata["subject_id"] = subject_id if not session_id: - session_id = helper_functions.collect_bids_part('ses', str(general_metadata_spreadsheet)) + session_id = helper_functions.collect_bids_part( + "ses", str(general_metadata_spreadsheet) + ) if session_id: - general_metadata['session_id'] = session_id + general_metadata["session_id"] = session_id else: - general_metadata['session_id'] = session_id + general_metadata["session_id"] = session_id column_set = list(general_metadata.keys()) # assert required columns are in input spreadsheets for k in required_fields.keys(): for field in required_fields[k]: field_exists = field in column_set - if k == 'mandatory' and not field_exists: - logging.warning(f"Input spreadsheet {general_metadata_spreadsheet} is missing required " - f"column {field}") - elif k == 'recommended' and not field_exists: - logging.info(f"Input spreadsheet(s) {general_metadata_spreadsheet} and is missing " - f"recommended column {field}") - elif k == 'optional': - logging.info(f"Input spreadsheet(s) {general_metadata_spreadsheet} is missing " - f"optional column {field}") + if k == "mandatory" and not field_exists: + logging.warning( + f"Input spreadsheet {general_metadata_spreadsheet} is missing required " + f"column {field}" + ) + elif k == "recommended" and not field_exists: + logging.info( + f"Input spreadsheet(s) {general_metadata_spreadsheet} and is missing " + f"recommended column {field}" + ) + elif k == "optional": + logging.info( + f"Input spreadsheet(s) {general_metadata_spreadsheet} is missing " + f"optional column {field}" + ) # check to see if there's a subject column in the multi subject data - accepted_column_names = ['participant_id', 'participant', 'subject', 'subject_id'] - columns = helper_functions.open_meta_data(metadata_path=general_metadata_spreadsheet).columns + accepted_column_names = [ + "participant_id", + "participant", + "subject", + "subject_id", + ] + columns = helper_functions.open_meta_data( + metadata_path=general_metadata_spreadsheet + ).columns found_column_names = [] for acceptable in accepted_column_names: if acceptable in columns: found_column_names.append(acceptable) if len(found_column_names) > 1: - error_message = f"single_subject_spreadsheet: {general_metadata_spreadsheet} must contain only one column " \ - f"of the following names: " + error_message = ( + f"single_subject_spreadsheet: {general_metadata_spreadsheet} must contain only one column " + f"of the following names: " + ) for name in accepted_column_names: error_message += name + " " error_message += f"\nContains these instead {found_column_names}." raise Exception(error_message) elif len(found_column_names) >= 0 and subject_id: if len(found_column_names) > 0: - logging.warning(f"Found subject id in filepath {general_metadata_spreadsheet} and column(s) " - f"{found_column_names}. Defaulting to the value {subject_id} found in the file path " - f"{general_metadata_spreadsheet}.") + logging.warning( + f"Found subject id in filepath {general_metadata_spreadsheet} and column(s) " + f"{found_column_names}. Defaulting to the value {subject_id} found in the file path " + f"{general_metadata_spreadsheet}." + ) elif len(found_column_names) == 1 and not subject_id: subject_id_column = found_column_names[0] subject_id = general_metadata.pop(subject_id_column) - general_metadata['subject_id'] = subject_id + general_metadata["subject_id"] = subject_id return general_metadata @@ -103,8 +127,11 @@ def read_single_subject_spreadsheets( raise FileNotFoundError(general_metadata_spreadsheet) -def write_single_subject_spreadsheets(subject_metadata: dict, output_path: typing.Union[str, pathlib.Path], - create_bids_tree: bool = False) -> str: +def write_single_subject_spreadsheets( + subject_metadata: dict, + output_path: typing.Union[str, pathlib.Path], + create_bids_tree: bool = False, +) -> str: """ Writes out a dictionary of subjects to a series of json files, if files exist updates them with new values obtained from spreadsheets. @@ -121,17 +148,17 @@ def write_single_subject_spreadsheets(subject_metadata: dict, output_path: typin :rtype: str """ - subject_id = subject_metadata.get('subject_id', None) + subject_id = subject_metadata.get("subject_id", None) if subject_id: - subject_metadata.pop('subject_id') + subject_metadata.pop("subject_id") else: - subject_id = helper_functions.collect_bids_part('sub', output_path) + subject_id = helper_functions.collect_bids_part("sub", output_path) - session_id = subject_metadata.get('session_id', None) + session_id = subject_metadata.get("session_id", None) if session_id: - subject_metadata.pop('session_id') + subject_metadata.pop("session_id") else: - session_id = helper_functions.collect_bids_part('ses', output_path) + session_id = helper_functions.collect_bids_part("ses", output_path) if create_bids_tree: if subject_id not in output_path.parts: @@ -143,19 +170,22 @@ def write_single_subject_spreadsheets(subject_metadata: dict, output_path: typin else: json_out_file_name = subject_id + "_pet.json" - if output_path.parts[-1] != 'pet': - output_path = output_path / 'pet' + if output_path.parts[-1] != "pet": + output_path = output_path / "pet" output_path.mkdir(parents=True, exist_ok=True) - JsonMAJ(json_path=os.path.join(output_path, json_out_file_name), update_values=subject_metadata).update() + JsonMAJ( + json_path=os.path.join(output_path, json_out_file_name), + update_values=subject_metadata, + ).update() else: output_path.expanduser().mkdir(exist_ok=True, parents=True) json_out_path = os.path.join(output_path, f"{subject_id}") if session_id: json_out_path += f"_{session_id}_pet.json" else: - json_out_path += '_pet.json' + json_out_path += "_pet.json" JsonMAJ(json_path=json_out_path, update_values=subject_metadata).update() @@ -169,27 +199,38 @@ def cli(): :return: a dictionary version of the subject sidecar json's that get written out :rtype: dict """ - parser = argparse.ArgumentParser() - parser.add_argument("spreadsheet", type=pathlib.Path, - help="Path to a spreadsheet with data applicable to mulitiple subjects") + parser = argparse.ArgumentParser( + description="allows reading and writing of a spreadsheet following the format " + "specified in PET2BIDS/spreadsheet_conversion/single_subject_sheet/ " + "to a json" + ) + parser.add_argument( + "spreadsheet", + type=pathlib.Path, + help="Path to a spreadsheet with data applicable to mulitiple subjects", + ) parser.add_argument("--output-path", "-o", type=pathlib.Path) - parser.add_argument("--bids-tree", "-b", action='store_true') - parser.add_argument("--kwargs", "-k", nargs="*", action=helper_functions.ParseKwargs, default={}) + parser.add_argument("--bids-tree", "-b", action="store_true") + parser.add_argument( + "--kwargs", "-k", nargs="*", action=helper_functions.ParseKwargs, default={} + ) args = parser.parse_args() subject = read_single_subject_spreadsheets( - general_metadata_spreadsheet=args.spreadsheet, - **args.kwargs) + general_metadata_spreadsheet=args.spreadsheet, **args.kwargs + ) if args.output_path: output_path = args.output_path.expanduser() else: output_path = pathlib.Path(os.getcwd()) - write_single_subject_spreadsheets(output_path=output_path.expanduser(), - subject_metadata=subject, - create_bids_tree=args.bids_tree) + write_single_subject_spreadsheets( + output_path=output_path.expanduser(), + subject_metadata=subject, + create_bids_tree=args.bids_tree, + ) return subject -if __name__ == '__main__': +if __name__ == "__main__": x = cli() diff --git a/pypet2bids/pypet2bids/update_json_pet_file.py b/pypet2bids/pypet2bids/update_json_pet_file.py new file mode 100644 index 00000000..dbd1bd74 --- /dev/null +++ b/pypet2bids/pypet2bids/update_json_pet_file.py @@ -0,0 +1,876 @@ +from pathlib import Path +from os.path import join +from os import listdir +import json +from json_maj.main import JsonMAJ, load_json_or_dict +import re +from dateutil import parser +import argparse +import pydicom +import datetime +from typing import Union + +try: + import helper_functions + import is_pet +except ModuleNotFoundError: + import pypet2bids.helper_functions as helper_functions + import pypet2bids.is_pet as is_pet + +# import logging +logger = helper_functions.logger("pypet2bids") + +# load module and metadata_json paths from helper_functions +module_folder, metadata_folder = ( + helper_functions.module_folder, + helper_functions.metadata_folder, +) + +try: + # collect metadata jsons in dev mode + metadata_jsons = [ + Path(join(metadata_folder, metadata_json)) + for metadata_json in listdir(metadata_folder) + if ".json" in metadata_json + ] +except FileNotFoundError: + metadata_jsons = [ + Path(join(module_folder, "metadata", metadata_json)) + for metadata_json in listdir(join(module_folder, "metadata")) + if ".json" in metadata_json + ] + +# create a dictionary to house the PET metadata files +metadata_dictionaries = {} + +for metadata_json in metadata_jsons: + try: + with open(metadata_json, "r") as infile: + dictionary = json.load(infile) + + metadata_dictionaries[metadata_json.name] = dictionary + except FileNotFoundError as err: + raise Exception( + f"Missing pet metadata file {metadata_json} in {metadata_folder}, unable to validate metadata." + ) + except json.decoder.JSONDecodeError as err: + raise IOError(f"Unable to read from {metadata_json}") + + +def check_json( + path_to_json, + items_to_check=None, + silent=False, + spreadsheet_metadata={}, + mandatory=True, + recommended=True, + logger_name="pypet2bids", + **additional_arguments, +): + """ + This method opens a json and checks to see if a set of mandatory values is present within that json, optionally it + also checks for recommended key value pairs. If fields are not present a warning is raised to the user. + + :param spreadsheet_metadata: + :type spreadsheet_metadata: + :param path_to_json: path to a json file e.g. a BIDS sidecar file created after running dcm2niix + :param items_to_check: a dictionary with items to check for within that json. If None is supplied defaults to the + PET_metadata.json contained in this repository + :param silent: Raises warnings or errors to stdout if this flag is set to True + :return: dictionary of items existence and value state, if key is True/False there exists/(does not exist) a + corresponding entry in the json the same can be said of value + """ + + logger = helper_functions.logger(logger_name) + + if silent: + logger.disabled = True + else: + logger.disabled = False + + # check if path exists + path_to_json = Path(path_to_json) + if not path_to_json.exists(): + raise FileNotFoundError(path_to_json) + + # check for default argument for dictionary of items to check + if items_to_check is None: + items_to_check = metadata_dictionaries["PET_metadata.json"] + # remove blood tsv data from items to check + if items_to_check.get("blood_recording_fields", None): + items_to_check.pop("blood_recording_fields") + + # open the json + with open(path_to_json, "r") as in_file: + json_to_check = json.load(in_file) + + # initialize warning colors and warning storage dictionary + storage = {} + flattened_spreadsheet_metadata = {} + flattened_spreadsheet_metadata.update(spreadsheet_metadata.get("nifti_json", {})) + flattened_spreadsheet_metadata.update(spreadsheet_metadata.get("blood_json", {})) + flattened_spreadsheet_metadata.update(spreadsheet_metadata.get("blood_tsv", {})) + + if mandatory: + for item in items_to_check["mandatory"]: + all_good = False + if ( + item in json_to_check.keys() + and ( + json_to_check.get(item, None) is not None + or json_to_check.get(item) != "" + ) + or item in additional_arguments + or item in flattened_spreadsheet_metadata.keys() + ): + # this json has both the key and a non-blank value do nothing + all_good = True + pass + elif item in json_to_check.keys() and ( + json_to_check.get(item, None) is None + or json_to_check.get(item, None) == "" + ): + logger.error(f"{item} present but has null value.") + storage[item] = {"key": True, "value": False} + elif not all_good: + logger.error( + f"{item} is not present in {path_to_json}. This will have to be " + f"corrected post conversion." + ) + storage[item] = {"key": False, "value": False} + + if recommended: + for item in items_to_check["recommended"]: + all_good = False + if ( + item in json_to_check.keys() + and ( + json_to_check.get(item, None) is not None + or json_to_check.get(item) != "" + ) + or item in additional_arguments + or item in flattened_spreadsheet_metadata.keys() + ): + # this json has both the key and a non-blank value do nothing + all_good = True + pass + elif item in json_to_check.keys() and ( + json_to_check.get(item, None) is None + or json_to_check.get(item, None) == "" + ): + logger.info(f"{item} present but has null value.") + storage[item] = {"key": True, "value": False} + elif not all_good: + logger.info(f"{item} is recommended but not present in {path_to_json}") + storage[item] = {"key": False, "value": False} + + return storage + + +def update_json_with_dicom_value( + path_to_json, + missing_values, + dicom_header, + dicom2bids_json=None, + silent=True, + **additional_arguments, +): + """ + We go through all the missing values or keys that we find in the sidecar json and attempt to extract those + missing entities from the dicom source. This function relies on many heuristics a.k.a. many unique conditionals and + simply is what it is, hate the game not the player. + + :param path_to_json: path to the sidecar json to check + :param missing_values: dictionary output from check_json indicating missing fields and/or values + :param dicom_header: the dicom or dicoms that may contain information not picked up by dcm2niix + :param dicom2bids_json: a json file that maps dicom header entities to their corresponding BIDS entities + :return: a dictionary of sucessfully updated (written to the json file) fields and values + """ + + if silent: + logger.disabled = True + + json_sidecar_path = Path(path_to_json) + if not json_sidecar_path.exists(): + with open(path_to_json, "w") as outfile: + json.dump({}, outfile) + + # load the sidecar json + sidecar_json = load_json_or_dict(str(path_to_json)) + + # purely to clean up the generated read the docs page from sphinx, otherwise the entire json appears in the + # read the docs page. + if dicom2bids_json is None: + dicom2bids_json = metadata_dictionaries["dicom2bids.json"] + + # Units gets written as Unit in older versions of dcm2niix here we check for missing Units and present Unit entity + units = missing_values.get("Units", None) + if units: + try: + # Units is missing, check to see if Unit is present + if sidecar_json.get("Unit", None): + temp = JsonMAJ( + path_to_json, {"Units": sidecar_json.get("Unit")}, bids_null=True + ) + temp.remove("Unit") + else: # we source the Units value from the dicom header and update the json + JsonMAJ(path_to_json, {"Units": dicom_header.Units}, bids_null=True) + except AttributeError: + logger.error( + f"Dicom is missing Unit(s) field, are you sure this is a PET dicom?" + ) + # pair up dicom fields with bids sidecar json field, we do this in a separate json file + # it's loaded when this script is run and stored in metadata dictionaries + dcmfields = dicom2bids_json["dcmfields"] + jsonfields = dicom2bids_json["jsonfields"] + + regex_cases = ["ReconstructionMethod", "ConvolutionKernel"] + + # strip excess characters from dcmfields + dcmfields = [re.sub("[^0-9a-zA-Z]+", "", field) for field in dcmfields] + paired_fields = {} + for index, field in enumerate(jsonfields): + paired_fields[field] = dcmfields[index] + + logger.info("Attempting to locate missing BIDS fields in dicom header") + # go through missing fields and reach into dicom to pull out values + json_updater = JsonMAJ(json_path=path_to_json, bids_null=True) + for key, value in paired_fields.items(): + missing_bids_field = missing_values.get(key, None) + # if field is missing look into dicom + if missing_bids_field and key not in additional_arguments: + # there are a few special cases that require regex splitting of the dicom entries + # into several bids sidecar entities + try: + dicom_field = getattr(dicom_header, value) + logger.info(f"FOUND {value} corresponding to BIDS {key}: {dicom_field}") + except AttributeError: + dicom_field = None + logger.info( + f"NOT FOUND {value} corresponding to BIDS {key} in dicom header." + ) + + if dicom_field and value in regex_cases: + # if it exists get rid of it, we don't want no part of it. + if sidecar_json.get("ReconMethodName", None): + json_updater.remove("ReconstructionMethod") + if dicom_header.get("ReconstructionMethod", None): + reconstruction_method = dicom_header.ReconstructionMethod + json_updater.remove("ReconstructionMethod") + reconstruction_method = helper_functions.get_recon_method( + reconstruction_method + ) + + json_updater.update(reconstruction_method) + + elif dicom_field: + # update json + json_updater.update({key: dicom_field}) + + # Additional Heuristics are included below + + # See if time zero is missing in json or additional args + if missing_values.get("TimeZero", None): + if ( + missing_values.get("TimeZero")["key"] is False + or missing_values.get("TimeZero")["value"] is False + ): + time_parser = parser + if sidecar_json.get("AcquisitionTime", None): + acquisition_time = ( + time_parser.parse(sidecar_json.get("AcquisitionTime")) + .time() + .strftime("%H:%M:%S") + ) + else: + acquisition_time = ( + time_parser.parse(dicom_header["SeriesTime"].value) + .time() + .strftime("%H:%M:%S") + ) + + json_updater.update({"TimeZero": acquisition_time}) + json_updater.remove("AcquisitionTime") + json_updater.update({"ScanStart": 0}) + else: + pass + + if missing_values.get("ScanStart", None): + if ( + missing_values.get("ScanStart")["key"] is False + or missing_values.get("ScanStart")["value"] is False + ): + json_updater.update({"ScanStart": 0}) + if missing_values.get("InjectionStart", None): + if ( + missing_values.get("InjectionStart")["key"] is False + or missing_values.get("InjectionStart")["value"] is False + ): + json_updater.update({"InjectionStart": 0}) + + # check to see if units are BQML + json_updater = JsonMAJ(str(path_to_json), bids_null=True) + if json_updater.get("Units") == "BQML": + json_updater.update({"Units": "Bq/mL"}) + + # Add radionuclide to json + Radionuclide = get_radionuclide(dicom_header) + if Radionuclide: + json_updater.update({"TracerRadionuclide": Radionuclide}) + + # remove scandate if it exists + json_updater.remove("ScanDate") + + # after updating raise warnings to user if values in json don't match values in dicom headers, only warn! + updated_values = json.load(open(path_to_json, "r")) + for key, value in paired_fields.items(): + try: + json_field = updated_values.get(key) + dicom_field = dicom_header.__getattr__(key) + if json_field != dicom_field: + logger.info( + f"WARNING!!!! JSON Field {key} with value {json_field} does not match dicom value " + f"of {dicom_field}" + ) + except AttributeError: + pass + + +def update_json_with_dicom_value_cli(): + """ + Command line interface for update_json_with_dicom_value, updates a PET json with values from a dicom header, + optionally can update with values from a spreadsheet or via values passed in as additional arguments with the -k + --additional_arguments flag. This command be accessed after installation of pypet2bids via + `updatepetjsonfromdicom`. + """ + dicom_update_parser = argparse.ArgumentParser( + description="Updates a PET json with values from a dicom header " + "optionally can update with values from a spreadsheet or " + "via values passed in as additional arguments" + ) + dicom_update_parser.add_argument( + "-j", "--json", help="path to json to update", required=True + ) + dicom_update_parser.add_argument( + "-d", "--dicom", help="path to dicom to extract values from", required=True + ) + dicom_update_parser.add_argument( + "-m", "--metadata-path", help="path to metadata json", default=None + ) + dicom_update_parser.add_argument( + "-k", + "--additional_arguments", + help="additional key value pairs to update json with", + nargs="*", + action=helper_functions.ParseKwargs, + default={}, + ) + + args = dicom_update_parser.parse_args() + + try: + # get missing values + missing_values = check_json(args.json, silent=True) + except FileNotFoundError: + with open(args.json, "w") as outfile: + json.dump({}, outfile) + missing_values = check_json(args.json, silent=True) + + # load dicom header + dicom_header = pydicom.dcmread(args.dicom, stop_before_pixels=True) + + if args.metadata_path: + spreadsheet_metadata = get_metadata_from_spreadsheet( + args.metadata_path, args.dicom, dicom_header, **args.additional_arguments + ) + # update json + update_json_with_dicom_value( + args.json, + missing_values, + dicom_header, + silent=True, + spreadsheet_metadata=spreadsheet_metadata["nifti_json"], + **args.additional_arguments, + ) + + JsonMAJ(args.json, update_values=spreadsheet_metadata["nifti_json"]).update() + else: + update_json_with_dicom_value( + args.json, + missing_values, + dicom_header, + silent=True, + **args.additional_arguments, + ) + + JsonMAJ(args.json, update_values=args.additional_arguments).update() + + # check json again after updating + check_json( + args.json, + required=True, + recommended=True, + silent=False, + logger_name="check_json", + ) + + +def update_json_cli(): + """ + Updates a json file with user supplied values or values from a spreadsheet. This command can be accessed after + conversion via `updatepetjson` if so required. + """ + update_json_parser = argparse.ArgumentParser( + description="Updates a json file with user supplied values or values " + "from a spreadsheet. This command can be accessed after " + "conversion via `updatepetjson` if so required. Primarily" + "intended to 'touch up' or create sidecar json files." + ) + update_json_parser.add_argument( + "-j", "--json", help="path to json to update", required=True + ) + update_json_parser.add_argument( + "-k", + "--additional_arguments", + help="additional key value pairs to update json with", + nargs="*", + action=helper_functions.ParseKwargs, + default={}, + ) + update_json_parser.add_argument( + "-m", "--metadata-path", help="path to metadata json", default=None + ) + + update_json_args = update_json_parser.parse_args() + + if update_json_args.metadata_path: + nifti_sidecar_metadata = ( + get_metadata_from_spreadsheet( + update_json_args.metadata_path, update_json_args.json + ) + )["nifti_json"] + else: + nifti_sidecar_metadata = {} + + j = JsonMAJ(update_json_args.json, update_values=nifti_sidecar_metadata) + j.update() + j.update(update_json_args.additional_arguments) + + # check meta radio inputs + j.update(check_meta_radio_inputs(j.json_data)) + + # check json again after updating + check_json( + update_json_args.json, + required=True, + recommended=True, + silent=False, + logger_name="check_json", + ) + + +def get_radionuclide(pydicom_dicom): + """ + Gets the radionuclide if given a pydicom_object if + pydicom_object.RadiopharmaceuticalInformationSequence[0].RadionuclideCodeSequence exists + + :param pydicom_dicom: dicom object collected by pydicom.dcmread("dicom_file.img") + :return: Labeled Radionuclide e.g. 11Carbon, 18Flourine + """ + radionuclide = "" + try: + radiopharmaceutical_information_sequence = ( + pydicom_dicom.RadiopharmaceuticalInformationSequence + ) + radionuclide_code_sequence = radiopharmaceutical_information_sequence[ + 0 + ].RadionuclideCodeSequence + code_value = radionuclide_code_sequence[0].CodeValue + code_meaning = radionuclide_code_sequence[0].CodeMeaning + extraction_good = True + except AttributeError: + logger.info( + "Unable to extract RadioNuclideCodeSequence from RadiopharmaceuticalInformationSequence" + ) + extraction_good = False + + if extraction_good: + # check to see if these nucleotides appear in our verified values + verified_nucleotides = metadata_dictionaries["dicom2bids.json"][ + "RadionuclideCodes" + ] + + check_code_value = "" + check_code_meaning = "" + + if code_value in verified_nucleotides.keys(): + check_code_value = code_value + else: + logger.warning( + f"Radionuclide Code {code_value} does not match any known codes in dcm2bids.json\n" + f"will attempt to infer from code meaning {code_meaning}" + ) + + if code_meaning in verified_nucleotides.values(): + radionuclide = re.sub(r"\^", "", code_meaning) + check_code_meaning = code_meaning + else: + logger.warning( + f"Radionuclide Meaning {code_meaning} not in known values in dcm2bids json" + ) + if code_value in verified_nucleotides.keys(): + radionuclide = re.sub(r"\^", "", verified_nucleotides[code_value]) + + # final check + if check_code_meaning and check_code_value: + pass + else: + logger.warning( + f"WARNING!!!! Possible mismatch between nuclide code meaning {code_meaning} and {code_value} in dicom " + f"header" + ) + + return radionuclide + + +def check_meta_radio_inputs(kwargs: dict, logger="pypet2bids") -> dict: + """ + Routine to check input consistency, possibly generate new ones from PET + BIDS metadata - this only makes sense if you respect the input units as + indicated + + e.g. arguments in are provided via the following params (key/value pairs) + - 'InjectedRadioctivity',81.24 + - 'SpecificRadioactivity',1.3019e+04 + + Units are transformed as follows: + + InjectedRadioactivity: in MBq + InjectedMass: in ug + SpecificRadioactivity: in Bq/g or MBq/ug + MolarActivity: in GBq/umol + MolecularWeight: in g/mol + + :param kwargs: metadata key pair's to examine + :type kwargs: dict + :return: fitted/massaged metadata, return type is an update on input `kwargs` + :rtype: dict + """ + + logger = helper_functions.logger(logger) + + InjectedRadioactivity = kwargs.get("InjectedRadioactivity", None) + InjectedMass = kwargs.get("InjectedMass", None) + SpecificRadioactivity = kwargs.get("SpecificRadioactivity", None) + MolarActivity = kwargs.get("MolarActivity", None) + MolecularWeight = kwargs.get("MolecularWeight", None) + + data_out = {} + + if InjectedRadioactivity and InjectedMass: + data_out["InjectedRadioactivity"] = InjectedRadioactivity + data_out["InjectedRadioactivityUnits"] = "MBq" + data_out["InjectedMass"] = InjectedMass + data_out["InjectedMassUnits"] = "ug" + # check for strings where there shouldn't be strings + numeric_check = [ + helper_functions.is_numeric(str(InjectedRadioactivity)), + helper_functions.is_numeric(str(InjectedMass)), + ] + if False in numeric_check: + data_out["InjectedMass"] = "n/a" + data_out["InjectedMassUnits"] = "n/a" + else: + tmp = (InjectedRadioactivity * 10**6) / (InjectedMass * 10**6) + if SpecificRadioactivity: + if SpecificRadioactivity != tmp: + logger.warning( + "Inferred SpecificRadioactivity in Bq/g doesn't match InjectedRadioactivity " + "and InjectedMass, could be a unit issue" + ) + data_out["SpecificRadioactivity"] = SpecificRadioactivity + data_out["SpecificRadioactivityUnits"] = kwargs.get( + "SpecificRadioactivityUnityUnits", "n/a" + ) + else: + data_out["SpecificRadioactivity"] = tmp + data_out["SpecificRadioactivityUnits"] = "Bq/g" + + if InjectedRadioactivity and SpecificRadioactivity: + data_out["InjectedRadioactivity"] = InjectedRadioactivity + data_out["InjectedRadioactivityUnits"] = "MBq" + data_out["SpecificRadioactivity"] = SpecificRadioactivity + data_out["SpecificRadioactivityUnits"] = "Bq/g" + numeric_check = [ + helper_functions.is_numeric(str(InjectedRadioactivity)), + helper_functions.is_numeric(str(SpecificRadioactivity)), + ] + if False in numeric_check: + data_out["InjectedMass"] = "n/a" + data_out["InjectedMassUnits"] = "n/a" + else: + tmp = (InjectedRadioactivity * (10**6) / SpecificRadioactivity) * (10**6) + if InjectedMass: + if InjectedMass != tmp: + logger.warning( + "Inferred InjectedMass in ug doesn't match InjectedRadioactivity and " + "InjectedMass, could be a unit issue" + ) + data_out["InjectedMass"] = InjectedMass + data_out["InjectedMassUnits"] = kwargs.get("InjectedMassUnits", "n/a") + else: + data_out["InjectedMass"] = tmp + data_out["InjectedMassUnits"] = "ug" + + if InjectedMass and SpecificRadioactivity: + data_out["InjectedMass"] = InjectedMass + data_out["InjectedMassUnits"] = "ug" + data_out["SpecificRadioactivity"] = SpecificRadioactivity + data_out["SpecificRadioactivityUnits"] = "Bq/g" + numeric_check = [ + helper_functions.is_numeric(str(SpecificRadioactivity)), + helper_functions.is_numeric(str(InjectedMass)), + ] + if False in numeric_check: + data_out["InjectedRadioactivity"] = "n/a" + data_out["InjectedRadioactivityUnits"] = "n/a" + else: + tmp = ((InjectedMass / (10**6)) * SpecificRadioactivity) / ( + 10**6 + ) # ((ug / 10 ^ 6) / Bq / g)/10 ^ 6 = MBq + if InjectedRadioactivity: + if InjectedRadioactivity != tmp: + logger.warning( + "Inferred InjectedRadioactivity in MBq doesn't match SpecificRadioactivity " + "and InjectedMass, could be a unit issue" + ) + data_out["InjectedRadioactivity"] = InjectedRadioactivity + data_out["InjectedRadioactivityUnits"] = kwargs.get( + "InjectedRadioactivityUnits", "n/a" + ) + else: + data_out["InjectedRadioactivity"] = tmp + data_out["InjectedRadioactivityUnits"] = "MBq" + + if MolarActivity and MolecularWeight: + data_out["MolarActivity"] = MolarActivity + data_out["MolarActivityUnits"] = "GBq/umol" + data_out["MolecularWeight"] = MolecularWeight + data_out["MolecularWeightUnits"] = "g/mol" + numeric_check = [ + helper_functions.is_numeric(str(MolarActivity)), + helper_functions.is_numeric(str(MolecularWeight)), + ] + if False in numeric_check: + data_out["SpecificRadioactivity"] = "n/a" + data_out["SpecificRadioactivityUnits"] = "n/a" + else: + tmp = ( + MolarActivity * (10**3) + ) / MolecularWeight # (GBq / umol * 10 ^ 6) / (g / mol / * 10 ^ 6) = Bq / g + if SpecificRadioactivity: + if SpecificRadioactivity != tmp: + logger.warning( + "Inferred SpecificRadioactivity in MBq/ug doesn't match Molar Activity and Molecular " + "Weight, could be a unit issue" + ) + data_out["SpecificRadioactivity"] = SpecificRadioactivity + data_out["SpecificRadioactivityUnits"] = kwargs.get( + "SpecificRadioactivityUnityUnits", "n/a" + ) + else: + data_out["SpecificRadioactivity"] = tmp + data_out["SpecificRadioactivityUnits"] = "Bq/g" + + if MolarActivity and SpecificRadioactivity: + data_out["SpecificRadioactivity"] = SpecificRadioactivity + data_out["SpecificRadioactivityUnits"] = "MBq/ug" + data_out["MolarActivity"] = MolarActivity + data_out["MolarActivityUnits"] = "GBq/umol" + numeric_check = [ + helper_functions.is_numeric(str(SpecificRadioactivity)), + helper_functions.is_numeric(str(MolarActivity)), + ] + if False in numeric_check: + data_out["MolecularWeight"] = "n/a" + data_out["MolecularWeightUnits"] = "n/a" + else: + tmp = ( + MolarActivity * 1000 + ) / SpecificRadioactivity # (MBq / ug / 1000) / (GBq / umol) = g / mol + if MolecularWeight: + if MolecularWeight != tmp: + logger.warning( + "Inferred MolecularWeight in MBq/ug doesn't match Molar Activity and " + "Molecular Weight, could be a unit issue" + ) + + data_out["MolecularWeight"] = tmp + data_out["MolecularWeightUnits"] = kwargs.get( + "MolecularWeightUnits", "n/a" + ) + else: + data_out["MolecularWeight"] = tmp + data_out["MolecularWeightUnits"] = "g/mol" + + if MolecularWeight and SpecificRadioactivity: + data_out["SpecificRadioactivity"] = SpecificRadioactivity + data_out["SpecificRadioactivityUnits"] = "MBq/ug" + data_out["MolecularWeight"] = MolarActivity + data_out["MolecularWeightUnits"] = "g/mol" + numeric_check = [ + helper_functions.is_numeric(str(SpecificRadioactivity)), + helper_functions.is_numeric(str(MolecularWeight)), + ] + if False in numeric_check: + data_out["MolarActivity"] = "n/a" + data_out["MolarActivityUnits"] = "n/a" + else: + tmp = MolecularWeight * ( + SpecificRadioactivity / 1000 + ) # g / mol * (MBq / ug / 1000) = GBq / umol + if MolarActivity: + if MolarActivity != tmp: + logger.warning( + "Inferred MolarActivity in GBq/umol doesn't match Specific Radioactivity and " + "Molecular Weight, could be a unit issue" + ) + data_out["MolarActivity"] = MolarActivity + data_out["MolarActivityUnits"] = kwargs.get("MolarActivityUnits", "n/a") + else: + data_out["MolarActivity"] = tmp + data_out["MolarActivityUnits"] = "GBq/umol" + + return data_out + + +def get_metadata_from_spreadsheet( + metadata_path: Union[str, Path], + image_folder, + image_header_dict={}, + **additional_arguments, +) -> dict: + """ + Extracts metadata from a spreadsheet and returns a dictionary of metadata organized under + three main keys: nifti_json, blood_json, and blood_tsv + + :param metadata_path: path to a spreadsheet + :type metadata_path: [str, pathlib.Path] + :param image_folder: path to image folder + :type image_folder: [str, pathlib.Path] + :param image_header_dict: dictionary of image header values + :type image_header_dict: dict + :param additional_arguments: additional arguments to pass on, typically user sourced key value pairs + :type additional_arguments: dict + :return: dictionary of metadata + :rtype: dict + """ + spreadsheet_metadata = {"nifti_json": {}, "blood_json": {}, "blood_tsv": {}} + spreadsheet_values = {} + if Path(metadata_path).is_file(): + spreadsheet_values = helper_functions.single_spreadsheet_reader( + path_to_spreadsheet=metadata_path, + dicom_metadata=image_header_dict, + **additional_arguments, + ) + + # remove any dates from the spreadsheet time values + for key, value in spreadsheet_values.items(): + if "time" in key.lower(): + if isinstance(value, str): + # check to see if the value converts to a datetime object with a date + try: + time_value = parser.parse(value).time().strftime("%H:%M:%S") + spreadsheet_values[key] = time_value + except ValueError: + pass + if isinstance(value, datetime.datetime): + spreadsheet_values[key] = value.time().strftime("%H:%M:%S") + + if Path(metadata_path).is_dir() or metadata_path == "": + # we accept folder input as well as no input, in the + # event of no input we search for spreadsheets in the + # image folder + if metadata_path == "": + metadata_path = image_folder + + spreadsheets = helper_functions.collect_spreadsheets(metadata_path) + pet_spreadsheets = [ + spreadsheet for spreadsheet in spreadsheets if is_pet.pet_file(spreadsheet) + ] + spread_sheet_values = {} + + for pet_spreadsheet in pet_spreadsheets: + spreadsheet_values.update( + helper_functions.single_spreadsheet_reader( + path_to_spreadsheet=pet_spreadsheet, + dicom_metadata=image_header_dict, + **additional_arguments, + ) + ) + + # check for any blood (tsv) data or otherwise in the given spreadsheet values + blood_tsv_columns = [ + "time", + "plasma_radioactivity", + "metabolite_parent_fraction", + "whole_blood_radioactivity", + ] + blood_json_columns = [ + "PlasmaAvail", + "WholeBloodAvail", + "MetaboliteAvail", + "MetaboliteMethod", + "MetaboliteRecoveryCorrectionApplied", + "DispersionCorrected", + ] + + # check for existing tsv columns + for column in blood_tsv_columns: + try: + values = spreadsheet_values[column] + spreadsheet_metadata["blood_tsv"][column] = values + # pop found data from spreadsheet values after it's been found + spreadsheet_values.pop(column) + except KeyError: + pass + + # even out the values in the blood tsv columns if they're different lengths by appending zeros to the end + # of each column/list + # determine the longest column + column_lengths = [ + len(column) for column in spreadsheet_metadata["blood_tsv"].values() + ] + + try: + longest_column = max(column_lengths) + except ValueError: + # columns are all the same length or there are no columns + longest_column = None + if longest_column: + # iterate over each column, determine how many zeros to append to the end of each column + for column in spreadsheet_metadata["blood_tsv"].keys(): + zeros_to_append = longest_column - len( + spreadsheet_metadata["blood_tsv"][column] + ) + spreadsheet_metadata["blood_tsv"][column] += [0] * zeros_to_append + + # check for existing blood json values + for column in blood_json_columns: + try: + values = spreadsheet_values[column] + spreadsheet_metadata["blood_json"][column] = values + # pop found data from spreadsheet values after it's been found + spreadsheet_values.pop(column) + except KeyError: + pass + + if not spreadsheet_metadata.get("nifti_json", None): + spreadsheet_metadata["nifti_json"] = {} + spreadsheet_metadata["nifti_json"].update(spreadsheet_values) + + return spreadsheet_metadata + + +if __name__ == "__main__": + update_json_cli() diff --git a/pypet2bids/pypet2bids/write_ecat.py b/pypet2bids/pypet2bids/write_ecat.py index 33b7f387..7c3db225 100644 --- a/pypet2bids/pypet2bids/write_ecat.py +++ b/pypet2bids/pypet2bids/write_ecat.py @@ -29,11 +29,18 @@ import struct from math import ceil, floor -from pypet2bids.read_ecat import ecat_header_maps, get_buffer_size +from pypet2bids.read_ecat import ecat_header_maps, get_buffer_size import numpy from pathlib import Path -def write_header(ecat_file, schema: dict, values: dict = {}, byte_position: int = 0, seek: bool = False): + +def write_header( + ecat_file, + schema: dict, + values: dict = {}, + byte_position: int = 0, + seek: bool = False, +): """ Given a filepath and a schema this function will write an ecat header to a file. If supplied a dictionary of values it will populate the written header with them, if not it will fill the header with empty values. If a byte position @@ -52,14 +59,17 @@ def write_header(ecat_file, schema: dict, values: dict = {}, byte_position: int byte_position = ecat_file.tell() for entry in schema: - byte_position, variable_name, struct_fmt = entry['byte'], entry['variable_name'], entry[ - 'struct'] - struct_fmt = '>' + struct_fmt + byte_position, variable_name, struct_fmt = ( + entry["byte"], + entry["variable_name"], + entry["struct"], + ) + struct_fmt = ">" + struct_fmt byte_width = struct.calcsize(struct_fmt) value_to_write = values.get(variable_name, None) # if no value is supplied in the value dict, pack with empty bytes as well - if not value_to_write: + if value_to_write is None: pack_empty = True # set variable to false if neither of these conditions is met else: @@ -67,10 +77,10 @@ def write_header(ecat_file, schema: dict, values: dict = {}, byte_position: int # for fill or empty values supplied in the values dictionary if pack_empty: - fill = byte_width * 'x' + fill = byte_width * "x" ecat_file.write(struct.pack(fill)) - elif 's' in struct_fmt: - value_to_write = bytes(value_to_write, 'ascii') + elif "s" in struct_fmt: + value_to_write = bytes(value_to_write, "ascii") ecat_file.write(struct.pack(struct_fmt, value_to_write)) # for all other cases else: @@ -81,20 +91,26 @@ def write_header(ecat_file, schema: dict, values: dict = {}, byte_position: int try: ecat_file.write(struct.pack(struct_fmt, *value_to_write)) except struct.error as err: - if values['DATA_TYPE'] == 5 and 'MAX' in variable_name: + if values["DATA_TYPE"] == 5 and "MAX" in variable_name: value_to_write = 32767 - elif values['DATA_TYPE'] == 5 and 'MIN' in variable_name: + elif values["DATA_TYPE"] == 5 and "MIN" in variable_name: value_to_write = -32767 - print('Uncertain how to handle min and max datatypes for float arrays when writing ecats.\n' - 'if you know more about what header min and max values should be in the case of an float32\n' - 'image matrix please consider making a pull request to this library or posting an issue.') + print( + "Uncertain how to handle min and max datatypes for float arrays when writing ecats.\n" + "if you know more about what header min and max values should be in the case of an float32\n" + "image matrix please consider making a pull request to this library or posting an issue." + ) else: - print(f"Oh no {value_to_write} is out of range for {struct_fmt}, variable {variable_name}") + print( + f"Oh no {value_to_write} is out of range for {struct_fmt}, variable {variable_name}" + ) raise err return byte_width + byte_position -def create_directory_table(num_frames: int = 0, pixel_dimensions: dict = {}, pixel_byte_size: int = 2): +def create_directory_table( + num_frames: int = 0, pixel_dimensions: dict = {}, pixel_byte_size: int = 2 +): """ Creates directory tables for an ecat file when provided with the number of frames, the total number of pixels per frame, and the byte size of each of those pixels. Ecat's can have int16 (2 byte widths) or float 32 (4 byte @@ -124,24 +140,28 @@ def create_directory_table(num_frames: int = 0, pixel_dimensions: dict = {}, pix # why is the initial table byte position equal to 3? Because the first header and pixel data lies # after the main header byte block at position 0 to 1 and the directory table itself occupies byte blocks # 1 to 2, thus the first frame (header and pixel data) will land at byte block number 3! Whoopee! - table_byte_position = 3 #1 + required_directory_blocks + table_byte_position = 3 # 1 + required_directory_blocks for i in range(required_directory_blocks): # initialize empty 4 x 32 array - table = numpy.ndarray((4, 32), dtype='>i4') + table = numpy.ndarray((4, 32), dtype=">i4") # populate first column of table with codes and undetermined codes # note these table values are only extrapolated from a data set w/ 45 frames and datasets with less than # 31 frames. Behavior or values for frames numbering above 45 (or more specifically 62) is unknown. if i == (required_directory_blocks - 1): frames_to_iterate = num_frames % 31 - table[0, 0] = 31 - frames_to_iterate # number of un-used columns in the directory table + table[0, 0] = ( + 31 - frames_to_iterate + ) # number of un-used columns in the directory table table[1, 0] = 2 # stop value table[2, 0] = 2 # stop value table[3, 0] = frames_to_iterate # number of frames referenced in this table else: frames_to_iterate = floor(num_frames / 31) * 31 table[0, 0] = 0 - table[1, 0] = 0 # note this is just a placeholder the real value is entered after the final position of the + table[1, 0] = ( + 0 # note this is just a placeholder the real value is entered after the final position of the + ) # last entry in the first directory is calculated table[2, 0] = 0 table[3, 0] = frames_to_iterate @@ -154,7 +174,9 @@ def create_directory_table(num_frames: int = 0, pixel_dimensions: dict = {}, pix table[0, column + 1] = directory_order.pop(0) table[1, column + 1] = table_byte_position # frame byte position is shifted - table_byte_position = int((pixel_byte_size * pixel_volume) / 512 + table_byte_position) + table_byte_position = int( + (pixel_byte_size * pixel_volume) / 512 + table_byte_position + ) table[2, column + 1] = table_byte_position table[3, column + 1] = 1 @@ -196,16 +218,18 @@ def write_directory_table(file, directory_tables: list, seek: bool = False): flattened_transpose = numpy.reshape(transpose_table, (4, -1)).flatten() for index in range(flattened_transpose.size): - file.write(struct.pack('>l', flattened_transpose[index])) + file.write(struct.pack(">l", flattened_transpose[index])) return file.tell() -def write_pixel_data(ecat_file, pixel_data: numpy.ndarray, byte_position: int=None, seek: bool=None): +def write_pixel_data( + ecat_file, pixel_data: numpy.ndarray, byte_position: int = None, seek: bool = None +): if seek and byte_position: ecat_file.seek(byte_position) - #elif (seek and not byte_position) or (byte_position and not seek): - #raise Exception("Must provide seek boolean and byte position") + # elif (seek and not byte_position) or (byte_position and not seek): + # raise Exception("Must provide seek boolean and byte position") else: pass flattened_pixels = pixel_data.flatten().tobytes() @@ -213,48 +237,54 @@ def write_pixel_data(ecat_file, pixel_data: numpy.ndarray, byte_position: int=No return 0 -def write_ecat(ecat_file: Path, - mainheader_schema: dict, - mainheader_values: dict, - subheaders_values: list, - subheader_schema: dict, - number_of_frames: int, - pixel_x_dimension: int, - pixel_y_dimension: int, - pixel_z_dimension: int, - pixel_byte_size: int, - pixel_data: list=[]): +def write_ecat( + ecat_file: Path, + mainheader_schema: dict, + mainheader_values: dict, + subheaders_values: list, + subheader_schema: dict, + number_of_frames: int, + pixel_x_dimension: int, + pixel_y_dimension: int, + pixel_z_dimension: int, + pixel_byte_size: int, + pixel_data: list = [], +): # open the ecat file! - with open(ecat_file, 'w+b') as outfile: + with open(ecat_file, "w+b") as outfile: # first things first, write the main header with supplied information - write_header(ecat_file=outfile, - schema=mainheader_schema, - values=mainheader_values) + write_header( + ecat_file=outfile, schema=mainheader_schema, values=mainheader_values + ) position_post_header_write = outfile.tell() # create the directory table - directory_table = create_directory_table(num_frames=number_of_frames, - pixel_dimensions={'x':pixel_x_dimension, - 'y':pixel_y_dimension, - 'z':pixel_z_dimension}, - pixel_byte_size=pixel_byte_size) + directory_table = create_directory_table( + num_frames=number_of_frames, + pixel_dimensions={ + "x": pixel_x_dimension, + "y": pixel_y_dimension, + "z": pixel_z_dimension, + }, + pixel_byte_size=pixel_byte_size, + ) # write the directory tabel to the file - write_directory_table(file=outfile, - directory_tables=directory_table) + write_directory_table(file=outfile, directory_tables=directory_table) position_post_table_write = outfile.tell() # write subheaders followed by pixel data for index, subheader in enumerate(subheaders_values): position = outfile.tell() table_position = directory_table[0][1, index + 1] * 512 - write_header(ecat_file=outfile, - schema=subheader_schema, - values=subheader, - byte_position=outfile.tell()) + write_header( + ecat_file=outfile, + schema=subheader_schema, + values=subheader, + byte_position=outfile.tell(), + ) position_post_subheader_write = outfile.tell() - write_pixel_data(ecat_file=outfile, - pixel_data=pixel_data[index]) + write_pixel_data(ecat_file=outfile, pixel_data=pixel_data[index]) position_post_subheader_pixel_data_write = outfile.tell() - return ecat_file \ No newline at end of file + return ecat_file diff --git a/pypet2bids/pyproject.toml b/pypet2bids/pyproject.toml index 8d4c758e..bc8e236f 100644 --- a/pypet2bids/pyproject.toml +++ b/pypet2bids/pyproject.toml @@ -1,16 +1,20 @@ [tool.poetry] name = "pypet2bids" -version = "1.2.10" -description = "A python implementation of an ECAT to BIDS converter." +version = "1.3.12" +description = "A python library for converting PET imaging and blood data to BIDS." authors = ["anthony galassi <28850131+bendhouseart@users.noreply.github.com>"] license = "MIT" include = [ 'pypet2bids/metadata/*', 'pypet2bids/pyproject.toml', + 'pypet2bids/README.md', ] +documentation = "https://pypet2bids.readthedocs.io/en/latest/" +repository = "https://github.com/openneuropet/pet2bids/" +readme = "README.md" [tool.poetry.dependencies] -python = ">=3.8,<=3.11" +python = ">=3.8,<=3.12" nibabel = ">=3.2.1" numpy = "^1.21.3" pyparsing = "^3.0.4" @@ -18,11 +22,9 @@ python-dateutil = "^2.8.2" python-dotenv = "^0.19.1" scipy = "^1.7.1" six = "^1.16.0" -pytest = ">=6.2.5, <8.0.0" pydicom = "^2.2.2" openpyxl = "^3.0.9" xlrd = "^2.0.1" -termcolor = "^1.1.0" json-maj = "^0.0.8" pandas = "^1.4.4" pyxlsb = "^1.0.9" @@ -31,9 +33,10 @@ toml = ">=0.10.2" [tool.poetry.dev-dependencies] +pytest = ">=6.2.5, <8.0.0" sphinx = "<=4.5.0" sphinx-rtd-theme = "^1.0.0" -sphinxcontrib-matlabdomain = "^0.13.0" +sphinxcontrib-matlabdomain = "^0.21.4" [tool.poetry.scripts] ecatpet2bids = 'pypet2bids.ecat_cli:main' @@ -42,6 +45,10 @@ dcm2niix4pet = 'pypet2bids.dcm2niix4pet:main' pet2bids-spreadsheet-template = 'pypet2bids.helper_functions:write_out_module' convert-pmod-to-blood = 'pypet2bids.convert_pmod_to_blood:main' ispet = 'pypet2bids.is_pet:main' +updatepetjsonfromdicom = 'pypet2bids.dcm2niix4pet:update_json_with_dicom_value_cli' +updatepetjsonfromecat = 'pypet2bids.ecat_cli:update_json_with_ecat_value_cli' +updatepetjson = 'pypet2bids.update_json:update_json_cli' +ecatheaderupdate = 'pypet2bids.ecat_header_update:main' [tool.poetry.group.dev.dependencies] pyinstaller = "^5.4.1" diff --git a/pypet2bids/tests/metadata_excel_example_reader.py b/pypet2bids/tests/metadata_excel_example_reader.py index e4423d00..3046f0af 100644 --- a/pypet2bids/tests/metadata_excel_example_reader.py +++ b/pypet2bids/tests/metadata_excel_example_reader.py @@ -17,42 +17,42 @@ def flatten_series(series): elif len(simplified_series_object) == 1: simplified_series_object = simplified_series_object[0] else: - raise(f"Invalid Series: {series}") + raise (f"Invalid Series: {series}") return simplified_series_object def translate_metadata(metadata_dataframe, image_path=NotImplemented): nifti_json = { - 'Manufacturer': '', - 'ManufacturersModelName': '', - 'Units': '', - 'TracerName': '', - 'TracerRadionuclide': '', - 'InjectedRadioactivity': 0, - 'InjectedRadioactivityUnits': '', - 'InjectedMass': 0, - 'InjectedMassUnits': '', - 'SpecificRadioactivity': 0, - 'SpecificRadioactivityUnits': '', - 'ModeOfAdministration': '', - 'TimeZero': 0, - 'ScanStart': 0, - 'InjectionStart': 0, - 'FrameTimesStart': [], - 'FrameDuration': [], - 'AcquisitionMode': '', - 'ImageDecayCorrected': '', - 'ImageDecayCorrectionTime': 0, - 'ReconMethodName': '', - 'ReconMethodParameterLabels': [], - 'ReconMethodParameterUnits': [], - 'ReconMethodParameterValues': [], - 'ReconFilterType': '', - 'ReconFilterSize': 0, - 'AttenuationCorrection': '', - 'InstitutionName': '', - 'InstitutionalDepartmentName': '' + "Manufacturer": "", + "ManufacturersModelName": "", + "Units": "", + "TracerName": "", + "TracerRadionuclide": "", + "InjectedRadioactivity": 0, + "InjectedRadioactivityUnits": "", + "InjectedMass": 0, + "InjectedMassUnits": "", + "SpecificRadioactivity": 0, + "SpecificRadioactivityUnits": "", + "ModeOfAdministration": "", + "TimeZero": 0, + "ScanStart": 0, + "InjectionStart": 0, + "FrameTimesStart": [], + "FrameDuration": [], + "AcquisitionMode": "", + "ImageDecayCorrected": "", + "ImageDecayCorrectionTime": 0, + "ReconMethodName": "", + "ReconMethodParameterLabels": [], + "ReconMethodParameterUnits": [], + "ReconMethodParameterValues": [], + "ReconFilterType": "", + "ReconFilterSize": 0, + "AttenuationCorrection": "", + "InstitutionName": "", + "InstitutionalDepartmentName": "", } for key in nifti_json.keys(): @@ -61,12 +61,8 @@ def translate_metadata(metadata_dataframe, image_path=NotImplemented): except KeyError: warnings.warn(f"{key} not found in metadata extracted from spreadsheet") - blood_json = { + blood_json = {} - } - - blood_tsv = { - - } + blood_tsv = {} - return {'nifti_json': nifti_json, 'blood_json': blood_json, 'blood_tsv': blood_tsv} + return {"nifti_json": nifti_json, "blood_json": blood_json, "blood_tsv": blood_tsv} diff --git a/pypet2bids/tests/test_convert_pmod_to_blood.py b/pypet2bids/tests/test_convert_pmod_to_blood.py index f9926b61..6ed46de8 100644 --- a/pypet2bids/tests/test_convert_pmod_to_blood.py +++ b/pypet2bids/tests/test_convert_pmod_to_blood.py @@ -15,20 +15,20 @@ def test_type_cast_cli_input(): - assert True == type_cast_cli_input('true') - assert True == type_cast_cli_input('t') - assert True == type_cast_cli_input('True') - assert False == type_cast_cli_input('f') - assert False == type_cast_cli_input('false') - assert False == type_cast_cli_input('False') - assert [1, 2, 3] == type_cast_cli_input('[1, 2, 3]') - assert [1.0, 2, 3] == type_cast_cli_input('[1.0, 2, 3]') - assert [1.0, 2.0, 3.0] == type_cast_cli_input('[1.0, 2.0, 3.0]') - assert ['a', 'b'] == type_cast_cli_input("['a', 'b']") - assert {'a': 'b'} == type_cast_cli_input("{'a': 'b'}") - assert 1 == type_cast_cli_input('1') - assert 1.0 == type_cast_cli_input('1.0') - assert 'string' == type_cast_cli_input('string') + assert True == type_cast_cli_input("true") + assert True == type_cast_cli_input("t") + assert True == type_cast_cli_input("True") + assert False == type_cast_cli_input("f") + assert False == type_cast_cli_input("false") + assert False == type_cast_cli_input("False") + assert [1, 2, 3] == type_cast_cli_input("[1, 2, 3]") + assert [1.0, 2, 3] == type_cast_cli_input("[1.0, 2, 3]") + assert [1.0, 2.0, 3.0] == type_cast_cli_input("[1.0, 2.0, 3.0]") + assert ["a", "b"] == type_cast_cli_input("['a', 'b']") + assert {"a": "b"} == type_cast_cli_input("{'a': 'b'}") + assert 1 == type_cast_cli_input("1") + assert 1.0 == type_cast_cli_input("1.0") + assert "string" == type_cast_cli_input("string") @pytest.fixture() @@ -40,18 +40,22 @@ def Ex_bld_whole_blood_only_files(): """ pmod_blood_dir = os.path.join( project_dir, - 'spreadsheet_conversion', - 'blood', - 'pmod', - 'Ex_bld_wholeblood_and_plasma_only') - bld_files = \ - [os.path.join(pmod_blood_dir, bld) for bld in os.listdir(pmod_blood_dir) if pathlib.Path(bld).suffix == '.bld'] - blood_files = {'plasma': [], 'whole': []} + "spreadsheet_conversion", + "blood", + "pmod", + "Ex_bld_wholeblood_and_plasma_only", + ) + bld_files = [ + os.path.join(pmod_blood_dir, bld) + for bld in os.listdir(pmod_blood_dir) + if pathlib.Path(bld).suffix == ".bld" + ] + blood_files = {"plasma": [], "whole": []} for index, bld_file in enumerate(bld_files): for key in blood_files.keys(): if key in pathlib.Path(bld_file).name: blood_files[key].append(bld_file) - + yield blood_files @@ -64,22 +68,26 @@ def Ex_bld_manual_and_autosampled_mixed(): """ pmod_blood_dir = os.path.join( project_dir, - 'spreadsheet_conversion', - 'blood', - 'pmod', - 'Ex_bld_manual_and_autosampled_mixed') - bld_files = \ - [os.path.join(pmod_blood_dir, bld) for bld in os.listdir(pmod_blood_dir) if pathlib.Path(bld).suffix == '.bld'] + "spreadsheet_conversion", + "blood", + "pmod", + "Ex_bld_manual_and_autosampled_mixed", + ) + bld_files = [ + os.path.join(pmod_blood_dir, bld) + for bld in os.listdir(pmod_blood_dir) + if pathlib.Path(bld).suffix == ".bld" + ] pop_indexes = [] for f in bld_files: - if 'ratio' in str(f).lower(): + if "ratio" in str(f).lower(): pop_indexes.append(f) for pop in pop_indexes: bld_files.remove(pop) - blood_files = {'plasma': [], 'whole': [], 'parent': []} + blood_files = {"plasma": [], "whole": [], "parent": []} for index, bld_file in enumerate(bld_files): for key in blood_files.keys(): if key in str(pathlib.Path(bld_file).name).lower(): @@ -97,15 +105,19 @@ def Ex_txt_manual_and_autosampled_mixed(): """ pmod_blood_dir = os.path.join( project_dir, - 'spreadsheet_conversion', - 'blood', - 'pmod', - 'Ex_txt_manual_and_autosampled_mixed') - - bld_files = \ - [os.path.join(pmod_blood_dir, bld) for bld in os.listdir(pmod_blood_dir) if pathlib.Path(bld).suffix == '.txt'] - - blood_files = {'plasma': [], 'whole': [], 'parent': []} + "spreadsheet_conversion", + "blood", + "pmod", + "Ex_txt_manual_and_autosampled_mixed", + ) + + bld_files = [ + os.path.join(pmod_blood_dir, bld) + for bld in os.listdir(pmod_blood_dir) + if pathlib.Path(bld).suffix == ".txt" + ] + + blood_files = {"plasma": [], "whole": [], "parent": []} for index, bld_file in enumerate(bld_files): for key in blood_files.keys(): if key in str(pathlib.Path(bld_file).name).lower(): @@ -117,110 +129,174 @@ def Ex_txt_manual_and_autosampled_mixed(): class TestPmodToBlood: def test_load_bld_files_blood_only(self, Ex_bld_whole_blood_only_files): kwargs_input = { - 'whole_blood_activity_collection_method': 'automatic', - 'parent_fraction_collection_method': 'automatic', - 'plasma_activity_collection_method': 'automatic' - } + "whole_blood_activity_collection_method": "automatic", + "parent_fraction_collection_method": "automatic", + "plasma_activity_collection_method": "automatic", + } with tempfile.TemporaryDirectory() as tempdir: pmod_to_blood = PmodToBlood( - whole_blood_activity=pathlib.Path(Ex_bld_whole_blood_only_files['whole'][0]), - plasma_activity=pathlib.Path(Ex_bld_whole_blood_only_files['plasma'][0]), + whole_blood_activity=pathlib.Path( + Ex_bld_whole_blood_only_files["whole"][0] + ), + plasma_activity=pathlib.Path( + Ex_bld_whole_blood_only_files["plasma"][0] + ), output_path=pathlib.Path(tempdir), **kwargs_input ) def test_load_bld_files_mixed(self, Ex_bld_manual_and_autosampled_mixed): kwargs_input = { - 'whole_blood_activity_collection_method': 'automatic', - 'parent_fraction_collection_method': 'manual', - 'plasma_activity_collection_method': 'automatic' + "whole_blood_activity_collection_method": "automatic", + "parent_fraction_collection_method": "manual", + "plasma_activity_collection_method": "automatic", } with tempfile.TemporaryDirectory() as tempdir: pmod_to_blood = PmodToBlood( - whole_blood_activity=pathlib.Path(Ex_bld_manual_and_autosampled_mixed['whole'][0]), - plasma_activity=pathlib.Path(Ex_bld_manual_and_autosampled_mixed['plasma'][0]), - parent_fraction=pathlib.Path(Ex_bld_manual_and_autosampled_mixed['parent'][0]), + whole_blood_activity=pathlib.Path( + Ex_bld_manual_and_autosampled_mixed["whole"][0] + ), + plasma_activity=pathlib.Path( + Ex_bld_manual_and_autosampled_mixed["plasma"][0] + ), + parent_fraction=pathlib.Path( + Ex_bld_manual_and_autosampled_mixed["parent"][0] + ), output_path=tempdir, **kwargs_input ) def test_bld_output_manual_popped_values(self, Ex_bld_manual_and_autosampled_mixed): kwargs_input = { - 'whole_blood_activity_collection_method': 'automatic', - 'parent_fraction_collection_method': 'manual', - 'plasma_activity_collection_method': 'automatic' + "whole_blood_activity_collection_method": "automatic", + "parent_fraction_collection_method": "manual", + "plasma_activity_collection_method": "automatic", } # test bld (pmod files first) with tempfile.TemporaryDirectory() as tempdir: pmod_to_blood = PmodToBlood( - whole_blood_activity=pathlib.Path(Ex_bld_manual_and_autosampled_mixed['whole'][0]), - plasma_activity=pathlib.Path(Ex_bld_manual_and_autosampled_mixed['plasma'][0]), - parent_fraction=pathlib.Path(Ex_bld_manual_and_autosampled_mixed['parent'][0]), + whole_blood_activity=pathlib.Path( + Ex_bld_manual_and_autosampled_mixed["whole"][0] + ), + plasma_activity=pathlib.Path( + Ex_bld_manual_and_autosampled_mixed["plasma"][0] + ), + parent_fraction=pathlib.Path( + Ex_bld_manual_and_autosampled_mixed["parent"][0] + ), output_path=tempdir, **kwargs_input ) - created_files = [pathlib.Path(os.path.join(tempdir, created)) for created in os.listdir(pathlib.Path(tempdir))] - assert len([created for created in created_files if 'automatic' in created.name]) >= 1 - assert len([created for created in created_files if 'manual' in created.name]) >= 1 + created_files = [ + pathlib.Path(os.path.join(tempdir, created)) + for created in os.listdir(pathlib.Path(tempdir)) + ] + assert ( + len( + [ + created + for created in created_files + if "automatic" in created.name + ] + ) + >= 1 + ) + assert ( + len([created for created in created_files if "manual" in created.name]) + >= 1 + ) for f in created_files: - if 'manual' in f.name and f.suffix == '.tsv': - manual_df = pandas.read_csv(f, sep='\t') + if "manual" in f.name and f.suffix == ".tsv": + manual_df = pandas.read_csv(f, sep="\t") - assert 'plasma_radioactivity' in manual_df.columns - assert 'whole_blood_radioactivity' in manual_df.columns + assert "plasma_radioactivity" in manual_df.columns + assert "whole_blood_radioactivity" in manual_df.columns def test_load_txt_files_mixed(self, Ex_txt_manual_and_autosampled_mixed): kwargs_input = { - 'whole_blood_activity_collection_method': 'automatic', - 'parent_fraction_collection_method': 'manual', - 'plasma_activity_collection_method': 'manual' + "whole_blood_activity_collection_method": "automatic", + "parent_fraction_collection_method": "manual", + "plasma_activity_collection_method": "manual", } with tempfile.TemporaryDirectory() as tempdir: pmod_to_blood = PmodToBlood( - whole_blood_activity=pathlib.Path(Ex_txt_manual_and_autosampled_mixed['whole'][0]), - plasma_activity=pathlib.Path(Ex_txt_manual_and_autosampled_mixed['plasma'][0]), - parent_fraction=pathlib.Path(Ex_txt_manual_and_autosampled_mixed['parent'][0]), + whole_blood_activity=pathlib.Path( + Ex_txt_manual_and_autosampled_mixed["whole"][0] + ), + plasma_activity=pathlib.Path( + Ex_txt_manual_and_autosampled_mixed["plasma"][0] + ), + parent_fraction=pathlib.Path( + Ex_txt_manual_and_autosampled_mixed["parent"][0] + ), output_path=tempdir, **kwargs_input ) def test_txt_output_manual_popped_values(self, Ex_txt_manual_and_autosampled_mixed): kwargs_input = { - 'whole_blood_activity_collection_method': 'automatic', - 'parent_fraction_collection_method': 'manual', - 'plasma_activity_collection_method': 'manual' + "whole_blood_activity_collection_method": "automatic", + "parent_fraction_collection_method": "manual", + "plasma_activity_collection_method": "manual", } # next test txt files with tempfile.TemporaryDirectory() as tempdir: pmod_to_blood = PmodToBlood( - whole_blood_activity=pathlib.Path(Ex_txt_manual_and_autosampled_mixed['whole'][0]), - plasma_activity=pathlib.Path(Ex_txt_manual_and_autosampled_mixed['plasma'][0]), - parent_fraction=pathlib.Path(Ex_txt_manual_and_autosampled_mixed['parent'][0]), + whole_blood_activity=pathlib.Path( + Ex_txt_manual_and_autosampled_mixed["whole"][0] + ), + plasma_activity=pathlib.Path( + Ex_txt_manual_and_autosampled_mixed["plasma"][0] + ), + parent_fraction=pathlib.Path( + Ex_txt_manual_and_autosampled_mixed["parent"][0] + ), output_path=tempdir, **kwargs_input ) - created_files = [pathlib.Path(os.path.join(tempdir, created)) for created in os.listdir(pathlib.Path(tempdir))] - assert len([created for created in created_files if 'automatic' in created.name]) >= 1 - assert len([created for created in created_files if 'manual' in created.name]) >= 1 + created_files = [ + pathlib.Path(os.path.join(tempdir, created)) + for created in os.listdir(pathlib.Path(tempdir)) + ] + assert ( + len( + [ + created + for created in created_files + if "automatic" in created.name + ] + ) + >= 1 + ) + assert ( + len([created for created in created_files if "manual" in created.name]) + >= 1 + ) for f in created_files: - if 'manual' in f.name and f.suffix == '.tsv': - manual_df = pandas.read_csv(f, sep='\t') + if "manual" in f.name and f.suffix == ".tsv": + manual_df = pandas.read_csv(f, sep="\t") # we want to calculate plasma radioactivity if we have whole blood and parent fraction, for the # manual blood file at least - assert 'plasma_radioactivity' in manual_df.columns - assert 'whole_blood_radioactivity' in manual_df.columns - assert 'metabolite_parent_fraction' in manual_df.columns - assert len(manual_df['plasma_radioactivity']) == len(manual_df['metabolite_parent_fraction']) - - if 'auto' in f.name and f.suffix == '.tsv': - automatic_df = pandas.read_csv(f, sep='\t') - original_whole_blood = open_meta_data(Ex_txt_manual_and_autosampled_mixed['whole'][0]) - assert 'whole_blood_radioactivity' in automatic_df.columns - assert len(automatic_df) + len(manual_df) == len(original_whole_blood) \ No newline at end of file + assert "plasma_radioactivity" in manual_df.columns + assert "whole_blood_radioactivity" in manual_df.columns + assert "metabolite_parent_fraction" in manual_df.columns + assert len(manual_df["plasma_radioactivity"]) == len( + manual_df["metabolite_parent_fraction"] + ) + + if "auto" in f.name and f.suffix == ".tsv": + automatic_df = pandas.read_csv(f, sep="\t") + original_whole_blood = open_meta_data( + Ex_txt_manual_and_autosampled_mixed["whole"][0] + ) + assert "whole_blood_radioactivity" in automatic_df.columns + assert len(automatic_df) + len(manual_df) == len( + original_whole_blood + ) diff --git a/pypet2bids/tests/test_dcm2niix4pet.py b/pypet2bids/tests/test_dcm2niix4pet.py index dd4d1a8e..b02b8e3b 100644 --- a/pypet2bids/tests/test_dcm2niix4pet.py +++ b/pypet2bids/tests/test_dcm2niix4pet.py @@ -1,5 +1,13 @@ -from pypet2bids.dcm2niix4pet import Dcm2niix4PET, dicom_datetime_to_dcm2niix_time, check_json, collect_date_time_from_file_name, update_json_with_dicom_value -from pypet2bids.dcm2niix4pet import check_meta_radio_inputs +from pypet2bids.dcm2niix4pet import ( + Dcm2niix4PET, + dicom_datetime_to_dcm2niix_time, + collect_date_time_from_file_name, +) +from pypet2bids.update_json_pet_file import ( + check_meta_radio_inputs, + check_json, + update_json_with_dicom_value, +) import shutil import dotenv @@ -13,21 +21,22 @@ from unittest import TestCase - # collect config files # fields to check for module_folder = Path(__file__).parent.resolve() python_folder = module_folder.parent pet2bids_folder = python_folder.parent -spreadsheet_folder = join(pet2bids_folder, 'spreadsheet_conversion') +spreadsheet_folder = join(pet2bids_folder, "spreadsheet_conversion") # collect paths to test files/folders dotenv.load_dotenv(dotenv.find_dotenv()) -test_dicom_image_folder = os.environ['TEST_DICOM_IMAGE_FOLDER'] -test_dicom_convert_metadata_path = os.environ['TEST_DICOM_CONVERT_METADATA_PATH'] -test_dicom_convert_nifti_output_path = os.environ['TEST_DICOM_CONVERT_NIFTI_OUTPUT_PATH'] +test_dicom_image_folder = os.environ["TEST_DICOM_IMAGE_FOLDER"] +test_dicom_convert_metadata_path = os.environ["TEST_DICOM_CONVERT_METADATA_PATH"] +test_dicom_convert_nifti_output_path = os.environ[ + "TEST_DICOM_CONVERT_NIFTI_OUTPUT_PATH" +] # collect dicoms from test input path representative_dicoms = {} @@ -37,7 +46,10 @@ for f in files: try: dicom_header = pydicom.dcmread(os.path.join(root, f)) - representative_dicoms[dicom_header.SeriesDescription] = {'filepath': os.path.join(root, f), 'header': dicom_header} + representative_dicoms[dicom_header.SeriesDescription] = { + "filepath": os.path.join(root, f), + "header": dicom_header, + } break except pydicom.errors.InvalidDicomError: @@ -45,25 +57,27 @@ def test_check_json(capsys): - from pypet2bids.helper_functions import log - logger = log() + from pypet2bids.helper_functions import logger as log + + logger = log("pypet2bids") with TemporaryDirectory() as tempdir: tempdir_path = Path(tempdir) bad_json = {"Nothing": "but trouble"} - bad_json_path = os.path.join(tempdir, 'bad_json.json') - with open(bad_json_path, 'w') as outfile: + bad_json_path = os.path.join(tempdir, "bad_json.json") + with open(bad_json_path, "w") as outfile: json.dump(bad_json, outfile) check_results = check_json(bad_json_path) check_output = capsys.readouterr() - assert check_results['Manufacturer'] == {'key': False, 'value': False} - #assert f'WARNING - Manufacturer is not present in {bad_json_path}' in check_output.out - + assert check_results["Manufacturer"] == {"key": False, "value": False} + # assert f'WARNING - Manufacturer is not present in {bad_json_path}' in check_output.out def test_extract_dicom_headers(): - converter = Dcm2niix4PET(test_dicom_image_folder, test_dicom_convert_nifti_output_path) + converter = Dcm2niix4PET( + test_dicom_image_folder, test_dicom_convert_nifti_output_path + ) converter.extract_dicom_headers() for key, value in converter.dicom_headers.items(): assert os.path.isfile(os.path.join(test_dicom_image_folder, key)) @@ -100,15 +114,15 @@ def test_match_dicom_header_to_file(): dicom_study_time = dicom_datetime_to_dcm2niix_time(dicom_header) for output_file in output_files: # first check nifti json - if '.json' in output_file: + if ".json" in output_file: # assert json filename follows our standard conventions assert dicom_study_time in output_file with open(output_file) as nifti_json: nifti_dict = json.load(nifti_json) - assert dicom_header.SeriesNumber == nifti_dict['SeriesNumber'] + assert dicom_header.SeriesNumber == nifti_dict["SeriesNumber"] # check .nii as well - if '.nii' in output_file or '.nii.gz' in output_file: + if ".nii" in output_file or ".nii.gz" in output_file: assert dicom_study_time in output_file @@ -119,7 +133,9 @@ def test_collect_date_from_file_name(): converter = Dcm2niix4PET(test_dicom_image_folder, tempdir_path) converter.extract_dicom_headers() - first_dicom_header = converter.dicom_headers[next(iter(converter.dicom_headers))] + first_dicom_header = converter.dicom_headers[ + next(iter(converter.dicom_headers)) + ] StudyDate = first_dicom_header.StudyDate StudyTime = str(round(float(first_dicom_header.StudyTime))) if len(StudyTime) < 6: @@ -138,20 +154,28 @@ def test_collect_date_from_file_name(): def test_run_dcm2niix(): - converter = Dcm2niix4PET(test_dicom_image_folder, test_dicom_convert_nifti_output_path, file_format = '%p_%i_%t_%s', - silent=True) + converter = Dcm2niix4PET( + test_dicom_image_folder, + test_dicom_convert_nifti_output_path, + file_format="%p_%i_%t_%s", + silent=True, + ) converter.run_dcm2niix() contents_output = os.listdir(test_dicom_convert_nifti_output_path) - created_jsons = [file for file in contents_output if '.json' in file] - created_niftis = [file for file in contents_output if '.nii' in file] + created_jsons = [file for file in contents_output if ".json" in file] + created_niftis = [file for file in contents_output if ".nii" in file] dcm2niix_output = {} for file in created_jsons: path_object = Path(file) - checks = check_json(join(test_dicom_convert_nifti_output_path, path_object), silent=True) + checks = check_json( + join(test_dicom_convert_nifti_output_path, path_object), silent=True + ) output_file_stem = os.path.join(*path_object.parents, path_object.stem) if dcm2niix_output.get(output_file_stem, None): - dcm2niix_output[output_file_stem] = dcm2niix_output[output_file_stem].append(path_object.resolve()) + dcm2niix_output[output_file_stem] = dcm2niix_output[ + output_file_stem + ].append(path_object.resolve()) else: dcm2niix_output[output_file_stem] = [] dcm2niix_output[output_file_stem].append(path_object.resolve()) @@ -160,7 +184,9 @@ def test_run_dcm2niix(): path_object = Path(file) output_file_stem = os.path.join(*path_object.parents, path_object.stem) if dcm2niix_output.get(output_file_stem, None): - dcm2niix_output[output_file_stem] = dcm2niix_output[output_file_stem].append(path_object.resolve()) + dcm2niix_output[output_file_stem] = dcm2niix_output[ + output_file_stem + ].append(path_object.resolve()) else: dcm2niix_output[output_file_stem] = [] dcm2niix_output[output_file_stem].append(path_object.resolve()) @@ -170,7 +196,7 @@ def test_run_dcm2niix(): def test_manufacturers(): # As far as we know there are these three manufacturers - manufacturers = ['phillips', 'siemens', 'ge'] + manufacturers = ["phillips", "siemens", "ge"] # make sure paths are attached with vendor/manufacturer specific dicoms manufacturer_paths = {} @@ -180,7 +206,7 @@ def test_manufacturers(): if dicom_folder_path != "": dicom_folder_path = Path(dicom_folder_path) if dicom_folder_path.exists(): - manufacturer_paths[manu] = {'dicom_path': dicom_folder_path} + manufacturer_paths[manu] = {"dicom_path": dicom_folder_path} # open a temporary directory to write output to with TemporaryDirectory() as tempdir: @@ -190,22 +216,24 @@ def test_manufacturers(): for manu in manus: nifti_path = os.path.join(tempdir_path, f"{manu}_nifti_output") os.mkdir(nifti_path) - manufacturer_paths[manu]['nifti_path'] = nifti_path + manufacturer_paths[manu]["nifti_path"] = nifti_path # convert these things - input_path = manufacturer_paths[manu]['dicom_path'] - output_path = manufacturer_paths[manu]['nifti_path'] + input_path = manufacturer_paths[manu]["dicom_path"] + output_path = manufacturer_paths[manu]["nifti_path"] converter = Dcm2niix4PET(input_path, output_path) - converter.run_dcm2niix() print(f"running converson on images at {input_path}") # add jsons to the manufacturer_paths output_path_files = os.listdir(output_path) - jsons = [os.path.join(output_path, output_json) for output_json in output_path_files if - '.json' in output_json] + jsons = [ + os.path.join(output_path, output_json) + for output_json in output_path_files + if ".json" in output_json + ] - manufacturer_paths[manu]['json_path'] = jsons + manufacturer_paths[manu]["json_path"] = jsons # now check the output of the jsons, see if the fields are all there for key, value in manufacturer_paths.items(): @@ -225,8 +253,8 @@ def test_update_json_with_dicom_value(): if running_in_github.lower() != "true": # open temporary directory b/c manual cleanup is a bother with TemporaryDirectory() as tempdir: - test_json_path = Path(os.path.join(tempdir, 'test.json')) - with open(test_json_path, 'w') as outfile: + test_json_path = Path(os.path.join(tempdir, "test.json")) + with open(test_json_path, "w") as outfile: json.dump({}, outfile) # now check json, we should be missing everything from it as it's an empty set @@ -238,9 +266,11 @@ def test_update_json_with_dicom_value(): dicom_folder = os.getenv("TEST_DICOM_IMAGE_FOLDER", "") # possible dicoms - possible_dicoms = [os.path.join(dicom_folder, d) for d in os.listdir(dicom_folder)] + possible_dicoms = [ + os.path.join(dicom_folder, d) for d in os.listdir(dicom_folder) + ] # we don't need all of the dicom 10% should be good - for i in range(int(0.2*len(possible_dicoms))): + for i in range(int(0.2 * len(possible_dicoms))): # 100 is plenty if len(dicom_headers) >= 100: break @@ -252,32 +282,44 @@ def test_update_json_with_dicom_value(): # now that we have dicom headers we can use our insert method to insert missing metadata # into a json - update_json_with_dicom_value(test_json_path, missing_fields, dicom_headers[0]) + update_json_with_dicom_value( + test_json_path, missing_fields, dicom_headers[0] + ) # load json to make assertions - with open(test_json_path, 'r') as infile: + with open(test_json_path, "r") as infile: test_json = json.load(infile) # only checking a handful of fields as every dicom varies, but our test data has the following fields # filled in - check_fields = ['Manufacturer', 'InstitutionName', 'Units', 'InstitutionalDepartmentName'] + check_fields = [ + "Manufacturer", + "InstitutionName", + "Units", + "InstitutionalDepartmentName", + ] for field in check_fields: assert test_json.get(field, "") != "" def test_additional_arguments(): - additional_args = {'additional1': 1, 'additional2': 2} + additional_args = {"additional1": 1, "additional2": 2} with TemporaryDirectory() as tempdir: - converter = Dcm2niix4PET(test_dicom_image_folder, tempdir, - file_format='%p_%i_%t_%s', - silent=True, - additional_arguments=additional_args) + converter = Dcm2niix4PET( + test_dicom_image_folder, + tempdir, + file_format="%p_%i_%t_%s", + silent=True, + additional_arguments=additional_args, + ) converter.run_dcm2niix() contents_output = os.listdir(tempdir) - created_jsons = [os.path.join(tempdir, file) for file in contents_output if '.json' in file] + created_jsons = [ + os.path.join(tempdir, file) for file in contents_output if ".json" in file + ] # load in json, make sure fields are there - with open(created_jsons[0], 'r') as infile: + with open(created_jsons[0], "r") as infile: json_contents = json.load(infile) for key, value in additional_args.items(): @@ -286,86 +328,105 @@ def test_additional_arguments(): def test_check_meta_radio_inputs(): # test first conditional given InjectedRadioactivity and InjectedMass - given = {'InjectedRadioactivity': 10, 'InjectedMass': 10} - solution = {'InjectedRadioactivityUnits': 'MBq', - 'InjectedMassUnits': 'ug', - 'SpecificRadioactivityUnits': 'Bq/g', - 'SpecificRadioactivity': 1} + given = {"InjectedRadioactivity": 10, "InjectedMass": 10} + solution = { + "InjectedRadioactivityUnits": "MBq", + "InjectedMassUnits": "ug", + "SpecificRadioactivityUnits": "Bq/g", + "SpecificRadioactivity": 1, + } solution.update(given) this = check_meta_radio_inputs(given) TestCase().assertEqual(this, solution) # first case + adding in a value for SpecificRadioactivity - given = {'InjectedRadioactivity': 10, 'InjectedMass': 10, 'SpecificRadioactivity':1} - solution = {'InjectedRadioactivityUnits': 'n/a', - 'InjectedMassUnits': 'ug', - 'SpecificRadioactivityUnits': 'Bq/g'} + given = { + "InjectedRadioactivity": 10, + "InjectedMass": 10, + "SpecificRadioactivity": 1, + } + solution = { + "InjectedRadioactivityUnits": "n/a", + "InjectedMassUnits": "ug", + "SpecificRadioactivityUnits": "Bq/g", + } solution.update(given) this = check_meta_radio_inputs(given) TestCase().assertEqual(this, solution) # second case + SpecificRadioactivityUnits adde to given - solution.update({'SpecificRadioactivityUnits': 'Bq/g'}) - given.update({'SpecificRadioactivityUnits': 'Bq/g'}) + solution.update({"SpecificRadioactivityUnits": "Bq/g"}) + given.update({"SpecificRadioactivityUnits": "Bq/g"}) this = check_meta_radio_inputs(given) TestCase().assertEqual(this, solution) # test second conditional given InjectedRadioactivity and SpecificRadioactivity - given = {'InjectedRadioactivity': 10, 'SpecificRadioactivity': 10} - solution = {'InjectedRadioactivityUnits': 'MBq', - 'InjectedMass': 1000000000000.0, - 'InjectedMassUnits': 'ug', - 'SpecificRadioactivityUnits': 'Bq/g', - 'SpecificRadioactivity': 10} + given = {"InjectedRadioactivity": 10, "SpecificRadioactivity": 10} + solution = { + "InjectedRadioactivityUnits": "MBq", + "InjectedMass": 1000000000000.0, + "InjectedMassUnits": "ug", + "SpecificRadioactivityUnits": "Bq/g", + "SpecificRadioactivity": 10, + } solution.update(given) this = check_meta_radio_inputs(given) TestCase().assertEqual(this, solution) # test SpecificRadioactivity is okay given = {"InjectedRadioactivity": 44.4, "InjectedMass": 6240} - solution = {"InjectedRadioactivityUnits": 'Bq/g', - "InjectedMass": given['InjectedMass'], - "InjectedMassUnits": 'ug', - "SpecificRadioactivityUnits": 'Bq/g', - "SpecificRadioactivity": (given['InjectedRadioactivity']*(10**6)) / (given['InjectedMass']*(10**6)) + solution = { + "InjectedRadioactivityUnits": "Bq/g", + "InjectedMass": given["InjectedMass"], + "InjectedMassUnits": "ug", + "SpecificRadioactivityUnits": "Bq/g", + "SpecificRadioactivity": (given["InjectedRadioactivity"] * (10**6)) + / (given["InjectedMass"] * (10**6)), } this = check_meta_radio_inputs(given) - TestCase().assertEqual(this['SpecificRadioactivity'], solution['SpecificRadioactivity']) + TestCase().assertEqual( + this["SpecificRadioactivity"], solution["SpecificRadioactivity"] + ) # check calc injected mass is okay - given = {'InjectedRadioactivity': 44, 'SpecificRadioactivity': 7.1154*(10**9)} - InjectedMass = ((given['InjectedRadioactivity']*(10**6))/(given['SpecificRadioactivity'])*(10**6)) + given = {"InjectedRadioactivity": 44, "SpecificRadioactivity": 7.1154 * (10**9)} + InjectedMass = ( + (given["InjectedRadioactivity"] * (10**6)) + / (given["SpecificRadioactivity"]) + * (10**6) + ) this = check_meta_radio_inputs(given) - TestCase().assertEqual(this['InjectedMass'], InjectedMass) + TestCase().assertEqual(this["InjectedMass"], InjectedMass) # check InjectedRadioactivity is okay - given = {'SpecificRadioactivity': 7.1154*(10**9), 'InjectedMass': 6240} - InjectedRadioactivity = ((given['InjectedMass']/(10**6)) * given['SpecificRadioactivity']) / 10**6 + given = {"SpecificRadioactivity": 7.1154 * (10**9), "InjectedMass": 6240} + InjectedRadioactivity = ( + (given["InjectedMass"] / (10**6)) * given["SpecificRadioactivity"] + ) / 10**6 this = check_meta_radio_inputs(given) - TestCase().assertEqual(this['InjectedRadioactivity'], InjectedRadioactivity) + TestCase().assertEqual(this["InjectedRadioactivity"], InjectedRadioactivity) # check SpecificRadioactivity is okay - given = {'MolarActivity': 135192600, 'MolecularWeight': 19} - SpecificRadioactivity = (given['MolarActivity']*1000)/given['MolecularWeight'] + given = {"MolarActivity": 135192600, "MolecularWeight": 19} + SpecificRadioactivity = (given["MolarActivity"] * 1000) / given["MolecularWeight"] this = check_meta_radio_inputs(given) - TestCase().assertEqual(this['SpecificRadioactivity'], SpecificRadioactivity) + TestCase().assertEqual(this["SpecificRadioactivity"], SpecificRadioactivity) # check MolecularWeight is okay - given = {'MolarActivity': 135192600, 'SpecificRadioactivity': 7.1154*(10**9)} - MolecularWeight = (given['MolarActivity']*1000)/SpecificRadioactivity + given = {"MolarActivity": 135192600, "SpecificRadioactivity": 7.1154 * (10**9)} + MolecularWeight = (given["MolarActivity"] * 1000) / SpecificRadioactivity this = check_meta_radio_inputs(given) - TestCase().assertEqual(this['MolecularWeight'], MolecularWeight) + TestCase().assertEqual(this["MolecularWeight"], MolecularWeight) # check MolarActivity is okay - given = {'MolecularWeight': 19, 'SpecificRadioactivity': 7.1154*(10**9)} - MolarActivity = (given['MolecularWeight']*given['SpecificRadioactivity'])/1000 + given = {"MolecularWeight": 19, "SpecificRadioactivity": 7.1154 * (10**9)} + MolarActivity = (given["MolecularWeight"] * given["SpecificRadioactivity"]) / 1000 this = check_meta_radio_inputs(given) - TestCase().assertEqual(this['MolarActivity'], MolarActivity) + TestCase().assertEqual(this["MolarActivity"], MolarActivity) def test_get_convolution_kernel(): - convolution_kernel_strings = [ - ] + convolution_kernel_strings = [] def test_run_dcm2niix4pet_with_full_blood_sheet(): @@ -378,15 +439,21 @@ def test_run_dcm2niix4pet_with_full_blood_sheet(): :return: None :rtype: None """ - spreadsheet = os.path.join(spreadsheet_folder, 'single_subject_sheet', 'subject_metadata_example.xlsx') + spreadsheet = os.path.join( + spreadsheet_folder, "single_subject_sheet", "subject_metadata_example.xlsx" + ) with TemporaryDirectory() as tempdir: - destination = os.path.join(tempdir, 'bids_test_dir/sub-01/pet') - dcm2niix4pet = Dcm2niix4PET(image_folder=test_dicom_image_folder, - destination_path=destination, - metadata_path=spreadsheet, - silent=True) + destination = os.path.join(tempdir, "bids_test_dir/sub-01/pet") + dcm2niix4pet = Dcm2niix4PET( + image_folder=test_dicom_image_folder, + destination_path=destination, + metadata_path=spreadsheet, + silent=True, + ) dcm2niix4pet.convert() - contents_output = [os.path.join(destination, f) for f in os.listdir(destination)] + contents_output = [ + os.path.join(destination, f) for f in os.listdir(destination) + ] # copy over dataset_description.json to bids dir dataset_description = { @@ -399,28 +466,30 @@ def test_run_dcm2niix4pet_with_full_blood_sheet(): "Sune Høgild Keller", "Gabriel Gonzalez-Escamilla", "Søren Baarsgaard Hansen", - "Maqsood Yaqub" + "Maqsood Yaqub", ], - "HowToAcknowledge": "Please cite the repository URL" + "HowToAcknowledge": "Please cite the repository URL", } - with open(os.path.join(tempdir, 'bids_test_dir', 'dataset_description.json'), 'w') as f: + with open( + os.path.join(tempdir, "bids_test_dir", "dataset_description.json"), "w" + ) as f: json.dump(dataset_description, f, indent=4) # copy over a readme file, we use the one in the metadata folder - readme = os.path.join(pet2bids_folder, 'metadata/', 'README') - shutil.copy(readme, os.path.join(tempdir, 'bids_test_dir')) + readme = os.path.join(pet2bids_folder, "metadata/", "README") + shutil.copy(readme, os.path.join(tempdir, "bids_test_dir")) # run the bids validator on the output - command = ['bids-validator', os.path.join(tempdir, 'bids_test_dir')] + command = ["bids-validator", os.path.join(tempdir, "bids_test_dir")] validation = subprocess.run(command, capture_output=True) # check exit code of subprocess assert validation.returncode == 0 # verify that the output is as expected - output = validation.stdout.decode('utf-8') - assert 'This dataset appears to be BIDS compatible' in output + output = validation.stdout.decode("utf-8") + assert "This dataset appears to be BIDS compatible" in output -if __name__ == '__main__': - test_update_json_with_dicom_value() \ No newline at end of file +if __name__ == "__main__": + test_update_json_with_dicom_value() diff --git a/pypet2bids/tests/test_ecat.py b/pypet2bids/tests/test_ecat.py new file mode 100644 index 00000000..8072c3b7 --- /dev/null +++ b/pypet2bids/tests/test_ecat.py @@ -0,0 +1,173 @@ +import os +import subprocess +import tempfile +import pathlib +import json +import pdb +from pypet2bids.ecat import Ecat + +TESTS_DIR = pathlib.Path(__file__).resolve().parent +PYPET2BIDS_DIR = TESTS_DIR.parent +PET2BIDS_DIR = PYPET2BIDS_DIR.parent + +# obtain ecat file path +ecat_file_path = PET2BIDS_DIR / "ecat_validation" / "ECAT7_multiframe.v.gz" +ecatpet2bids = PYPET2BIDS_DIR / "pypet2bids" / "ecat_cli.py" + +dataset_description_dictionary = { + "_Comment": "This is a very basic example of a dataset description json", + "Name": "PET Brain phantoms", + "BIDSVersion": "1.7.0", + "DatasetType": "raw", + "License": "CC0", + "Authors": [ + "Author1 Surname1", + "Author2 Surname2", + "Author3 Surname3", + "Author4 Middlename4 Surname4", + "Author5 Middlename5 Surname5", + ], + "HowToAcknowledge": "No worries this is fake.", + "ReferencesAndLinks": [ + "No you aren't getting any", + "Don't bother to ask", + "Fine, https://fake.fakelink.null", + ], +} + + +def test_kwargs_produce_valid_conversion(tmp_path): + # prepare a set of kwargs (stolen from a valid bids subject/dataset, mum's the word ;) ) + full_set_of_kwargs = { + "Modality": "PT", + "Manufacturer": "Siemens", + "ManufacturersModelName": "Biograph 64_mCT", + "InstitutionName": "NIH", + "InstitutionalDepartmentName": "NIMH MIB", + "InstitutionAddress": "10 Center Drive, Bethesda, MD 20892", + "DeviceSerialNumber": "60005", + "StationName": "MIAWP60005", + "PatientPosition": "FFS", + "SoftwareVersions": "VG60A", + "SeriesDescription": "PET Brain Dyn TOF", + "ProtocolName": "PET Brain Dyn TOF", + "ImageType": ["ORIGINAL", "PRIMARY"], + "SeriesNumber": 6, + "ScanStart": 2, + "TimeZero": "10:39:46", + "InjectionStart": 0, + "AcquisitionNumber": 2001, + "ImageComments": "Frame 1 of 33^AC_CT_Brain", + "Radiopharmaceutical": "ps13", + "RadionuclidePositronFraction": 0.997669, + "RadionuclideTotalDose": 714840000.0, + "RadionuclideHalfLife": 1220.04, + "DoseCalibrationFactor": 30806700.0, + "ConvolutionKernel": "XYZ Gauss2.00", + "Units": "Bq/mL", + "ReconstructionName": "PSF+TOF", + "ReconstructionParameterUnits": ["None", "None"], + "ReconstructionParameterLabels": ["subsets", "iterations"], + "ReconstructionParameterValues": [21, 3], + "ReconFilterType": "XYZ Gauss", + "ReconFilterSize": 2.0, + "AttenuationCorrection": "measured,AC_CT_Brain", + "DecayFactor": [1.00971], + "FrameTimesStart": [0], + "FrameDuration": [30], + "SliceThickness": 2, + "ImageOrientationPatientDICOM": [1, 0, 0, 0, 1, 0], + "ConversionSoftware": ["dcm2niix", "pypet2bids"], + "ConversionSoftwareVersion": ["v1.0.20211006", "0.0.8"], + "TracerName": "[11C]PS13", + "TracerRadionuclide": "11C", + "InjectedRadioactivity": 714840000.0, + "InjectedRadioactivityUnits": "Bq", + "InjectedMass": 5.331647109063877, + "InjectedMassUnits": "nmol", + "SpecificRadioactivity": 341066000000000, + "SpecificRadioactivityUnits": "Bq/mol", + "ModeOfAdministration": "bolus", + "AcquisitionMode": "dynamic", + "ImageDecayCorrected": True, + "ImageDecayCorrectionTime": 0, + "ReconMethodName": "Point-Spread Function + Time Of Flight", + "ReconMethodParameterLabels": ["subsets", "iterations"], + "ReconMethodParameterUnits": ["none", "none"], + "ReconMethodParameterValues": [21, 3], + "Haematocrit": 0.308, + } + + # test ecat converter + + # create ecat dir + ecat_bids_dir = tmp_path / "ecat_test/sub-ecat/ses-test/pet" + ecat_bids_dir.mkdir(parents=True, exist_ok=True) + + # we're going to want a dataset description json at a minimum + dataset_description_path = ( + ecat_bids_dir.parent.parent.parent / "dataset_description.json" + ) + with open(dataset_description_path, "w") as outfile: + json.dump(dataset_description_dictionary, outfile, indent=4) + + ecat_bids_nifti_path = ecat_bids_dir / "sub-ecat_ses-test_pet.nii" + + # run ecat converter + convert_ecat = Ecat( + ecat_file=str(ecat_file_path), + nifti_file=str(ecat_bids_nifti_path), + kwargs=full_set_of_kwargs, + collect_pixel_data=True, + ) + + convert_ecat.convert() + + # run validator + cmd = f"bids-validator {ecat_bids_dir.parent.parent.parent} --ignoreWarnings" + validate_ecat = subprocess.run(cmd, shell=True, capture_output=True) + + assert validate_ecat.returncode == 0, cmd + + +def test_spreadsheets_produce_valid_conversion_ecatpet2bids(tmp_path): + # collect spreadsheets + single_subject_spreadsheet = ( + PET2BIDS_DIR + / "spreadsheet_conversion/single_subject_sheet/subject_metadata_example.xlsx" + ) + + ecatpet2bids_test_dir = tmp_path / "ecatpet2bids_spreadsheet_input" + ecatpet2bids_test_dir.mkdir(parents=True, exist_ok=True) + subject_folder = ( + ecatpet2bids_test_dir / "sub-singlesubjectspreadsheetecat" / "ses-test" / "pet" + ) + + cmd = ( + f"python {ecatpet2bids} {ecat_file_path} " + f"--nifti {subject_folder}/sub-singlesubjectspreadsheetecat_ses-test_pet.nii.gz " + f"--metadata-path {single_subject_spreadsheet} " + f"--convert" + ) + + spreadsheet_ecat = Ecat( + ecat_file=str(ecat_file_path), + nifti_file=str(subject_folder) + + "/sub-singlesubjectspreadsheetecat_ses-test_pet.nii.gz", + metadata_path=single_subject_spreadsheet, + collect_pixel_data=True, + ) + + spreadsheet_ecat.convert() + + # copy over dataset_description + dataset_description_path = ecatpet2bids_test_dir / "dataset_description.json" + with open(dataset_description_path, "w") as outfile: + json.dump(dataset_description_dictionary, outfile, indent=4) + + validator_cmd = f"bids-validator {ecatpet2bids_test_dir} --ingnoreWarnings" + validate_ecat_w_spreadsheet = subprocess.run( + validator_cmd, shell=True, capture_output=True + ) + + assert validate_ecat_w_spreadsheet.returncode == 0 diff --git a/pypet2bids/tests/test_ecatread.py b/pypet2bids/tests/test_ecatread.py index 8e2ae58e..018aadde 100644 --- a/pypet2bids/tests/test_ecatread.py +++ b/pypet2bids/tests/test_ecatread.py @@ -10,14 +10,16 @@ parent_dir = pathlib.Path(__file__).parent.resolve() -code_dir = os.path.join(parent_dir.parent, 'pypet2bids') +code_dir = os.path.join(parent_dir.parent, "pypet2bids") # collect ecat header maps try: - with open(os.path.join(code_dir, 'ecat_headers.json'), 'r') as infile: + with open(os.path.join(code_dir, "ecat_headers.json"), "r") as infile: ecat_header_maps = json.load(infile) except FileNotFoundError: - raise Exception("Unable to load header definitions and map from ecat_headers.json. Aborting.") + raise Exception( + "Unable to load header definitions and map from ecat_headers.json. Aborting." + ) """ This script reads in an ecat file from .env/environment variable 'TEST_ECAT_PATH' and saves the pixel data into a @@ -29,12 +31,12 @@ """ # collect path to test ecat file dotenv.load_dotenv(dotenv.find_dotenv()) # load path from .env file into env -ecat_path = os.environ['TEST_ECAT_PATH'] -read_ecat_save_as_matlab = os.environ['READ_ECAT_SAVE_AS_MATLAB'] -nibabel_read_ecat_save_as_matlab = os.environ['NIBABEL_READ_ECAT_SAVE_AS_MATLAB'] +ecat_path = os.environ["TEST_ECAT_PATH"] +read_ecat_save_as_matlab = os.environ["READ_ECAT_SAVE_AS_MATLAB"] +nibabel_read_ecat_save_as_matlab = os.environ["NIBABEL_READ_ECAT_SAVE_AS_MATLAB"] -if __name__ == '__main__': +if __name__ == "__main__": # read in the ecat ecat_main_header, ecat_subheaders, ecat_image = read_ecat(ecat_file=ecat_path) @@ -44,18 +46,24 @@ nibabel_data = nibabel_ecat.get_fdata() # save the read ecat pixel object as a matlab object for comparison - matlab_dictionary = {'data': ecat_image} + matlab_dictionary = {"data": ecat_image} savemat(read_ecat_save_as_matlab, matlab_dictionary) - print(f"Saved read ecat datastructure as matlab matrix at:\n{read_ecat_save_as_matlab}") + print( + f"Saved read ecat datastructure as matlab matrix at:\n{read_ecat_save_as_matlab}" + ) # save the nibabel ecat read as matlab object for comparison - matlab_dictionary = {'nibabel_data': nibabel_data} + matlab_dictionary = {"nibabel_data": nibabel_data} try: - savemat(nibabel_read_ecat_save_as_matlab, matlab_dictionary, do_compression=True) - print(f"Saved read ecat datastructure as matlab matrix at:\n{nibabel_read_ecat_save_as_matlab}") + savemat( + nibabel_read_ecat_save_as_matlab, matlab_dictionary, do_compression=True + ) + print( + f"Saved read ecat datastructure as matlab matrix at:\n{nibabel_read_ecat_save_as_matlab}" + ) except scipy.io.matlab.miobase.MatWriteError as err: - print(f"Unable to write nibabel array of size {nibabel_data.shape} and " - f"datatype {nibabel_data.dtype} to matlab .mat file") + print( + f"Unable to write nibabel array of size {nibabel_data.shape} and " + f"datatype {nibabel_data.dtype} to matlab .mat file" + ) print(err) - - diff --git a/pypet2bids/tests/test_helper_functions.py b/pypet2bids/tests/test_helper_functions.py index 094b499b..52ba1362 100644 --- a/pypet2bids/tests/test_helper_functions.py +++ b/pypet2bids/tests/test_helper_functions.py @@ -2,6 +2,7 @@ import tempfile import unittest import datetime + try: import pypet2bids.helper_functions as helper_functions import pypet2bids.is_pet as is_pet @@ -18,17 +19,25 @@ module_folder = Path(__file__).parent.resolve() python_folder = module_folder.parent pet2bids_folder = python_folder.parent -metadata_folder = join(pet2bids_folder, 'spreadsheet_conversion') -single_subject_metadata_file = join(metadata_folder, 'single_subject_sheet', 'subject_metadata_example.xlsx') -multi_subject_metadata_file = join(metadata_folder, 'many_subjects_sheet', 'subjects_metadata_example.xlsx') -scanner_metadata_file = join(metadata_folder, 'many_subjects_sheet', 'scanner_metadata_example.xlsx') -multi_sheet_metadata_file = join(metadata_folder, 'single_subject_sheet', 'subject_metadata_multisheet_example.xlsx') +metadata_folder = join(pet2bids_folder, "spreadsheet_conversion") +single_subject_metadata_file = join( + metadata_folder, "single_subject_sheet", "subject_metadata_example.xlsx" +) +multi_subject_metadata_file = join( + metadata_folder, "many_subjects_sheet", "subjects_metadata_example.xlsx" +) +scanner_metadata_file = join( + metadata_folder, "many_subjects_sheet", "scanner_metadata_example.xlsx" +) +multi_sheet_metadata_file = join( + metadata_folder, "single_subject_sheet", "subject_metadata_multisheet_example.xlsx" +) class TestHelperFunctions(unittest.TestCase): @classmethod def setUp(cls) -> None: - test_config_string = ''' + test_config_string = """ test_int=0 test_int_list=[1,2,3,4] test_float=99.9 @@ -36,39 +45,41 @@ def setUp(cls) -> None: test_string=CharlieWorks test_string_list=['entry1', 'entry2', 'entry3', 'entry4'] test_mixed_list=['a', 1, 2.0, 'longstring'] - ''' - cls.test_env_file_path = '.test_env' - with open(cls.test_env_file_path, 'w') as outfile: + """ + cls.test_env_file_path = ".test_env" + with open(cls.test_env_file_path, "w") as outfile: outfile.write(test_config_string) cls.test_load = helper_functions.load_vars_from_config(cls.test_env_file_path) def test_load_vars_from_config(self): self.test_load = helper_functions.load_vars_from_config(self.test_env_file_path) - self.assertEqual(type(self.test_load), collections.OrderedDict) # add assertion here + self.assertEqual( + type(self.test_load), collections.OrderedDict + ) # add assertion here def test_int(self): - self.assertEqual(self.test_load['test_int'], 0) + self.assertEqual(self.test_load["test_int"], 0) def test_float(self): - self.assertEqual(self.test_load['test_float'], 99.9) + self.assertEqual(self.test_load["test_float"], 99.9) def test_int_list(self): - self.assertEqual(self.test_load['test_int_list'], [1, 2, 3, 4]) + self.assertEqual(self.test_load["test_int_list"], [1, 2, 3, 4]) def test_float_list(self): - self.assertEqual(self.test_load['test_float_list'], [1.0, 2.0, 3.0, 4.0]) + self.assertEqual(self.test_load["test_float_list"], [1.0, 2.0, 3.0, 4.0]) def test_string(self): - self.assertEqual(self.test_load['test_string'], 'CharlieWorks') + self.assertEqual(self.test_load["test_string"], "CharlieWorks") def test_string_list(self): - self.assertEqual(self.test_load['test_string_list'], - ['entry1', 'entry2', 'entry3', 'entry4']) + self.assertEqual( + self.test_load["test_string_list"], ["entry1", "entry2", "entry3", "entry4"] + ) def test_mixed_list(self): - self.assertEqual(self.test_load['test_mixed_list'], - ['a', 1, 2.0, 'longstring']) + self.assertEqual(self.test_load["test_mixed_list"], ["a", 1, 2.0, "longstring"]) @classmethod def tearDown(cls) -> None: @@ -92,44 +103,55 @@ def test_open_metadata(): def test_translate_metadata(): - test_translate_script_path = join(module_folder, 'metadata_excel_example_reader.py') + test_translate_script_path = join(module_folder, "metadata_excel_example_reader.py") - test_output = helper_functions.translate_metadata(single_subject_metadata_file,test_translate_script_path) + test_output = helper_functions.translate_metadata( + single_subject_metadata_file, test_translate_script_path + ) # values below manually parsed out of the file 'subject_metadata_example.xlsx' - assert test_output['nifti_json']['ImageDecayCorrectionTime'] == 0 - assert test_output['nifti_json']['ReconMethodName'] == '3D-OSEM-PSF' - assert test_output['nifti_json']['ReconMethodParameterLabels'] == ['subsets', 'iterations'] - assert test_output['nifti_json']['ReconMethodParameterUnits'] == ['none', 'none'] - assert test_output['nifti_json']['ReconMethodParameterValues'] == [16, 10] - assert test_output['nifti_json']['ReconFilterType'] == 'none' + assert test_output["nifti_json"]["ImageDecayCorrectionTime"] == 0 + assert test_output["nifti_json"]["ReconMethodName"] == "3D-OSEM-PSF" + assert test_output["nifti_json"]["ReconMethodParameterLabels"] == [ + "subsets", + "iterations", + ] + assert test_output["nifti_json"]["ReconMethodParameterUnits"] == ["none", "none"] + assert test_output["nifti_json"]["ReconMethodParameterValues"] == [16, 10] + assert test_output["nifti_json"]["ReconFilterType"] == "none" def test_collect_bids_parts(): - bids_like_path = '/home/users/user/bids_data/sub-NDAR123/ses-firstsession' - windows_bids_like_path = 'D:\BIDS\ONP\sub-NDAR123\ses-firstsession\pet' - subject_id = helper_functions.collect_bids_part('sub', bids_like_path) - assert subject_id == 'sub-NDAR123' - assert helper_functions.collect_bids_part('sub', windows_bids_like_path) == 'sub-NDAR123' - - session_id = helper_functions.collect_bids_part('ses', bids_like_path) - assert session_id == 'ses-firstsession' - assert helper_functions.collect_bids_part('ses', windows_bids_like_path) == 'ses-firstsession' - - not_bids_like_path = '/home/users/user/no/bids/here' - nope_sub = helper_functions.collect_bids_part('sub', not_bids_like_path) - assert nope_sub == '' - - nope_ses = helper_functions.collect_bids_part('ses', not_bids_like_path) - assert nope_ses == '' - - pet_path = '/home/users/user/bids_data/sub-NDAR123_ses-01_task-countbackwards_trc-CBGB_rec-ACDYN_run-03' - assert helper_functions.collect_bids_part('sub', pet_path) == 'sub-NDAR123' - assert helper_functions.collect_bids_part('ses', pet_path) == 'ses-01' - assert helper_functions.collect_bids_part('task', pet_path) == 'task-countbackwards' - assert helper_functions.collect_bids_part('trc', pet_path) == 'trc-CBGB' - assert helper_functions.collect_bids_part('rec', pet_path) == 'rec-ACDYN' - assert helper_functions.collect_bids_part('run', pet_path) == 'run-03' + bids_like_path = "/home/users/user/bids_data/sub-NDAR123/ses-firstsession" + windows_bids_like_path = "D:\BIDS\ONP\sub-NDAR123\ses-firstsession\pet" + subject_id = helper_functions.collect_bids_part("sub", bids_like_path) + assert subject_id == "sub-NDAR123" + assert ( + helper_functions.collect_bids_part("sub", windows_bids_like_path) + == "sub-NDAR123" + ) + + session_id = helper_functions.collect_bids_part("ses", bids_like_path) + assert session_id == "ses-firstsession" + assert ( + helper_functions.collect_bids_part("ses", windows_bids_like_path) + == "ses-firstsession" + ) + + not_bids_like_path = "/home/users/user/no/bids/here" + nope_sub = helper_functions.collect_bids_part("sub", not_bids_like_path) + assert nope_sub == "" + + nope_ses = helper_functions.collect_bids_part("ses", not_bids_like_path) + assert nope_ses == "" + + pet_path = "/home/users/user/bids_data/sub-NDAR123_ses-01_task-countbackwards_trc-CBGB_rec-ACDYN_run-03" + assert helper_functions.collect_bids_part("sub", pet_path) == "sub-NDAR123" + assert helper_functions.collect_bids_part("ses", pet_path) == "ses-01" + assert helper_functions.collect_bids_part("task", pet_path) == "task-countbackwards" + assert helper_functions.collect_bids_part("trc", pet_path) == "trc-CBGB" + assert helper_functions.collect_bids_part("rec", pet_path) == "rec-ACDYN" + assert helper_functions.collect_bids_part("run", pet_path) == "run-03" def test_transform_row_to_dict(): @@ -138,41 +160,54 @@ def test_transform_row_to_dict(): subject_with_frames_time_start_input = many_subjects_dataframe.iloc[0] # transform a row - transformed_row = helper_functions.transform_row_to_dict(subject_with_frames_time_start_input) + transformed_row = helper_functions.transform_row_to_dict( + subject_with_frames_time_start_input + ) - frame_times_start = subject_with_frames_time_start_input['FrameTimesStart'].split(',') + frame_times_start = subject_with_frames_time_start_input["FrameTimesStart"].split( + "," + ) frame_times_start = [int(entry) for entry in frame_times_start] - assert frame_times_start == transformed_row['FrameTimesStart'] + assert frame_times_start == transformed_row["FrameTimesStart"] # test whole dataframe transform - transform_row_from_dataframe = helper_functions.transform_row_to_dict(0, many_subjects_dataframe) + transform_row_from_dataframe = helper_functions.transform_row_to_dict( + 0, many_subjects_dataframe + ) - assert frame_times_start == transform_row_from_dataframe['FrameTimesStart'] + assert frame_times_start == transform_row_from_dataframe["FrameTimesStart"] # a simpler test - key = 'FrameTimesStart' - values = '0,1,2,3,4,5,6' + key = "FrameTimesStart" + values = "0,1,2,3,4,5,6" simpler_df = pandas.DataFrame({key: [values]}) - assert [int(v) for v in values.split(',')] == helper_functions.transform_row_to_dict(0, simpler_df)['FrameTimesStart'] + assert [ + int(v) for v in values.split(",") + ] == helper_functions.transform_row_to_dict(0, simpler_df)["FrameTimesStart"] def test_get_coordinates_containing(): given_data = { - 'columnA': ['string1', 'string2', 'string3', 'muchlongerstringVALUE'], - 'columnB': [0, 1, 2, 3], - 'columnC': [pandas.NA, 1.2, 3.1, pandas.NA] + "columnA": ["string1", "string2", "string3", "muchlongerstringVALUE"], + "columnB": [0, 1, 2, 3], + "columnC": [pandas.NA, 1.2, 3.1, pandas.NA], } given_dataframe = pandas.DataFrame(given_data) get_coords = helper_functions.get_coordinates_containing - assert get_coords('string3', given_dataframe) == [(2, 'columnA')] - assert get_coords('string3', given_dataframe, single=True) == (2, 'columnA') - assert get_coords('notthere', given_dataframe) == [] - assert get_coords('string', given_dataframe) == [(0, 'columnA'), (1, 'columnA'), (2, 'columnA'), (3, 'columnA')] - assert get_coords(0, given_dataframe, exact=True, single=True) == (0, 'columnB') - assert get_coords(0, given_dataframe, exact=True) == [(0, 'columnB')] + assert get_coords("string3", given_dataframe) == [(2, "columnA")] + assert get_coords("string3", given_dataframe, single=True) == (2, "columnA") + assert get_coords("notthere", given_dataframe) == [] + assert get_coords("string", given_dataframe) == [ + (0, "columnA"), + (1, "columnA"), + (2, "columnA"), + (3, "columnA"), + ] + assert get_coords(0, given_dataframe, exact=True, single=True) == (0, "columnB") + assert get_coords(0, given_dataframe, exact=True) == [(0, "columnB")] def test_get_recon_method(): @@ -196,7 +231,7 @@ def test_get_recon_method(): "ReconMethodName": "Point-Spread Function modelling Time Of Flight", "ReconMethodParameterUnits": ["none", "none"], "ReconMethodParameterLabels": ["subsets", "iterations"], - "ReconMethodParameterValues": [21, 3] + "ReconMethodParameterValues": [21, 3], }, { "contents": "OP-OSEM3i21s", @@ -205,7 +240,7 @@ def test_get_recon_method(): "ReconMethodName": "Ordinary Poisson Ordered Subset Expectation Maximization", "ReconMethodParameterUnits": ["none", "none"], "ReconMethodParameterLabels": ["subsets", "iterations"], - "ReconMethodParameterValues": [21, 3] + "ReconMethodParameterValues": [21, 3], }, { "contents": "PSF+TOF 3i21s", @@ -214,7 +249,7 @@ def test_get_recon_method(): "ReconMethodName": "Point-Spread Function modelling Time Of Flight", "ReconMethodParameterUnits": ["none", "none"], "ReconMethodParameterLabels": ["subsets", "iterations"], - "ReconMethodParameterValues": [21, 3] + "ReconMethodParameterValues": [21, 3], }, { "contents": "LOR-RAMLA", @@ -231,13 +266,13 @@ def test_get_recon_method(): "ReconMethodParameterLabels": ["none", "none"], }, { - "contents": 'OSEM:i3s15', + "contents": "OSEM:i3s15", "subsets": 15, "iterations": 3, "ReconMethodName": "Ordered Subset Expectation Maximization", "ReconMethodParameterUnits": ["none", "none"], "ReconMethodParameterLabels": ["subsets", "iterations"], - "ReconMethodParameterValues": [15, 3] + "ReconMethodParameterValues": [15, 3], }, { "contents": "LOR-RAMLA", @@ -251,31 +286,31 @@ def test_get_recon_method(): "subsets": None, "iterations": None, "ReconMethodName": "VUE Point HD using Time Of Flight with Point-Spread Function modelling", - "ReconMethodParameterLabels": ["none", "none"] + "ReconMethodParameterLabels": ["none", "none"], }, { "contents": "VPFX", "subsets": None, "iterations": None, "ReconMethodName": "VUE Point HD using Time Of Flight", - "ReconMethodParameterLabels": ["none", "none"] + "ReconMethodParameterLabels": ["none", "none"], }, { "contents": "inki-2006-may-OSEM3D-OP-PSFi10s16", "subsets": 16, "iterations": 10, "ReconMethodName": "3D Ordered Subset Expectation Maximization Ordinary Poisson Point-Spread Function modelling", - #"ReconMethodParameterLabels": ["subsets", "iterations", "lower_threshold", "upper_threshold"], - #"ReconMethodParameterUnits": ["none", "none", "keV", "keV"], - #"ReconMethodParameterValues": [16, 10, 400, 600] - } + # "ReconMethodParameterLabels": ["subsets", "iterations", "lower_threshold", "upper_threshold"], + # "ReconMethodParameterUnits": ["none", "none", "keV", "keV"], + # "ReconMethodParameterValues": [16, 10, 400, 600] + }, ] for recon_data in reconstruction_method_strings: - recon = helper_functions.get_recon_method(recon_data['contents']) + recon = helper_functions.get_recon_method(recon_data["contents"]) for key, value in recon_data.items(): - if key != "contents" and key != 'subsets' and key != 'iterations': - if key == 'ReconMethodName': + if key != "contents" and key != "subsets" and key != "iterations": + if key == "ReconMethodName": assert set(value.split(" ")) == set(recon[key].split(" ")) else: assert value == recon[key] @@ -285,8 +320,9 @@ def test_flatten_series(): flatten = helper_functions.flatten_series # create dataframe with empty columns import pandas as pd - empty = pd.Series({'empty': []}) - full = pd.Series({'full': [1, 2, 3, 4]}) + + empty = pd.Series({"empty": []}) + full = pd.Series({"full": [1, 2, 3, 4]}) empty_flattened = flatten(empty) assert empty_flattened == [] @@ -297,14 +333,15 @@ def test_flatten_series(): def test_read_multi_sheet(): open_metadata = helper_functions.open_meta_data multi_spreadsheet = open_metadata(multi_sheet_metadata_file) - assert(multi_spreadsheet['TimeZero'][0] == datetime.time(12,12,12)) - assert(multi_spreadsheet['TracerName'][0] == 'OverrideTracerNameIn0thSheet') + assert multi_spreadsheet["TimeZero"][0] == datetime.time(12, 12, 12) + assert multi_spreadsheet["TracerName"][0] == "OverrideTracerNameIn0thSheet" def test_collect_pet_spreadsheets(): pet_spreadsheet_dir = Path(single_subject_metadata_file).parent pet_spreadsheets = helper_functions.collect_spreadsheets(pet_spreadsheet_dir) - assert(len(pet_spreadsheets) == 3) + assert len(pet_spreadsheets) == 3 + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/pypet2bids/tests/test_library_command_line_interfaces.py b/pypet2bids/tests/test_library_command_line_interfaces.py index 9d90eb15..7801952d 100644 --- a/pypet2bids/tests/test_library_command_line_interfaces.py +++ b/pypet2bids/tests/test_library_command_line_interfaces.py @@ -10,44 +10,52 @@ PET2BIDS_DIR = PYPET2BIDS_DIR.parent # obtain ecat file path -ecat_file_path = PET2BIDS_DIR / 'ecat_validation' / 'ECAT7_multiframe.v.gz' -dicom_source_folder = os.getenv('TEST_DICOM_IMAGE_FOLDER', None) +ecat_file_path = PET2BIDS_DIR / "ecat_validation" / "ECAT7_multiframe.v.gz" +dicom_source_folder = os.getenv("TEST_DICOM_IMAGE_FOLDER", None) if dicom_source_folder: dicom_source_folder = pathlib.Path(dicom_source_folder) if not dicom_source_folder: - dicom_source_folder = PET2BIDS_DIR / 'OpenNeuroPET-Phantoms' / 'source' / 'SiemensBiographPETMR-NRU' + dicom_source_folder = ( + PET2BIDS_DIR + / "OpenNeuroPET-Phantoms" + / "sourcedata" + / "SiemensBiographPETMR-NIMH" + / "AC_TOF" + ) if not dicom_source_folder.exists(): raise FileNotFoundError(dicom_source_folder) -dcm2niix4pet = PYPET2BIDS_DIR / 'pypet2bids' / 'dcm2niix4pet.py' -ecatpet2bids = PYPET2BIDS_DIR / 'pypet2bids' / 'ecat_cli.py' +dcm2niix4pet = PYPET2BIDS_DIR / "pypet2bids" / "dcm2niix4pet.py" +ecatpet2bids = PYPET2BIDS_DIR / "pypet2bids" / "ecat_cli.py" dataset_description_dictionary = { - "_Comment": "This is a very basic example of a dataset description json", - "Name": "PET Brain phantoms", - "BIDSVersion": "1.7.0", - "DatasetType": "raw", - "License": "CC0", - "Authors": [ - "Author1 Surname1", - "Author2 Surname2", - "Author3 Surname3", - "Author4 Middlename4 Surname4", - "Author5 Middlename5 Surname5" - ], - "HowToAcknowledge": "No worries this is fake.", - "ReferencesAndLinks": ["No you aren't getting any", "Don't bother to ask", "Fine, https://fake.fakelink.null"] + "_Comment": "This is a very basic example of a dataset description json", + "Name": "PET Brain phantoms", + "BIDSVersion": "1.7.0", + "DatasetType": "raw", + "License": "CC0", + "Authors": [ + "Author1 Surname1", + "Author2 Surname2", + "Author3 Surname3", + "Author4 Middlename4 Surname4", + "Author5 Middlename5 Surname5", + ], + "HowToAcknowledge": "No worries this is fake.", + "ReferencesAndLinks": [ + "No you aren't getting any", + "Don't bother to ask", + "Fine, https://fake.fakelink.null", + ], } # TODO parse these from pyproject.toml [tool.poetry.scripts] check_for_these_being_installed = { - 'dcm2niix4pet': False, - 'ecatpet2bids': False, - 'pet2bids-spreadsheet-template': False, - 'convert-pmod-to-blood': False, - 'bids-validator': False + "dcm2niix4pet": False, + "ecatpet2bids": False, + "convert-pmod-to-blood": False, } @@ -57,135 +65,106 @@ def test_cli_are_on_path(): check_installed = subprocess.run(f"{cli} -h", shell=True, capture_output=True) if check_installed.returncode == 0: check_for_these_being_installed[cli] = True - assert check_installed.returncode == 0, f"{cli} should be installed and reachable via command line, this will" \ - f"cause further tests to fail." + assert check_installed.returncode == 0, ( + f"{cli} should be installed and reachable via command line, this will" + f"cause further tests to fail." + ) def test_for_show_examples_argument(): - installed_cli = [key for key in check_for_these_being_installed.keys() if check_for_these_being_installed[key] is True] + installed_cli = [ + key + for key in check_for_these_being_installed.keys() + if check_for_these_being_installed[key] is True + ] for installed in installed_cli: - check_for_show_examples = subprocess.run(f"{installed} --show-examples", shell=True, capture_output=True) - assert check_for_show_examples.returncode == 0, f"{installed} does not have a --show-examples option" + check_for_show_examples = subprocess.run( + f"{installed} --show-examples", shell=True, capture_output=True + ) + assert ( + check_for_show_examples.returncode == 0 + ), f"{installed} does not have a --show-examples option" def test_kwargs_produce_valid_conversion(tmp_path): # prepare a set of kwargs (stolen from a valid bids subject/dataset, mum's the word ;) ) full_set_of_kwargs = { - "Modality": "PT", - "Manufacturer": "Siemens", - "ManufacturersModelName": "Biograph 64_mCT", - "InstitutionName": "NIH", - "InstitutionalDepartmentName": "NIMH MIB", - "InstitutionAddress": "10 Center Drive, Bethesda, MD 20892", - "DeviceSerialNumber": "60005", - "StationName": "MIAWP60005", - "PatientPosition": "FFS", - "SoftwareVersions": "VG60A", - "SeriesDescription": "PET Brain Dyn TOF", - "ProtocolName": "PET Brain Dyn TOF", - "ImageType": [ - "ORIGINAL", - "PRIMARY" - ], - "SeriesNumber": 6, - "ScanStart": 2, - "TimeZero": "10:39:46", - "InjectionStart": 0, - "AcquisitionNumber": 2001, - "ImageComments": "Frame 1 of 33^AC_CT_Brain", - "Radiopharmaceutical": "ps13", - "RadionuclidePositronFraction": 0.997669, - "RadionuclideTotalDose": 714840000.0, - "RadionuclideHalfLife": 1220.04, - "DoseCalibrationFactor": 30806700.0, - "ConvolutionKernel": "XYZ Gauss2.00", - "Units": "Bq/mL", - "ReconstructionName": "PSF+TOF", - "ReconstructionParameterUnits": [ - "None", - "None" - ], - "ReconstructionParameterLabels": [ - "subsets", - "iterations" - ], - "ReconstructionParameterValues": [ - 21, - 3 - ], - "ReconFilterType": "XYZ Gauss", - "ReconFilterSize": 2.0, - "AttenuationCorrection": "measured,AC_CT_Brain", - "DecayFactor": [ - 1.00971 - ], - "FrameTimesStart": [ - 0 - ], - "FrameDuration": [ - 30 - ], - "SliceThickness": 2, - "ImageOrientationPatientDICOM": [ - 1, - 0, - 0, - 0, - 1, - 0 - ], - "ConversionSoftware": [ - "dcm2niix", - "pypet2bids" - ], - "ConversionSoftwareVersion": [ - "v1.0.20211006", - "0.0.8" - ], - "TracerName": "[11C]PS13", - "TracerRadionuclide": "11C", - "InjectedRadioactivity": 714840000.0, - "InjectedRadioactivityUnits": "Bq", - "InjectedMass": 5.331647109063877, - "InjectedMassUnits": "nmol", - "SpecificRadioactivity": 341066000000000, - "SpecificRadioactivityUnits": "Bq/mol", - "ModeOfAdministration": "bolus", - "AcquisitionMode": "dynamic", - "ImageDecayCorrected": True, - "ImageDecayCorrectionTime": 0, - "ReconMethodName": "Point-Spread Function + Time Of Flight", - "ReconMethodParameterLabels": [ - "subsets", - "iterations" - ], - "ReconMethodParameterUnits": [ - "none, none" - ], - "ReconMethodParameterValues": [ - 21, - 3 - ], - "Haematocrit": 0.308 + "Modality": "PT", + "Manufacturer": "Siemens", + "ManufacturersModelName": "Biograph 64_mCT", + "InstitutionName": "NIH", + "InstitutionalDepartmentName": "NIMH MIB", + "InstitutionAddress": "10 Center Drive, Bethesda, MD 20892", + "DeviceSerialNumber": "60005", + "StationName": "MIAWP60005", + "PatientPosition": "FFS", + "SoftwareVersions": "VG60A", + "SeriesDescription": "PET Brain Dyn TOF", + "ProtocolName": "PET Brain Dyn TOF", + "ImageType": ["ORIGINAL", "PRIMARY"], + "SeriesNumber": 6, + "ScanStart": 2, + "TimeZero": "10:39:46", + "InjectionStart": 0, + "AcquisitionNumber": 2001, + "ImageComments": "Frame 1 of 33^AC_CT_Brain", + "Radiopharmaceutical": "ps13", + "RadionuclidePositronFraction": 0.997669, + "RadionuclideTotalDose": 714840000.0, + "RadionuclideHalfLife": 1220.04, + "DoseCalibrationFactor": 30806700.0, + "ConvolutionKernel": "XYZ Gauss2.00", + "Units": "Bq/mL", + "ReconstructionName": "PSF+TOF", + "ReconstructionParameterUnits": ["None", "None"], + "ReconstructionParameterLabels": ["subsets", "iterations"], + "ReconstructionParameterValues": [21, 3], + "ReconFilterType": "XYZ Gauss", + "ReconFilterSize": 2.0, + "AttenuationCorrection": "measured,AC_CT_Brain", + "DecayFactor": [1.00971], + "FrameTimesStart": [0], + "FrameDuration": [30], + "SliceThickness": 2, + "ImageOrientationPatientDICOM": [1, 0, 0, 0, 1, 0], + "ConversionSoftware": ["dcm2niix", "pypet2bids"], + "ConversionSoftwareVersion": ["v1.0.20211006", "0.0.8"], + "TracerName": "[11C]PS13", + "TracerRadionuclide": "11C", + "InjectedRadioactivity": 714840000.0, + "InjectedRadioactivityUnits": "Bq", + "InjectedMass": 5.331647109063877, + "InjectedMassUnits": "nmol", + "SpecificRadioactivity": 341066000000000, + "SpecificRadioactivityUnits": "Bq/mol", + "ModeOfAdministration": "bolus", + "AcquisitionMode": "dynamic", + "ImageDecayCorrected": True, + "ImageDecayCorrectionTime": 0, + "ReconMethodName": "Point-Spread Function + Time Of Flight", + "ReconMethodParameterLabels": ["subsets", "iterations"], + "ReconMethodParameterUnits": ["none", "none"], + "ReconMethodParameterValues": [21, 3], + "Haematocrit": 0.308, } # convert kwargs to string - kwargs_string = '' + kwargs_string = "" for key, value in full_set_of_kwargs.items(): - kwargs_string += f'{key}' + '=' + '"' + f'{str(value)}' + '" ' + kwargs_string += f"{key}" + "=" + '"' + f"{str(value)}" + '" ' # test ecat converter - - # create ecat dir ecat_bids_dir = tmp_path / "ecat_test/sub-ecat/ses-test/pet" ecat_bids_dir.mkdir(parents=True, exist_ok=True) # we're going to want a dataset description json at a minimum - dataset_description_path = ecat_bids_dir.parent.parent.parent / "dataset_description.json" - with open(dataset_description_path, 'w') as outfile: + dataset_description_path = ( + ecat_bids_dir.parent.parent.parent / "dataset_description.json" + ) + with open(dataset_description_path, "w") as outfile: json.dump(dataset_description_dictionary, outfile, indent=4) ecat_bids_nifti_path = ecat_bids_dir / "sub-ecat_ses-test_pet.nii" @@ -203,76 +182,123 @@ def test_kwargs_produce_valid_conversion(tmp_path): destination_path = tmp_path / "dicom_test/sub-dicom/ses-test/pet" destination_path.mkdir(parents=True, exist_ok=True) -# dicom_source_folder = os.getenv('TEST_DICOM_IMAGE_FOLDER', None) -# if dicom_source_folder: -# dicom_source_folder = pathlib.Path(dicom_source_folder) -# if not dicom_source_folder: -# dicom_source_folder = PET2BIDS_DIR / 'OpenNeuroPET-Phantoms' / 'source' / 'SiemensBiographPETMR-NRU' -# if not dicom_source_folder.exists(): -# raise FileNotFoundError(dicom_source_folder) - - dataset_description_path = destination_path.parent.parent.parent / 'dataset_description.json' - with open(dataset_description_path, 'w') as outfile: + # dicom_source_folder = os.getenv('TEST_DICOM_IMAGE_FOLDER', None) + # if dicom_source_folder: + # dicom_source_folder = pathlib.Path(dicom_source_folder) + # if not dicom_source_folder: + # dicom_source_folder = PET2BIDS_DIR / 'OpenNeuroPET-Phantoms' / 'source' / 'SiemensBiographPETMR-NRU' + # if not dicom_source_folder.exists(): + # raise FileNotFoundError(dicom_source_folder) + + dataset_description_path = ( + destination_path.parent.parent.parent / "dataset_description.json" + ) + with open(dataset_description_path, "w") as outfile: json.dump(dataset_description_dictionary, outfile, indent=4) # pass parsed objects to dcm2niix4pet convert_dicom_command = f"python {dcm2niix4pet} {dicom_source_folder} --destination-path {destination_path} --kwargs {kwargs_string}" - convert_dicom = subprocess.run(convert_dicom_command, shell=True, capture_output=True) + convert_dicom = subprocess.run( + convert_dicom_command, shell=True, capture_output=True + ) - dicom_cmd = f"bids-validator {destination_path.parent.parent.parent} --ignoreWarnings" + dicom_cmd = ( + f"bids-validator {destination_path.parent.parent.parent} --ignoreWarnings" + ) validate_dicom = subprocess.run(dicom_cmd, shell=True, capture_output=True) assert validate_dicom.returncode == 0, validate_dicom.stdout -def test_spreadsheets_produce_valid_conversion(tmp_path): - +def test_spreadsheets_produce_valid_conversion_dcm2niix4pet(tmp_path): # collect spreadsheets - #single_subject_spreadsheet = PET2BIDS_DIR / 'spreadsheet_conversion/single_subject_sheet/subject_metadata_example.xlsx' - - single_subject_spreadsheet = '/Users/galassiae/Projects/PET2BIDS/spreadsheet_conversion/single_subject_sheet/subject_metadata_example.xlsx' + single_subject_spreadsheet = ( + PET2BIDS_DIR + / "spreadsheet_conversion/single_subject_sheet/subject_metadata_example.xlsx" + ) - dcm2niix4pet_test_dir = tmp_path / 'dcm2niix_spreadsheet_input' + dcm2niix4pet_test_dir = tmp_path / "dcm2niix_spreadsheet_input" dcm2niix4pet_test_dir.mkdir(parents=True, exist_ok=True) - subject_folder = dcm2niix4pet_test_dir / 'sub-singlesubjectspreadsheet' / 'ses-test' / 'pet' + subject_folder = ( + dcm2niix4pet_test_dir / "sub-singlesubjectspreadsheetdicom" / "ses-test" / "pet" + ) cmd = f"python {dcm2niix4pet} {dicom_source_folder} --destination-path {subject_folder} --metadata-path {single_subject_spreadsheet}" run_dcm2niix4pet = subprocess.run(cmd, shell=True, capture_output=True) # copy over dataset_description - dataset_description_path = dcm2niix4pet_test_dir / 'dataset_description.json' - with open(dataset_description_path, 'w') as outfile: + dataset_description_path = dcm2niix4pet_test_dir / "dataset_description.json" + with open(dataset_description_path, "w") as outfile: json.dump(dataset_description_dictionary, outfile, indent=4) validator_cmd = f"bids-validator {dcm2niix4pet_test_dir} --ingnoreWarnings" - validate_dicom_w_spreadsheet = subprocess.run(validator_cmd, shell=True, capture_output=True) + validate_dicom_w_spreadsheet = subprocess.run( + validator_cmd, shell=True, capture_output=True + ) assert validate_dicom_w_spreadsheet.returncode == 0 +def test_spreadsheets_produce_valid_conversion_ecatpet2bids(tmp_path): + # collect spreadsheets + single_subject_spreadsheet = ( + PET2BIDS_DIR + / "spreadsheet_conversion/single_subject_sheet/subject_metadata_example.xlsx" + ) + + ecatpet2bids_test_dir = tmp_path / "ecatpet2bids_spreadsheet_input" + ecatpet2bids_test_dir.mkdir(parents=True, exist_ok=True) + subject_folder = ( + ecatpet2bids_test_dir / "sub-singlesubjectspreadsheetecat" / "ses-test" / "pet" + ) + + cmd = ( + f"python {ecatpet2bids} {ecat_file_path} " + f"--nifti {subject_folder}/sub-singlesubjectspreadsheetecat_ses-test_pet.nii.gz " + f"--metadata-path {single_subject_spreadsheet} " + f"--convert" + ) + + run_ecatpet2bids = subprocess.run(cmd, shell=True, capture_output=True) + + assert run_ecatpet2bids.returncode == 0 + + # copy over dataset_description + dataset_description_path = ecatpet2bids_test_dir / "dataset_description.json" + with open(dataset_description_path, "w") as outfile: + json.dump(dataset_description_dictionary, outfile, indent=4) + + validator_cmd = f"bids-validator {ecatpet2bids_test_dir} --ingnoreWarnings" + validate_ecat_w_spreadsheet = subprocess.run( + validator_cmd, shell=True, capture_output=True + ) + + assert validate_ecat_w_spreadsheet.returncode == 0 + + def test_scanner_params_produce_valid_conversion(tmp_path): # test scanner params txt with ecat conversion - ecatpet2bids_scanner_params_test_dir = tmp_path / 'ecat_scanner_params_test' + ecatpet2bids_scanner_params_test_dir = tmp_path / "ecat_scanner_params_test" ecatpet2bids_scanner_params_test_dir.mkdir(parents=True, exist_ok=True) - subject_folder = ecatpet2bids_scanner_params_test_dir / 'sub-01' / 'ses-testecat' / 'pet' - nifti_path = subject_folder / 'sub-01_ses-testecat_pet.nii' + subject_folder = ( + ecatpet2bids_scanner_params_test_dir / "sub-01" / "ses-testecat" / "pet" + ) + nifti_path = subject_folder / "sub-01_ses-testecat_pet.nii" # dump dataset description at destination path - dataset_descripion_path = ecatpet2bids_scanner_params_test_dir / 'dataset_description.json' - with open(dataset_descripion_path, 'w') as outfile: + dataset_descripion_path = ( + ecatpet2bids_scanner_params_test_dir / "dataset_description.json" + ) + with open(dataset_descripion_path, "w") as outfile: json.dump(dataset_description_dictionary, outfile, indent=4) - #scanner_params = PET2BIDS_DIR / 'tests' / 'scannerparams.txt' - scanner_params = "/Users/galassiae/Projects/PET2BIDS/pypet2bids/tests/scannerparams.txt" + scanner_params = PET2BIDS_DIR / "pypet2bids" / "tests" / "scannerparams.txt" not_full_set_of_kwargs = { "SeriesDescription": "PET Brain Dyn TOF", "ProtocolName": "PET Brain Dyn TOF", - "ImageType": [ - "ORIGINAL", - "PRIMARY" - ], + "ImageType": ["ORIGINAL", "PRIMARY"], "SeriesNumber": 6, "ScanStart": 2, "TimeZero": "10:39:46", @@ -286,40 +312,15 @@ def test_scanner_params_produce_valid_conversion(tmp_path): "DoseCalibrationFactor": 30806700.0, "ConvolutionKernel": "XYZ Gauss2.00", "Units": "Bq/mL", - "ReconstructionParameterUnits": [ - "None", - "None" - ], - "ReconstructionParameterValues": [ - 21, - 3 - ], - "DecayFactor": [ - 1.00971 - ], - "FrameTimesStart": [ - 0 - ], - "FrameDuration": [ - 30 - ], + "ReconstructionParameterUnits": ["None", "None"], + "ReconstructionParameterValues": [21, 3], + "DecayFactor": [1.00971], + "FrameTimesStart": [0], + "FrameDuration": [30], "SliceThickness": 2, - "ImageOrientationPatientDICOM": [ - 1, - 0, - 0, - 0, - 1, - 0 - ], - "ConversionSoftware": [ - "dcm2niix", - "pypet2bids" - ], - "ConversionSoftwareVersion": [ - "v1.0.20211006", - "0.0.8" - ], + "ImageOrientationPatientDICOM": [1, 0, 0, 0, 1, 0], + "ConversionSoftware": ["dcm2niix", "pypet2bids"], + "ConversionSoftwareVersion": ["v1.0.20211006", "0.0.8"], "TracerName": "[11C]PS13", "TracerRadionuclide": "11C", "InjectedRadioactivity": 714840000.0, @@ -330,25 +331,19 @@ def test_scanner_params_produce_valid_conversion(tmp_path): "SpecificRadioactivityUnits": "Bq/mol", "ModeOfAdministration": "bolus", "ImageDecayCorrectionTime": 0, - "ReconMethodParameterLabels": [ - "subsets", - "iterations" - ], - "ReconMethodParameterUnits": [ - "none, none" - ], - "ReconMethodParameterValues": [ - 21, - 3 - ], - "Haematocrit": 0.308 + "ReconMethodParameterLabels": ["subsets", "iterations"], + "ReconMethodParameterUnits": ["none, none"], + "ReconMethodParameterValues": [21, 3], + "Haematocrit": 0.308, } - kwargs_string = '' + kwargs_string = "" for key, value in not_full_set_of_kwargs.items(): - kwargs_string += f'{key}' + '=' + '"' + f'{str(value)}' + '" ' + kwargs_string += f"{key}" + "=" + '"' + f"{str(value)}" + '" ' - ecat_command = f"python {ecatpet2bids} {ecat_file_path} --scannerparams {scanner_params} --convert " \ - f"--nifti {nifti_path} --kwargs {kwargs_string}" + ecat_command = ( + f"python {ecatpet2bids} {ecat_file_path} --scannerparams {scanner_params} --convert " + f"--nifti {nifti_path} --kwargs {kwargs_string}" + ) ecat_run = subprocess.run(ecat_command, shell=True) cmd = f"bids-validator {ecatpet2bids_scanner_params_test_dir}" validate = subprocess.run(cmd, shell=True, capture_output=True) diff --git a/pypet2bids/tests/test_nifti_write.py b/pypet2bids/tests/test_nifti_write.py index eb112a68..09931e03 100644 --- a/pypet2bids/tests/test_nifti_write.py +++ b/pypet2bids/tests/test_nifti_write.py @@ -10,37 +10,46 @@ # collect path to test ecat file dotenv.load_dotenv(dotenv.find_dotenv()) # load path from .env file into env -ecat_path = os.environ['TEST_ECAT_PATH'] -nifti_path = os.environ['OUTPUT_NIFTI_PATH'] +ecat_path = os.environ["TEST_ECAT_PATH"] +nifti_path = os.environ["OUTPUT_NIFTI_PATH"] -if __name__ == '__main__': +if __name__ == "__main__": """ More manual testing of nifti writing functions works for now as there is no public or frozen dataset to test on that is accessible by CI. Requires a .env file with the following fields: - + TEST_ECAT_PATH= OUTPUT_NIFTI_PATH= - + usage: > python test_nifti_write.py - + or more likely this gets run w/ a debugger in an IDE such as pycharm or vs code. - + """ read_and_write = Ecat(ecat_file=ecat_path, nifti_file=nifti_path) - time_zero = datetime.fromtimestamp(read_and_write.ecat_header['DOSE_START_TIME']).strftime('%I:%M:%S') + time_zero = datetime.fromtimestamp( + read_and_write.ecat_header["DOSE_START_TIME"] + ).strftime("%I:%M:%S") read_and_write.populate_sidecar() read_and_write.prune_sidecar() - read_and_write.show_sidecar(os.path.join(os.path.dirname(nifti_path), - os.path.basename(nifti_path),) + '.json') + read_and_write.show_sidecar( + os.path.join( + os.path.dirname(nifti_path), + os.path.basename(nifti_path), + ) + + ".json" + ) # testing how much this image gets mangled w/ nifti conversion # write nifti and save binary of nibabel.Nifti1 object - read_from_ecat_but_not_written = ecat2nii(ecat_file=ecat_path, nifti_file=nifti_path, save_binary=True, sif_out=True) + read_from_ecat_but_not_written = ecat2nii( + ecat_file=ecat_path, nifti_file=nifti_path, save_binary=True, sif_out=True + ) # load pickled object why not? - pickled_nifti = pickle.load(open(nifti_path + ".pickle", 'rb')) + pickled_nifti = pickle.load(open(nifti_path + ".pickle", "rb")) # collect nifti from written out file written_nifti = load(nifti_path) @@ -64,11 +73,3 @@ print("Pickle difference zero.") else: raise Exception("Pickle differs from returned nifti from ecat2nii function.") - - - - - - - - diff --git a/pypet2bids/tests/test_thisbytes.py b/pypet2bids/tests/test_thisbytes.py index 450f9b66..e2688b98 100644 --- a/pypet2bids/tests/test_thisbytes.py +++ b/pypet2bids/tests/test_thisbytes.py @@ -1,41 +1,58 @@ -from pypet2bids.read_ecat import * +import pathlib +import re +import os +from pypet2bids.read_ecat import get_buffer_size, read_ecat, ecat_header_maps from dotenv import load_dotenv parent_dir = pathlib.Path(__file__).parent.resolve() -env_path = parent_dir.parent.resolve().joinpath('.env') +env_path = parent_dir.parent.resolve().joinpath(".env") if __name__ == "__main__": """ - Verifying that the byte positions and widths correspond to their datatype size as + Verifying that the byte positions and widths correspond to their datatype size as stated in ecat_headers.json (this is mostly a sanity check). The most likely error to - occur is from the user during json/schema creation. This test does some simple math - to verify the byte width between, say byte 4 and 8 corresponds to the length of an - integer2. This script is run at the command line and simply grepped to determine if - a mismatch occurs. - + occur is from the user during json/schema creation. This test does some simple math + to verify the byte width between, say byte 4 and 8 corresponds to the length of an + integer2. This script is run at the command line and simply grepped to determine if + a mismatch occurs. + usage: > python test_thisbytes.py | grep Mismatch - + Manual test, but still fairly useful. - + """ check_header_json = True if check_header_json: - for header, header_values in ecat_header_maps['ecat_headers'].items(): + for header, header_values in ecat_header_maps["ecat_headers"].items(): for header_name, header_map in header_values.items(): byte_position = 0 print(header + " " + header_name) for each in header_map: - print(each.get('byte'), each.get('variable_name'), each.get('type'), get_buffer_size(each.get('type'), each.get('variable_name')), byte_position) - if byte_position != each.get('byte'): - print(f"Mismatch in {header} between byte position {each.get('byte')} and calculated position {byte_position}.") + print( + each.get("byte"), + each.get("variable_name"), + each.get("type"), + get_buffer_size(each.get("type"), each.get("variable_name")), + byte_position, + ) + if byte_position != each.get("byte"): + print( + f"Mismatch in {header} between byte position {each.get('byte')} and calculated position {byte_position}." + ) try: - paren_error = re.findall(r'^.*?\([^\d]*(\d+)[^\d]*\).*$', each.get('variable_name')) + paren_error = re.findall( + r"^.*?\([^\d]*(\d+)[^\d]*\).*$", + each.get("variable_name"), + ) except TypeError: pass - byte_position = get_buffer_size(each['type'], each['variable_name']) + byte_position + byte_position = ( + get_buffer_size(each["type"], each["variable_name"]) + + byte_position + ) """ Checking reading of ECAT header and subheader, only works if there exists a ../.env w/ TEST_ECAT_PATH= @@ -47,7 +64,9 @@ load_dotenv(env_path) ecat_test_file = os.environ.get("TEST_ECAT_PATH") - test_main_header, test_subheaders, test_data = read_ecat(ecat_file=ecat_test_file) + test_main_header, test_subheaders, test_data = read_ecat( + ecat_file=ecat_test_file + ) print(f"Main header info:") for k, v in test_main_header.items(): print(f"{k}: {v}") @@ -57,4 +76,4 @@ for k, v in subheader.items(): print(f"{k}: {v}") print(f"Image Data, Dimensions {test_data.shape}, Datatype {test_data.dtype}") - print(test_data) \ No newline at end of file + print(test_data) diff --git a/pypet2bids/tests/test_write_ecat.py b/pypet2bids/tests/test_write_ecat.py index ecb5c213..bedc0279 100644 --- a/pypet2bids/tests/test_write_ecat.py +++ b/pypet2bids/tests/test_write_ecat.py @@ -1,8 +1,18 @@ import os import sys import unittest -from pypet2bids.write_ecat import create_directory_table, write_header, write_directory_table, write_pixel_data -from pypet2bids.read_ecat import read_ecat, get_directory_data, read_bytes, ecat_header_maps +from pypet2bids.write_ecat import ( + create_directory_table, + write_header, + write_directory_table, + write_pixel_data, +) +from pypet2bids.read_ecat import ( + read_ecat, + get_directory_data, + read_bytes, + ecat_header_maps, +) import numpy import dotenv import shutil @@ -10,49 +20,57 @@ dotenv.load_dotenv(dotenv.load_dotenv()) env_vars = os.environ -if env_vars.get('GITHUB_ACTIONS', None): +if env_vars.get("GITHUB_ACTIONS", None): print("Currently running in github actions; not running this test module") os._exit(0) -test_ecat_path = env_vars.get('TEST_ECAT_PATH') +test_ecat_path = env_vars.get("TEST_ECAT_PATH") class TestECATWrite(unittest.TestCase): @classmethod def setUp(cls) -> None: - cls.known_main_header, cls.known_subheaders, cls.known_pixel_data = read_ecat(test_ecat_path, - collect_pixel_data=False) + cls.known_main_header, cls.known_subheaders, cls.known_pixel_data = read_ecat( + test_ecat_path, collect_pixel_data=False + ) # collect directory table from ecat - directory_block = read_bytes(path_to_bytes=test_ecat_path, - byte_start=512, - byte_stop=512) + directory_block = read_bytes( + path_to_bytes=test_ecat_path, byte_start=512, byte_stop=512 + ) cls.known_directory_table = get_directory_data(directory_block, test_ecat_path) - cls.known_directory_table_raw = get_directory_data(directory_block, - test_ecat_path, - return_raw=True) + cls.known_directory_table_raw = get_directory_data( + directory_block, test_ecat_path, return_raw=True + ) cls.pixel_byte_size_int = 2 - cls.temp_file = 'test_tempfile.v' + cls.temp_file = "test_tempfile.v" cls.pixel_dimensions = { - 'x': cls.known_subheaders[0]['X_DIMENSION'], - 'y': cls.known_subheaders[0]['Y_DIMENSION'], - 'z': cls.known_subheaders[0]['Z_DIMENSION'] + "x": cls.known_subheaders[0]["X_DIMENSION"], + "y": cls.known_subheaders[0]["Y_DIMENSION"], + "z": cls.known_subheaders[0]["Z_DIMENSION"], } def test_create_directory_table(self): - generated_directory_table = create_directory_table(self.known_main_header['NUM_FRAMES'], - self.pixel_dimensions, - pixel_byte_size=self.pixel_byte_size_int) + generated_directory_table = create_directory_table( + self.known_main_header["NUM_FRAMES"], + self.pixel_dimensions, + pixel_byte_size=self.pixel_byte_size_int, + ) # dimensions of directory should be 4 x 32 self.assertEqual(generated_directory_table[0].shape, (4, 32)) # data type should be int 32 - self.assertTrue(generated_directory_table[0].dtype == numpy.dtype('>i4')) + self.assertTrue(generated_directory_table[0].dtype == numpy.dtype(">i4")) # assert spacing between dimensions is correct - width = (generated_directory_table[0][2, 1] - generated_directory_table[0][1, 1]) * 512 - calculated_width = \ - self.pixel_dimensions['x'] * self.pixel_dimensions['y'] * \ - self.pixel_dimensions['z'] * self.pixel_byte_size_int + width = ( + generated_directory_table[0][2, 1] - generated_directory_table[0][1, 1] + ) * 512 + calculated_width = ( + self.pixel_dimensions["x"] + * self.pixel_dimensions["y"] + * self.pixel_dimensions["z"] + * self.pixel_byte_size_int + ) self.assertEqual(width, calculated_width) self.assertTrue(generated_directory_table[0][2, 0] == 0) self.assertTrue(generated_directory_table[1][2, 0] == 2) @@ -60,15 +78,16 @@ def test_create_directory_table(self): def test_write_header(self): temp_file = self.temp_file shutil.copy(test_ecat_path, temp_file) - with open(temp_file, 'r+b') as outfile: - schema = ecat_header_maps['ecat_headers']['73']['mainheader'] + with open(temp_file, "r+b") as outfile: + schema = ecat_header_maps["ecat_headers"]["73"]["mainheader"] write_header( - ecat_file=outfile, - schema=schema, - values=self.known_main_header) + ecat_file=outfile, schema=schema, values=self.known_main_header + ) # now read the file w ecat read to see if it's changed. - check_header, check_subheaders, check_pixel_data = read_ecat(temp_file, collect_pixel_data=False) + check_header, check_subheaders, check_pixel_data = read_ecat( + temp_file, collect_pixel_data=False + ) os.remove(temp_file) for key, value in self.known_main_header.items(): @@ -76,74 +95,98 @@ def test_write_header(self): def test_write_directory_table(self): shutil.copy(test_ecat_path, self.temp_file) - with open(self.temp_file, 'r+b') as outfile: + with open(self.temp_file, "r+b") as outfile: # write header - schema = ecat_header_maps['ecat_headers']['73']['mainheader'] - write_header(ecat_file=outfile, - schema=schema, - values=self.known_main_header) + schema = ecat_header_maps["ecat_headers"]["73"]["mainheader"] + write_header( + ecat_file=outfile, schema=schema, values=self.known_main_header + ) file_position_after_main_header_write = outfile.tell() - directory_table = create_directory_table(self.known_main_header['NUM_FRAMES'], - pixel_dimensions=self.pixel_dimensions, - pixel_byte_size=self.pixel_byte_size_int - ) + directory_table = create_directory_table( + self.known_main_header["NUM_FRAMES"], + pixel_dimensions=self.pixel_dimensions, + pixel_byte_size=self.pixel_byte_size_int, + ) write_directory_table(outfile, directory_table) file_position_after_directory_table_write = outfile.tell() # collect newly written directory table from temp file - directory_block = read_bytes(path_to_bytes=self.temp_file, - byte_start=512, - byte_stop=512) - just_written_directory_table = get_directory_data(directory_block, self.temp_file, return_raw=True) + directory_block = read_bytes( + path_to_bytes=self.temp_file, byte_start=512, byte_stop=512 + ) + just_written_directory_table = get_directory_data( + directory_block, self.temp_file, return_raw=True + ) # dimensions of directory should be 4 x 32 self.assertEqual(just_written_directory_table[0].shape, (4, 32)) # data type should be int 32 - self.assertTrue(just_written_directory_table[0].dtype == numpy.dtype('>i4')) + self.assertTrue(just_written_directory_table[0].dtype == numpy.dtype(">i4")) # assert spacing between dimensions is correct - width = (just_written_directory_table[0][2, 1] - just_written_directory_table[0][1, 1]) * 512 - calculated_width = \ - self.pixel_dimensions['x'] * self.pixel_dimensions['y'] * self.pixel_dimensions[ - 'z'] * self.pixel_byte_size_int + width = ( + just_written_directory_table[0][2, 1] + - just_written_directory_table[0][1, 1] + ) * 512 + calculated_width = ( + self.pixel_dimensions["x"] + * self.pixel_dimensions["y"] + * self.pixel_dimensions["z"] + * self.pixel_byte_size_int + ) self.assertEqual(width, calculated_width) self.assertTrue(just_written_directory_table[0][2, 0] == 0) self.assertTrue(just_written_directory_table[1][2, 0] == 2) # assert additional directory table was created at correct byte position if len(directory_table) > 1: - directory_block = read_bytes(path_to_bytes=self.temp_file, - #byte_start=1024, - byte_start=(just_written_directory_table[0][1, 0] - 1) * 512, - byte_stop=512) - additional_directory_table = numpy.frombuffer(directory_block, dtype=numpy.dtype('>i4'), count=-1) - additional_directory_table = numpy.transpose(numpy.reshape(additional_directory_table, (-1, 4))) - numpy.testing.assert_array_equal(additional_directory_table, directory_table[1]) + directory_block = read_bytes( + path_to_bytes=self.temp_file, + # byte_start=1024, + byte_start=(just_written_directory_table[0][1, 0] - 1) * 512, + byte_stop=512, + ) + additional_directory_table = numpy.frombuffer( + directory_block, dtype=numpy.dtype(">i4"), count=-1 + ) + additional_directory_table = numpy.transpose( + numpy.reshape(additional_directory_table, (-1, 4)) + ) + numpy.testing.assert_array_equal( + additional_directory_table, directory_table[1] + ) # check to see if writing to the tempfile broke any other part of the datastructure - check_header, check_subheaders, check_pixel_data = read_ecat(ecat_file=self.temp_file, - collect_pixel_data=True) + check_header, check_subheaders, check_pixel_data = read_ecat( + ecat_file=self.temp_file, collect_pixel_data=True + ) def test_write_pixel_data(self): - self.known_main_header, self.known_subheaders, self.known_pixel_data = read_ecat(test_ecat_path, - collect_pixel_data=True) + self.known_main_header, self.known_subheaders, self.known_pixel_data = ( + read_ecat(test_ecat_path, collect_pixel_data=True) + ) shutil.copy(test_ecat_path, self.temp_file) # locate the first frame in the test file frame_one_start = self.known_directory_table[1, 0] * 512 frame_one_stop = self.known_directory_table[2, 0] * 512 frame_one = self.known_pixel_data[:, :, :, 0] - replacement_frame = numpy.random.randint(32767, size=frame_one.shape, dtype=numpy.uint16) + replacement_frame = numpy.random.randint( + 32767, size=frame_one.shape, dtype=numpy.uint16 + ) - with open(self.temp_file, 'r+b') as outfile: - write_pixel_data(ecat_file=outfile, - pixel_data=replacement_frame, - byte_position=frame_one_start, - seek=True) + with open(self.temp_file, "r+b") as outfile: + write_pixel_data( + ecat_file=outfile, + pixel_data=replacement_frame, + byte_position=frame_one_start, + seek=True, + ) # reread in the pixel data, verify that it has been written - write_pixel_main_header, write_pixel_subheaders, write_pixel_pixel_data = read_ecat(self.temp_file, - collect_pixel_data=True) + write_pixel_main_header, write_pixel_subheaders, write_pixel_pixel_data = ( + read_ecat(self.temp_file, collect_pixel_data=True) + ) written_frame = write_pixel_pixel_data[:, :, :, 0] numpy.testing.assert_array_equal(replacement_frame, written_frame) @@ -156,5 +199,5 @@ def tearDown(cls) -> None: pass -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/scripts/compare_jsons.py b/scripts/compare_jsons.py index f5bc9b75..927fdae3 100644 --- a/scripts/compare_jsons.py +++ b/scripts/compare_jsons.py @@ -5,18 +5,17 @@ import argparse - class Style: - BLACK = '\033[30m' - RED = '\033[31m' - GREEN = '\033[32m' - YELLOW = '\033[33m' - BLUE = '\033[34m' - MAGENTA = '\033[35m' - CYAN = '\033[36m' - WHITE = '\033[37m' - UNDERLINE = '\033[4m' - RESET = '\033[0m' + BLACK = "\033[30m" + RED = "\033[31m" + GREEN = "\033[32m" + YELLOW = "\033[33m" + BLUE = "\033[34m" + MAGENTA = "\033[35m" + CYAN = "\033[36m" + WHITE = "\033[37m" + UNDERLINE = "\033[4m" + RESET = "\033[0m" def fetch_pairs_of_jsons(path_one: Path, path_two: Path): @@ -27,14 +26,17 @@ def fetch_pairs_of_jsons(path_one: Path, path_two: Path): for path in paths.keys(): for root, folders, files in os.walk(path): for f in files: - if Path(f).suffix == '.json': + if Path(f).suffix == ".json": paths[path][f] = os.path.join(root, f) paths = {path_one.name: paths[path_one], path_two.name: paths[path_two]} pairs = [] for sidecar in paths[next(iter(paths))]: - if sidecar in paths[path_one.name].keys() and sidecar in paths[path_two.name].keys(): + if ( + sidecar in paths[path_one.name].keys() + and sidecar in paths[path_two.name].keys() + ): pairs.append(sidecar) to_compare = {} @@ -69,30 +71,48 @@ def compare_jsons(json_paths, show_matches=True): if len(str(left[i])) > left_element_length: left_element_length = len(str(left[i])) - name_padding = name_element_length left_padding = left_element_length - comparison_string = f"\nComparison between {json_files[0]} and {json_files[1]}" print(Style.RESET + comparison_string) # print out a header/column names - header = " keyname".ljust(name_padding, ' ') + " \tleft".ljust(left_padding, ' ') + " \tright" + header = ( + " keyname".ljust(name_padding, " ") + + " \tleft".ljust(left_padding, " ") + + " \tright" + ) print(Style.RESET + header) for i in list(intersection): print(Style.RESET, end="") left_value, right_value = left[i], right[i] - approximate=False + approximate = False if type(left_value) is str and type(right_value) is str: if left_value == right_value and show_matches: - print(Style.GREEN + f"== {i.ljust(name_padding, ' ')}\t{str(left_value).ljust(left_padding, ' ')}\t{right_value}") - elif set(left_value.lower().split(" ")) == set(right_value.lower().split(" ")) and show_matches: - print(Style.YELLOW + f"~= {i.ljust(name_padding, ' ')}\t{str(left_value).ljust(left_padding, ' ')}\t{right_value}") + print( + Style.GREEN + + f"== {i.ljust(name_padding, ' ')}\t{str(left_value).ljust(left_padding, ' ')}\t{right_value}" + ) + elif ( + set(left_value.lower().split(" ")) + == set(right_value.lower().split(" ")) + and show_matches + ): + print( + Style.YELLOW + + f"~= {i.ljust(name_padding, ' ')}\t{str(left_value).ljust(left_padding, ' ')}\t{right_value}" + ) approximate = True if left_value == right_value and show_matches: - print(Style.GREEN + f"== {i.ljust(name_padding, ' ')}\t{str(left_value).ljust(left_padding, ' ')}\t{right_value}") + print( + Style.GREEN + + f"== {i.ljust(name_padding, ' ')}\t{str(left_value).ljust(left_padding, ' ')}\t{right_value}" + ) elif not approximate: - print(Style.RED + f"!= {i.ljust(name_padding, ' ')}\t{str(left_value).ljust(left_padding, ' ')}\t{right_value}") + print( + Style.RED + + f"!= {i.ljust(name_padding, ' ')}\t{str(left_value).ljust(left_padding, ' ')}\t{right_value}" + ) print(Style.RESET, end="") # record where sets differ @@ -103,13 +123,15 @@ def compare_jsons(json_paths, show_matches=True): if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('leftpath', type=Path, help="The first path to examine json files in.") - parser.add_argument('right_path', type=Path, help="The second path to examine json files in.") + parser.add_argument( + "leftpath", type=Path, help="The first path to examine json files in." + ) + parser.add_argument( + "right_path", type=Path, help="The second path to examine json files in." + ) args = parser.parse_args() - x = fetch_pairs_of_jsons( - args.leftpath, args.right_path) - + x = fetch_pairs_of_jsons(args.leftpath, args.right_path) - y = compare_jsons(x) \ No newline at end of file + y = compare_jsons(x) diff --git a/scripts/matlab_conversions.m b/scripts/matlab_conversions.m index 56bcfb71..9d790f3c 100644 --- a/scripts/matlab_conversions.m +++ b/scripts/matlab_conversions.m @@ -51,284 +51,286 @@ message = 'Failed to convert subject, moving onto next.'; -%% Neurobiology Research Unit - Copenhagen -% ---------------------------------------- -% Siemens HRRT -% ------------ -try - clear meta - meta.TimeZero = 'ScanStart'; - meta.Manufacturer = 'Siemens'; - meta.ManufacturersModelName = 'HRRT'; - meta.InstitutionName = 'Rigshospitalet, NRU, DK'; - meta.BodyPart = 'Phantom'; - meta.Units = 'Bq/mL'; - meta.TracerName = 'FDG'; - meta.TracerRadionuclide = 'F18'; - meta.InjectedRadioactivity = 81.24; % Mbq - meta.SpecificRadioactivity = 1.3019e+04; % ~ 81240000 Bq/ 6240 g - meta.ModeOfAdministration = 'infusion'; - meta.AcquisitionMode = 'list mode'; - meta.ImageDecayCorrected = true; % when passing this as string it fails validation - meta.ImageDecayCorrectionTime = 0; - meta.ReconFilterType = 'none'; - meta.ReconFilterSize = 0; - meta.AttenuationCorrection = '10-min transmission scan'; - % meta.FrameDuration = 1200; % not needed, encoded in ecat even for 4D images - % meta.FrameTimesStart = 0; - - out = ecat2nii(fullfile(source,['SiemensHRRT-NRU' filesep 'XCal-Hrrt-2022.04.21.15.43.05_EM_3D.v']),... - meta,'gz',true,'FileListOut',fullfile(destination,['sub-SiemensHRRTNRU' filesep 'pet' filesep 'sub-SiemensHRRTNRU.nii'])); - -catch - disp(message); -end - -% Siemens Biograph -% --------------------------- -try - clear meta - meta.TimeZero = 'ScanStart'; - meta.Manufacturer = 'Siemens'; - meta.ManufacturersModelName = 'Biograph'; - meta.InstitutionName = 'Rigshospitalet, NRU, DK'; - meta.BodyPart = 'Phantom'; - meta.Units = 'Bq/mL'; - meta.TracerName = 'FDG'; - meta.TracerRadionuclide = 'F18'; - meta.InjectedRadioactivity = 81.24; % Mbq - meta.SpecificRadioactivity = 1.3019e+04; % ~ 81240000 Bq/ 6240 g - meta.ModeOfAdministration = 'infusion'; - meta.FrameTimesStart = 0; - meta.AcquisitionMode = 'list mode'; - meta.ImageDecayCorrected = 'true'; - meta.ImageDecayCorrectionTime = 0; - meta.AttenuationCorrection = 'MR-corrected'; - meta.FrameDuration = 300; - meta.FrameTimesStart = 0; - meta.ReconFilterType = "none"; - meta.ReconFilterSize = 1; - - dcm2niix4pet(fullfile(source,'SiemensBiographPETMR-NRU'),meta,... - 'o',fullfile(destination,['sub-SiemensBiographNRU' filesep 'pet'])); % note we only need to use folders here -catch - disp(message); -end - -%% Århus University Hospital -% --------------------------- -try - clear meta - meta.TimeZero = 'ScanStart'; - meta.Manufacturer = 'General Electric'; - meta.ManufacturersModelName = 'Discovery'; - meta.InstitutionName = 'Århus University Hospital, DK'; - meta.BodyPart = 'Phantom'; - meta.Units = 'Bq/mL'; - meta.TracerName = 'FDG'; - meta.TracerRadionuclide = 'F18'; - meta.InjectedRadioactivity = 25.5; % Mbq - meta.SpecificRadioactivity = 4.5213e+03; % ~ 25500000 / 5640 ml - meta.ModeOfAdministration = 'infusion'; - meta.FrameTimesStart = 0; - meta.AcquisitionMode = 'list mode'; - meta.ImageDecayCorrected = 'true'; - meta.ImageDecayCorrectionTime = 0; - meta.AttenuationCorrection = 'MR-corrected'; - meta.FrameDuration = 1200; - meta.FrameTimesStart = 0; - meta.ReconMethodParameterLabels = ["none"]; - meta.ReconMethodParameterUnits = ["none"]; - meta.ReconParameterValues = [0]; - meta.ReconFilterType = "none"; - meta.ReconFilterSize = 0; - - dcm2niix4pet(fullfile(source,'GeneralElectricDiscoveryPETCT-Aarhus'),meta,... - 'o',fullfile(destination,['sub-GeneralElectricDiscoveryAarhus' filesep 'pet'])); % note we only need to use folders here - -catch - disp(message); -end - -try - clear meta - meta.TimeZero = 'ScanStart'; - meta.Manufacturer = 'General Electric'; - meta.ManufacturersModelName = 'Signa PET/MR'; - meta.InstitutionName = 'Århus University Hospital, DK'; - meta.BodyPart = 'Phantom'; - meta.Units = 'Bq/mL'; - meta.TracerName = 'FDG'; - meta.TracerRadionuclide = 'F18'; - meta.InjectedRadioactivity = 21; % Mbq - meta.SpecificRadioactivity = 3.7234e+03; % ~ 21000000 Bq/ 5640 ml - meta.ModeOfAdministration = 'infusion'; - meta.FrameTimesStart = 0; - meta.AcquisitionMode = 'list mode'; - meta.ImageDecayCorrected = 'true'; - meta.ImageDecayCorrectionTime = 0; - meta.AttenuationCorrection = 'MR-corrected'; - meta.FrameDuration = 600; - meta.FrameTimesStart = 0; - meta.ReconFilterType = "none"; - meta.ReconFilterSize = 0; - meta.ReconMethodParameterLabels = ["none"]; - meta.ReconMethodParameterUnits = ["none"]; - meta.ReconMethodParameterValues = [0]; - meta.InjectionEnd = [10]; - - dcm2niix4pet(fullfile(source,'GeneralElectricSignaPETMR-Aarhus'),meta,... - 'o',fullfile(destination,['sub-GeneralElectricSignaAarhus' filesep 'pet'])); % note we only need to use folders here - -catch - disp(message); -end - - -%% Johannes Gutenberg University of Mainz -% -------------------------------------- -% this phillips misbehaves not running it -try - clear meta - meta.TimeZero = 'ScanStart'; - meta.Manufacturer = 'Philips Medical Systems'; - meta.ManufacturersModelName = 'PET/CT Gemini TF16'; - meta.InstitutionName = 'Unimedizin, Mainz, DE'; - meta.BodyPart = 'Phantom'; - meta.Units = 'Bq/mL'; - meta.TracerName = 'Fallypride'; - meta.TracerRadionuclide = 'F18'; - meta.InjectedRadioactivity = 114; % Mbq - meta.SpecificRadioactivity = 800; % ~ 114000000 Bq/ 142500 g - meta.ModeOfAdministration = 'infusion'; %'Intravenous route' - meta.AcquisitionMode = 'list mode'; - meta.ImageDecayCorrected = 'true'; - meta.ImageDecayCorrectionTime = 0; - meta.AttenuationCorrection = 'CTAC-SG'; - meta.ScatterCorrectionMethod = 'SS-SIMUL'; - meta.ReconstructionMethod = 'LOR-RAMLA'; - meta.FrameDuration = 1798; - meta.FrameTimesStart = 0; - meta.ReconFilterType = "none"; - meta.ReconFilterSize = 1; - - dcm2niix4pet(fullfile(source,['PhilipsGeminiPETMR-Unimedizin' filesep 'reqCTAC']),meta,... - 'o',fullfile(destination,['sub-PhilipsGeminiUnimedizinMainz' filesep 'pet'])); - - % meta.AttenuationCorrection = 'NONE'; - % meta.ScatterCorrectionMethod = 'NONE';%GEG - % meta.ReconstructionMethod = '3D-RAMLA';%GEG - % dcm2niix4pet(fullfile(source,['PhilipsGemini-Unimedizin' filesep 'reqNAC']),meta,... - % 'o',fullfile(destination,['sub-PhilipsGeminiNAC-UnimedizinMainz' filesep 'pet'])); - -catch - disp(message); -end - -%% Amsterdam UMC -% --------------------------- - -% Philips Ingenuity PET-CT -% ----------------------- -try - clear meta - meta.TimeZero = 'ScanStart'; - meta.Manufacturer = 'Philips Medical Systems'; - meta.ManufacturersModelName = 'Ingenuity TF PET/CT'; - meta.InstitutionName = 'AmsterdamUMC,VUmc'; - meta.BodyPart = 'Phantom'; - meta.Units = 'Bq/mL'; - meta.TracerName = 'Butanol'; - meta.TracerRadionuclide = 'O15'; - meta.InjectedRadioactivity = 185; % Mbq - meta.SpecificRadioactivity = 1.9907e+04; % ~ 185000000 Bq/ 9293 ml - meta.ModeOfAdministration = 'infusion'; %'Intravenous route' - meta.AcquisitionMode = 'list mode'; - meta.ImageDecayCorrected = 'true'; - meta.ImageDecayCorrectionTime = 0; - meta.AttenuationCorrection = 'CTAC-SG'; - meta.RandomsCorrectionMethod = 'DLYD'; % field added in our library - meta.ScatterCorrectionMethod = 'SS-SIMUL'; % field added in our library - meta.ReconstructionMethod = 'BLOB-OS-TF'; - %meta.FrameDuration = repmat(122.238007,20,1); - meta.FrameTimesStart = 0; - meta.ReconFilterType = "unknown"; - meta.ReconFilterSize = 1; - - dcm2niix4pet(fullfile(source,'PhilipsIngenuityPETCT-AmsterdamUMC'),meta,... - 'o',fullfile(destination,['sub-PhilipsIngenuityPETCTAmsterdamUMC' filesep 'pet'])); -catch - disp(message); -end - -% Philips Ingenuity PET-MRI -% ------------------------- -try - clear meta - meta.TimeZero = 'ScanStart'; - meta.Manufacturer = 'Philips Medical Systems'; - meta.ManufacturersModelName = 'Ingenuity TF PET/MR'; - meta.InstitutionName = 'AmsterdamUMC,VUmc'; - meta.BodyPart = 'Phantom'; - meta.Units = 'Bq/mL'; - meta.TracerName = '11C-PIB'; - meta.TracerRadionuclide = 'C11'; - meta.InjectedRadioactivity = 135.1; % Mbq - meta.SpecificRadioactivity = 1.4538e+04; % ~ 135100000 Bq/ 9293 ml - meta.ModeOfAdministration = 'infusion'; %'Intravenous route' - meta.AcquisitionMode = 'list mode'; - meta.ImageDecayCorrected = 'true'; - meta.ImageDecayCorrectionTime = 0; - meta.ReconFilterType = 'none'; - meta.ReconFilterSize = 0; - meta.AttenuationCorrection = 'MRAC'; - meta.RandomsCorrectionMethod = 'DLYD'; - meta.ScatterCorrectionMethod = 'SS-SIMUL'; - meta.ReconstructionMethod = 'LOR-RAMLA'; - %meta.FrameDuration = repmat(122.238007,40,1); - meta.FrameTimesStart = 0; - meta.ReconFilterType = "unknown"; - meta.ReconFilterSize = 1; - - dcm2niix4pet(fullfile(source,'PhilipsIngenuityPETMR-AmsterdamUMC'),meta,... - 'o',fullfile(destination,['sub-PhilipsIngenuityPETMRAmsterdamUMC' filesep 'pet'])); -catch - disp(message); -end - -% PhillipsVereosPET-CT -% ------------------- -try - clear meta - meta.TimeZero = 'ScanStart'; - meta.Manufacturer = 'Philips Medical Systems'; - meta.ManufacturersModelName = 'Vereos PET/CT'; - meta.InstitutionName = 'AmsterdamUMC,VUmc'; - meta.BodyPart = 'Phantom'; - meta.Units = 'Bq/mL'; - meta.TracerName = '11C-PIB'; - meta.TracerRadionuclide = 'C11'; - meta.InjectedRadioactivity = 202.5; % Mbq - meta.SpecificRadioactivity = 2.1791e+04; % ~ 202500000 Bq/ 9293 ml - meta.ModeOfAdministration = 'infusion'; %'Intravenous route' - meta.AcquisitionMode = 'list mode'; - meta.ImageDecayCorrected = 'true'; - meta.ImageDecayCorrectionTime = 0; - meta.AttenuationCorrection = 'CTAC-SG'; - meta.ScatterCorrectionMethod = 'SS-SIMUL'; - meta.RandomsCorrectionMethod = 'DLYD'; - meta.ReconstructionMethod = 'OSEM:i3s15'; - %meta.FrameDuration = repmat(1221.780029,40,1); - meta.FrameTimesStart = 0; - meta.ReconFilterType = "unknown"; - meta.ReconFilterSize = 1; - - dcm2niix4pet(fullfile(source,'PhillipsVereosPETCT-AmsterdamUMC'),meta,... - 'o',fullfile(destination,['sub-PhillipsVereosAmsterdamUMC' filesep 'pet'])); -catch - disp(message); -end +%%% Neurobiology Research Unit - Copenhagen +%% ---------------------------------------- +% +%% Siemens HRRT +%% ------------ +%try +% clear meta +% meta.TimeZero = 'ScanStart'; +% meta.Manufacturer = 'Siemens'; +% meta.ManufacturersModelName = 'HRRT'; +% meta.InstitutionName = 'Rigshospitalet, NRU, DK'; +% meta.BodyPart = 'Phantom'; +% meta.Units = 'Bq/mL'; +% meta.TracerName = 'FDG'; +% meta.TracerRadionuclide = 'F18'; +% meta.InjectedRadioactivity = 81.24; % Mbq +% meta.SpecificRadioactivity = 1.3019e+04; % ~ 81240000 Bq/ 6240 g +% meta.ModeOfAdministration = 'infusion'; +% meta.AcquisitionMode = 'list mode'; +% meta.ImageDecayCorrected = true; % when passing this as string it fails validation +% meta.ImageDecayCorrectionTime = 0; +% meta.ReconFilterType = 'none'; +% meta.ReconFilterSize = 0; +% meta.AttenuationCorrection = '10-min transmission scan'; +% % meta.FrameDuration = 1200; % not needed, encoded in ecat even for 4D images +% % meta.FrameTimesStart = 0; +% +% out = ecat2nii(fullfile(source,['SiemensHRRT-NRU' filesep 'XCal-Hrrt-2022.04.21.15.43.05_EM_3D.v']),... +% meta,'gz',true,'FileListOut',fullfile(destination,['sub-SiemensHRRTNRU' filesep 'pet' filesep 'sub-SiemensHRRTNRU.nii'])); +% +%catch +% disp(message); +%end +% +%% Siemens Biograph +%% --------------------------- +%try +% clear meta +% meta.TimeZero = 'ScanStart'; +% meta.Manufacturer = 'Siemens'; +% meta.ManufacturersModelName = 'Biograph'; +% meta.InstitutionName = 'Rigshospitalet, NRU, DK'; +% meta.BodyPart = 'Phantom'; +% meta.Units = 'Bq/mL'; +% meta.TracerName = 'FDG'; +% meta.TracerRadionuclide = 'F18'; +% meta.InjectedRadioactivity = 81.24; % Mbq +% meta.SpecificRadioactivity = 1.3019e+04; % ~ 81240000 Bq/ 6240 g +% meta.ModeOfAdministration = 'infusion'; +% meta.FrameTimesStart = 0; +% meta.AcquisitionMode = 'list mode'; +% meta.ImageDecayCorrected = 'true'; +% meta.ImageDecayCorrectionTime = 0; +% meta.AttenuationCorrection = 'MR-corrected'; +% meta.FrameDuration = 300; +% meta.FrameTimesStart = 0; +% meta.ReconFilterType = "none"; +% meta.ReconFilterSize = 1; +% +% dcm2niix4pet(fullfile(source,'SiemensBiographPETMR-NRU'),meta,... +% 'o',fullfile(destination,['sub-SiemensBiographNRU' filesep 'pet'])); % note we only need to use folders here +%catch +% disp(message); +%end +% +%%% Århus University Hospital +%% --------------------------- +%try +% clear meta +% meta.TimeZero = 'ScanStart'; +% meta.Manufacturer = 'General Electric'; +% meta.ManufacturersModelName = 'Discovery'; +% meta.InstitutionName = 'Århus University Hospital, DK'; +% meta.BodyPart = 'Phantom'; +% meta.Units = 'Bq/mL'; +% meta.TracerName = 'FDG'; +% meta.TracerRadionuclide = 'F18'; +% meta.InjectedRadioactivity = 25.5; % Mbq +% meta.SpecificRadioactivity = 4.5213e+03; % ~ 25500000 / 5640 ml +% meta.ModeOfAdministration = 'infusion'; +% meta.FrameTimesStart = 0; +% meta.AcquisitionMode = 'list mode'; +% meta.ImageDecayCorrected = 'true'; +% meta.ImageDecayCorrectionTime = 0; +% meta.AttenuationCorrection = 'MR-corrected'; +% meta.FrameDuration = 1200; +% meta.FrameTimesStart = 0; +% meta.ReconMethodParameterLabels = ["none"]; +% meta.ReconMethodParameterUnits = ["none"]; +% meta.ReconParameterValues = [0]; +% meta.ReconFilterType = "none"; +% meta.ReconFilterSize = 0; +% +% dcm2niix4pet(fullfile(source,'GeneralElectricDiscoveryPETCT-Aarhus'),meta,... +% 'o',fullfile(destination,['sub-GeneralElectricDiscoveryAarhus' filesep 'pet'])); % note we only need to use folders here +% +%catch +% disp(message); +%end +% +%try +% clear meta +% meta.TimeZero = 'ScanStart'; +% meta.Manufacturer = 'General Electric'; +% meta.ManufacturersModelName = 'Signa PET/MR'; +% meta.InstitutionName = 'Århus University Hospital, DK'; +% meta.BodyPart = 'Phantom'; +% meta.Units = 'Bq/mL'; +% meta.TracerName = 'FDG'; +% meta.TracerRadionuclide = 'F18'; +% meta.InjectedRadioactivity = 21; % Mbq +% meta.SpecificRadioactivity = 3.7234e+03; % ~ 21000000 Bq/ 5640 ml +% meta.ModeOfAdministration = 'infusion'; +% meta.FrameTimesStart = 0; +% meta.AcquisitionMode = 'list mode'; +% meta.ImageDecayCorrected = 'true'; +% meta.ImageDecayCorrectionTime = 0; +% meta.AttenuationCorrection = 'MR-corrected'; +% meta.FrameDuration = 600; +% meta.FrameTimesStart = 0; +% meta.ReconFilterType = "none"; +% meta.ReconFilterSize = 0; +% meta.ReconMethodParameterLabels = ["none"]; +% meta.ReconMethodParameterUnits = ["none"]; +% meta.ReconMethodParameterValues = [0]; +% meta.InjectionEnd = [10]; +% +% dcm2niix4pet(fullfile(source,'GeneralElectricSignaPETMR-Aarhus'),meta,... +% 'o',fullfile(destination,['sub-GeneralElectricSignaAarhus' filesep 'pet'])); % note we only need to use folders here +% +%catch +% disp(message); +%end +% +% +%%% Johannes Gutenberg University of Mainz +%% -------------------------------------- +%% this phillips misbehaves not running it +%try +% clear meta +% meta.TimeZero = 'ScanStart'; +% meta.Manufacturer = 'Philips Medical Systems'; +% meta.ManufacturersModelName = 'PET/CT Gemini TF16'; +% meta.InstitutionName = 'Unimedizin, Mainz, DE'; +% meta.BodyPart = 'Phantom'; +% meta.Units = 'Bq/mL'; +% meta.TracerName = 'Fallypride'; +% meta.TracerRadionuclide = 'F18'; +% meta.InjectedRadioactivity = 114; % Mbq +% meta.SpecificRadioactivity = 800; % ~ 114000000 Bq/ 142500 g +% meta.ModeOfAdministration = 'infusion'; %'Intravenous route' +% meta.AcquisitionMode = 'list mode'; +% meta.ImageDecayCorrected = 'true'; +% meta.ImageDecayCorrectionTime = 0; +% meta.AttenuationCorrection = 'CTAC-SG'; +% meta.ScatterCorrectionMethod = 'SS-SIMUL'; +% meta.ReconstructionMethod = 'LOR-RAMLA'; +% meta.FrameDuration = 1798; +% meta.FrameTimesStart = 0; +% meta.ReconFilterType = "none"; +% meta.ReconFilterSize = 1; +% +% %dcm2niix4pet(fullfile(source,['PhilipsGeminiPETMR-Unimedizin' filesep 'reqCTAC']),meta,... +% % 'o',fullfile(destination,['sub-PhilipsGeminiUnimedizinMainz' filesep 'pet'])); +% +% +% % meta.AttenuationCorrection = 'NONE'; +% % meta.ScatterCorrectionMethod = 'NONE';%GEG +% % meta.ReconstructionMethod = '3D-RAMLA';%GEG +% % dcm2niix4pet(fullfile(source,['PhilipsGemini-Unimedizin' filesep 'reqNAC']),meta,... +% % 'o',fullfile(destination,['sub-PhilipsGeminiNAC-UnimedizinMainz' filesep 'pet'])); +% +%catch +% disp(message); +%end +% +%%% Amsterdam UMC +%% --------------------------- +% +%% Philips Ingenuity PET-CT +%% ----------------------- +%try +% clear meta +% meta.TimeZero = 'ScanStart'; +% meta.Manufacturer = 'Philips Medical Systems'; +% meta.ManufacturersModelName = 'Ingenuity TF PET/CT'; +% meta.InstitutionName = 'AmsterdamUMC,VUmc'; +% meta.BodyPart = 'Phantom'; +% meta.Units = 'Bq/mL'; +% meta.TracerName = 'Butanol'; +% meta.TracerRadionuclide = 'O15'; +% meta.InjectedRadioactivity = 185; % Mbq +% meta.SpecificRadioactivity = 1.9907e+04; % ~ 185000000 Bq/ 9293 ml +% meta.ModeOfAdministration = 'infusion'; %'Intravenous route' +% meta.AcquisitionMode = 'list mode'; +% meta.ImageDecayCorrected = 'true'; +% meta.ImageDecayCorrectionTime = 0; +% meta.AttenuationCorrection = 'CTAC-SG'; +% meta.RandomsCorrectionMethod = 'DLYD'; % field added in our library +% meta.ScatterCorrectionMethod = 'SS-SIMUL'; % field added in our library +% meta.ReconstructionMethod = 'BLOB-OS-TF'; +% %meta.FrameDuration = repmat(122.238007,20,1); +% meta.FrameTimesStart = 0; +% meta.ReconFilterType = "unknown"; +% meta.ReconFilterSize = 1; +% +% dcm2niix4pet(fullfile(source,'PhilipsIngenuityPETCT-AmsterdamUMC'),meta,... +% 'o',fullfile(destination,['sub-PhilipsIngenuityPETCTAmsterdamUMC' filesep 'pet'])); +%catch +% disp(message); +%end +% +%% Philips Ingenuity PET-MRI +%% ------------------------- +%try +% clear meta +% meta.TimeZero = 'ScanStart'; +% meta.Manufacturer = 'Philips Medical Systems'; +% meta.ManufacturersModelName = 'Ingenuity TF PET/MR'; +% meta.InstitutionName = 'AmsterdamUMC,VUmc'; +% meta.BodyPart = 'Phantom'; +% meta.Units = 'Bq/mL'; +% meta.TracerName = '11C-PIB'; +% meta.TracerRadionuclide = 'C11'; +% meta.InjectedRadioactivity = 135.1; % Mbq +% meta.SpecificRadioactivity = 1.4538e+04; % ~ 135100000 Bq/ 9293 ml +% meta.ModeOfAdministration = 'infusion'; %'Intravenous route' +% meta.AcquisitionMode = 'list mode'; +% meta.ImageDecayCorrected = 'true'; +% meta.ImageDecayCorrectionTime = 0; +% meta.ReconFilterType = 'none'; +% meta.ReconFilterSize = 0; +% meta.AttenuationCorrection = 'MRAC'; +% meta.RandomsCorrectionMethod = 'DLYD'; +% meta.ScatterCorrectionMethod = 'SS-SIMUL'; +% meta.ReconstructionMethod = 'LOR-RAMLA'; +% %meta.FrameDuration = repmat(122.238007,40,1); +% meta.FrameTimesStart = 0; +% meta.ReconFilterType = "unknown"; +% meta.ReconFilterSize = 1; +% +% dcm2niix4pet(fullfile(source,'PhilipsIngenuityPETMR-AmsterdamUMC'),meta,... +% 'o',fullfile(destination,['sub-PhilipsIngenuityPETMRAmsterdamUMC' filesep 'pet'])); +%catch +% disp(message); +%end +% +%% PhillipsVereosPET-CT +%% ------------------- +%try +% clear meta +% meta.TimeZero = 'ScanStart'; +% meta.Manufacturer = 'Philips Medical Systems'; +% meta.ManufacturersModelName = 'Vereos PET/CT'; +% meta.InstitutionName = 'AmsterdamUMC,VUmc'; +% meta.BodyPart = 'Phantom'; +% meta.Units = 'Bq/mL'; +% meta.TracerName = '11C-PIB'; +% meta.TracerRadionuclide = 'C11'; +% meta.InjectedRadioactivity = 202.5; % Mbq +% meta.SpecificRadioactivity = 2.1791e+04; % ~ 202500000 Bq/ 9293 ml +% meta.ModeOfAdministration = 'infusion'; %'Intravenous route' +% meta.AcquisitionMode = 'list mode'; +% meta.ImageDecayCorrected = 'true'; +% meta.ImageDecayCorrectionTime = 0; +% meta.AttenuationCorrection = 'CTAC-SG'; +% meta.ScatterCorrectionMethod = 'SS-SIMUL'; +% meta.RandomsCorrectionMethod = 'DLYD'; +% meta.ReconstructionMethod = 'OSEM:i3s15'; +% %meta.FrameDuration = repmat(1221.780029,40,1); +% meta.FrameTimesStart = 0; +% meta.ReconFilterType = "unknown"; +% meta.ReconFilterSize = 1; +% +% dcm2niix4pet(fullfile(source,'PhillipsVereosPETCT-AmsterdamUMC'),meta,... +% 'o',fullfile(destination,['sub-PhillipsVereosAmsterdamUMC' filesep 'pet'])); +%catch +% disp(message); +%end %% National Institute of Mental Health, Bethesda % ---------------------------------------------- @@ -449,11 +451,34 @@ disp(message); end -% -% CPS Innovations HRRT - dcm2niix fails -% --------------------------------------- -% dcm2niix4pet(fullfile(source,'CPSInnovationsHRRT-NIMH'),meta,... -% 'o',fullfile(destination,['sub-CPSInnovationsHRRT-NIMH' filesep 'pet'])); +% Siemens HRRT +% ------------ +try + clear meta + meta.TimeZero = 'ScanStart'; + meta.Manufacturer = 'Siemens'; + meta.ManufacturersModelName = 'HRRT'; + meta.InstitutionName = 'JHU'; + meta.BodyPart = 'Phantom'; + meta.Units = 'Bq/mL'; + meta.TracerName = 'FDG'; + meta.TracerRadionuclide = 'F18'; + meta.InjectedRadioactivity = 81.24; % Mbq + meta.SpecificRadioactivity = 1.3019e+04; % ~ 81240000 Bq/ 6240 g + meta.ModeOfAdministration = 'infusion'; + meta.AcquisitionMode = 'list mode'; + meta.ImageDecayCorrected = true; % when passing this as string it fails validation + meta.ImageDecayCorrectionTime = 0; + meta.ReconFilterType = 'none'; + meta.ReconFilterSize = 0; + meta.AttenuationCorrection = '10-min transmission scan'; + meta.ScatterFraction = 0.0; + meta.PromptRate = 0.0; + meta.RandomsRate = 0.0; + meta.SinglesRate = 0.0; + + out = ecat2nii(fullfile(source,['SiemensHRRT-JHU' filesep 'Hoffman.v']),... + meta,'gz',true,'FileListOut',fullfile(destination,['sub-SiemensHRRTJHU' filesep 'pet' filesep 'sub-SiemensHRRTJHU.nii'])); %% Johns Hopkins University @@ -574,4 +599,4 @@ catch disp(message); -end \ No newline at end of file +end diff --git a/scripts/python_conversions.sh b/scripts/python_conversions.sh index 2ca17816..e10e2df9 100644 --- a/scripts/python_conversions.sh +++ b/scripts/python_conversions.sh @@ -9,7 +9,6 @@ # --kwargs accepts arguments passed to in in the form of JS or Python types: int, float, string, list/array. Where lists/arrays should be # wrapped in double quotes. - # set paths where the repo is parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) repo_path=$( cd "$(dirname "${parent_path}")" ; pwd -P ) @@ -22,276 +21,258 @@ echo "SOURCE_FOLDER ${SOURCE_FOLDER}, DESTINATION ${DESTINATION}" mkdir $DESTINATION cp $repo_path/dataset_description.json $DESTINATION/dataset_description.json -# Neurobiology Research Unit - Copenhagen -# ---------------------------------------- - -# Siemens HRRT -# ------------ -echo "${SOURCE_FOLDER}/SiemensHRRT-NRU/XCal-Hrrt-2022.04.21.15.43.05_EM_3D.v" -ecatpet2bids $SOURCE_FOLDER/SiemensHRRT-NRU/XCal-Hrrt-2022.04.21.15.43.05_EM_3D.v --nifti $DESTINATION/sub-SiemensHRRTNRU/pet/sub-SiemensHRRTNRU_pet.nii --convert --kwargs \ -Manufacturer=Siemens \ -ManufacturersModelName=HRRT \ -InstitutionName="Rigshospitalet, NRU, DK" \ -BodyPart="Phantom" \ -Units="Bq/mL" \ -TracerName=FDG \ -TracerRadionuclide=F18 \ -InjectedRadioactivity=81.24 \ -SpecificRadioactivity="1.3019e+04" \ -ModeOfAdministration=infusion \ -AcquisitionMode="list mode" \ -ImageDecayCorrected="true" \ -ImageDecayCorrectionTime=0 \ -ReconFilterSize=0 \ -AttenuationCorrection="10-min transmission scan" \ -SpecificRadioactivityUnits="Bq" \ -ScanStart=0 \ -InjectionStart=0 \ -InjectedRadioactivityUnits='Bq' \ -ReconFilterType="none" \ -#TimeZero="10:10:10" \ -#InjectedMass=1 \ -#InjectedMassUnits=g \ - -# Siemens Biograph -# --------------------------- -echo "${SOURCE_FOLDER}/SiemensBiographPETMR-NRU" -dcm2niix4pet $SOURCE_FOLDER/SiemensBiographPETMR-NRU --destination-path $DESTINATION/sub-SiemensBiographNRU/pet --kwargs \ -Manufacturer=Siemens \ -ManufacturersModelName=Biograph \ -InstitutionName="Rigshospitalet, NRU, DK" \ -BodyPart=Phantom \ -Units="Bq/mL" \ -TracerName="FDG" \ -TracerRadionuclide="F18" \ -InjectedRadioactivity=81.24 \ -SpecificRadioactivity=1.3019e+04 \ -ModeOfAdministration="infusion" \ -AcquisitionMode="list mode" \ -FrameTimesStart="[0]" \ -FrameDuration=[300] \ -ImageDecayCorrected="true" \ -ImageDecayCorrectionTime=0 \ -DecayCorrectionFactor=[1] \ -AttenuationCorrection="MR-corrected" \ -InjectionStart=0 \ ---silent - - -# Århus University Hospital -# --------------------------- -echo "${SOURCE_FOLDER}/GeneralElectricDiscoveryPETCT-Aarhus" -dcm2niix4pet $SOURCE_FOLDER/GeneralElectricDiscoveryPETCT-Aarhus --destination-path $DESTINATION/sub-GeneralElectricDiscoveryAarhus/pet --kwargs \ -Manufacturer="General Electric" \ -ManufacturersModelName="Discovery" \ -InstitutionName="Århus University Hospital, DK" \ -BodyPart="Phantom" \ -Units="Bq/mL" \ -TracerName="FDG" \ -TracerRadionuclide="F18" \ -InjectedRadioactivity=25.5 \ -SpecificRadioactivity=4.5213e+03 \ -ModeOfAdministration="infusion" \ -AcquisitionMode="list mode" \ -ImageDecayCorrected=True \ -ImageDecayCorrectionTime=0 \ -AttenuationCorrection="MR-corrected" \ -FrameDuration=[1200] \ -ReconFilterSize=0 \ -ReconFilterType='none' \ -FrameTimesStart=[0] \ -ReconMethodParameterLabels="[none]" \ -ReconMethodParameterUnits="[none]" \ -ReconMethodParameterValues="[0]" \ ---silent -#DecayCorrectionFactor="[1]" \ - -echo "${SOURCE_FOLDER}/GeneralElectricSignaPETMR-Aarhus" -dcm2niix4pet $SOURCE_FOLDER/GeneralElectricSignaPETMR-Aarhus --destination-path $DESTINATION/sub-GeneralElectricSignaAarhus/pet \ ---kwargs \ -Manufacturer="General Electric" \ -ManufacturersModelName="Signa PETMR" \ -InstitutionName="Århus University Hospital, DK" \ -BodyPart="Phantom" \ -Units="Bq/mL" \ -TracerName="FDG" \ -TracerRadionuclide="F18" \ -InjectedRadioactivity=21 \ -SpecificRadioactivity=3.7234e+03 \ -ModeOfAdministration="infusion" \ -FrameDuration=[600] \ -FrameTimesStart=[0] \ -AcquisitionMode="list mode" \ -ImageDecayCorrected="true" \ -ImageDecayCorrectionTime=0 \ -AttenuationCorrection="MR-corrected" \ -ReconFilterType='unknown' \ -ReconFilterSize=1 \ -ReconMethodParameterLabels="[none, none]" \ -ReconMethodParameterUnits="[none, none]" \ -ReconMethodParameterValues="[0, 0]" \ ---silent - -# Johannes Gutenberg University of Mainz -# -------------------------------------- - -# PhilipsGeminiPETMR -# -------------------------------------- - -echo "${SOURCE_FOLDER}/PhilipsGeminiPETMR-Unimedizin/reqCTAC" -dcm2niix4pet $SOURCE_FOLDER/PhilipsGeminiPETMR-Unimedizin/reqCTAC --destination-path $DESTINATION/sub-PhilipsGeminiUnimedizinMainz/pet \ ---kwargs \ -Manufacturer="Philips Medical Systems" \ -ManufacturersModelName="PET/CT Gemini TF16" \ -InstitutionName="Unimedizin, Mainz, DE" \ -BodyPart="Phantom" \ -Units="Bq/mL" \ -TracerName="Fallypride" \ -TracerRadionuclide="F18" \ -InjectedRadioactivity=114 \ -SpecificRadioactivity=800 \ -ModeOfAdministration="infusion" \ -AcquisitionMode="list mode" \ -ImageDecayCorrected=True \ -ImageDecayCorrectionTime=0 \ -ReconFilterType='n/a' \ -ReconFilterSize=0 \ -AttenuationCorrection="CTAC-SG" \ -ScatterCorrectionMethod="SS-SIMUL" \ -ReconstructionMethod="LOR-RAMLA" \ -ReconMethodParameterValues="[1,1]" \ -FrameDuration=[1798] \ -ReconMethodParameterLabels="[none, none]" \ -ReconMethodParameterUnits="[none, none]" \ -FrameTimesStart=[0] \ ---silent - -echo "${SOURCE_FOLDER}/PhilipsGeminiPETMR-Unimedizin/reqNAC" -dcm2niix4pet $SOURCE_FOLDER/PhilipsGeminiPETMR-Unimedizin/reqNAC --destination-path $DESTINATION/sub-PhilipsGeminiNACUnimedizinMainz/pet \ ---kwargs \ -Manufacturer="Philips Medical Systems" \ -ManufacturersModelName="PET/CT Gemini TF16" \ -InstitutionName="Unimedizin, Mainz, DE" \ -BodyPart="Phantom" \ -Units="Bq/mL" \ -TracerName="Fallypride" \ -TracerRadionuclide="F18" \ -InjectedRadioactivity=114 \ -SpecificRadioactivity=800 \ -ModeOfAdministration="infusion" \ -AcquisitionMode="list mode" \ -ImageDecayCorrected=True \ -ImageDecayCorrectionTime=0 \ -ReconFilterType=None \ -ReconFilterSize=0 \ -AttenuationCorrection="None" \ -ScatterCorrectionMethod="None" \ -ReconstructionMethod="3D-RAMLA" \ -FrameDuration=[1798] \ -ReconMethodParameterLabels="[none, none]" \ -ReconMethodParameterUnits="[none, none]" \ -ReconMethodParameterValues="[1,1]" \ -FrameTimesStart=[0] \ ---silent - - -# Amsterdam UMC -# --------------------------- - -# Philips Ingenuity PET-CT -# ----------------------- -echo "${SOURCE_FOLDER}/PhilipsIngenuityPETCT-AmsterdamUMC" -dcm2niix4pet $SOURCE_FOLDER/PhilipsIngenuityPETCT-AmsterdamUMC --destination-path $DESTINATION/sub-PhilipsIngenuityPETCTAmsterdamUMC/pet \ ---kwargs \ -Manufacturer="Philips Medical Systems" \ -ManufacturersModelName="Ingenuity TF PET/CT" \ -InstitutionName="AmsterdamUMC,VUmc" \ -BodyPart="Phantom" \ -Units="Bq/mL" \ -TracerName="Butanol" \ -TracerRadionuclide="O15" \ -InjectedRadioactivity=185 \ -SpecificRadioactivity=1.9907e+04 \ -ModeOfAdministration="infusion" \ -AcquisitionMode="list mode" \ -ImageDecayCorrected="true" \ -ImageDecayCorrectionTime=0 \ -DecayCorrectionFactor=[1] \ -ReconFilterSize=0 \ -ReconMethodParameterValues=[1] \ -AttenuationCorrection="CTAC-SG" \ -RandomsCorrectionMethod="DLYD" \ -ScatterCorrectionMethod="SS-SIMUL" \ -ReconstructionMethod="BLOB-OS-TF" \ -ReconMethodParameterLabels="[none, none]" \ -ReconMethodParameterUnits="[none, none]" \ -ReconMethodParameterValues="[0, 0]" \ -ReconFilterType="none" \ ---silent -#FrameTimesStart=[0] -#TimeZero="12:12:12" \ - - -# Philips Ingenuity PET-MRI -# ------------------------- -echo "${SOURCE_FOLDER}/PhilipsIngenuityPETMR-AmsterdamUMC" -dcm2niix4pet $SOURCE_FOLDER/PhilipsIngenuityPETMR-AmsterdamUMC --destination-path $DESTINATION/sub-PhilipsIngenuityPETMRAmsterdamUMC/pet \ ---kwargs \ -Manufacturer="Philips Medical Systems" \ -ManufacturersModelName="Ingenuity TF PET/MR" \ -InstitutionName="AmsterdamUMC,VUmc" \ -BodyPart="Phantom" \ -Units="Bq/mL" \ -TracerName="11C-PIB" \ -TracerRadionuclide="C11" \ -InjectedRadioactivity=135.1 \ -SpecificRadioactivity=1.4538e+04 \ -ModeOfAdministration="infusion" \ -AcquisitionMode="list mode" \ -ImageDecayCorrected="True" \ -ImageDecayCorrectionTime=0 \ -ReconFilterType="None" \ -ReconFilterSize=0 \ -AttenuationCorrection="MRAC" \ -RandomsCorrectionMethod="DLYD" \ -ScatterCorrectionMethod="SS-SIMUL" \ -ReconstructionMethod="LOR-RAMLA" \ -ReconMethodParameterValues="[1, 1]" \ -ReconFilterType="['n/a', 'n/a']" \ -DecayCorrectionFactor=[1] \ -ReconMethodParameterLabels="[none, none]" \ -ReconMethodParameterUnits="[none, none]" \ -ReconMethodParameterValues="[0, 0]" \ ---silent - -# philipsVereosPET-CT -# ------------------- -echo "${SOURCE_FOLDER}/PhilipsVereosPETCT-AmsterdamUMC" -dcm2niix4pet $SOURCE_FOLDER/PhillipsVereosPETCT-AmsterdamUMC --destination-path $DESTINATION/sub-PhillipsVereosAmsterdamUMC/pet \ ---kwargs \ -Manufacturer="Philips Medical Systems" \ -ManufacturersModelName="Vereos PET/CT" \ -InstitutionName="AmsterdamUMC,VUmc" \ -BodyPart="Phantom" \ -Units="Bq/mL" \ -TracerName="11C-PIB" \ -TracerRadionuclide="C11" \ -InjectedRadioactivity=202.5 \ -SpecificRadioactivity=2.1791e+04 \ -ModeOfAdministration="infusion" \ -AcquisitionMode="list mode" \ -ImageDecayCorrected="True" \ -ImageDecayCorrectionTime=0 \ -ReconFilterType="None" \ -ReconFilterSize=0 \ -AttenuationCorrection="CTAC-SG" \ -ScatterCorrectionMethod="SS-SIMUL" \ -RandomsCorrectionMethod="DLYD" \ -ReconstructionMethod="OSEMi3s15" \ -TimeZero="11:40:24" \ ---silent -#FrameDuration=[1221] - +## Neurobiology Research Unit - Copenhagen +## ---------------------------------------- +# +## Siemens HRRT +## ------------ +#echo "${SOURCE_FOLDER}/SiemensHRRT-NRU/XCal-Hrrt-2022.04.21.15.43.05_EM_3D.v" +#ecatpet2bids $SOURCE_FOLDER/SiemensHRRT-NRU/XCal-Hrrt-2022.04.21.15.43.05_EM_3D.v --nifti $DESTINATION/sub-SiemensHRRTNRU/pet/sub-SiemensHRRTNRU_pet.nii --convert --kwargs \ +#Manufacturer=Siemens \ +#ManufacturersModelName=HRRT \ +#InstitutionName="Rigshospitalet, NRU, DK" \ +#BodyPart="Phantom" \ +#Units="Bq/mL" \ +#TracerName=FDG \ +#TracerRadionuclide=F18 \ +#InjectedRadioactivity=81.24 \ +#SpecificRadioactivity="1.3019e+04" \ +#ModeOfAdministration=infusion \ +#AcquisitionMode="list mode" \ +#ImageDecayCorrected="true" \ +#ImageDecayCorrectionTime=0 \ +#ReconFilterSize=0 \ +#AttenuationCorrection="10-min transmission scan" \ +#SpecificRadioactivityUnits="Bq" \ +#ScanStart=0 \ +#InjectionStart=0 \ +#InjectedRadioactivityUnits='Bq' \ +#ReconFilterType="none" +# +## Siemens Biograph +## --------------------------- +#echo "${SOURCE_FOLDER}/SiemensBiographPETMR-NRU" +#dcm2niix4pet $SOURCE_FOLDER/SiemensBiographPETMR-NRU --destination-path $DESTINATION/sub-SiemensBiographNRU/pet --kwargs \ +#Manufacturer=Siemens \ +#ManufacturersModelName=Biograph \ +#InstitutionName="Rigshospitalet, NRU, DK" \ +#BodyPart=Phantom \ +#Units="Bq/mL" \ +#TracerName="FDG" \ +#TracerRadionuclide="F18" \ +#InjectedRadioactivity=81.24 \ +#SpecificRadioactivity=1.3019e+04 \ +#ModeOfAdministration="infusion" \ +#AcquisitionMode="list mode" \ +#FrameTimesStart="[0]" \ +#FrameDuration=[300] \ +#ImageDecayCorrected="true" \ +#ImageDecayCorrectionTime=0 \ +#DecayCorrectionFactor=[1] \ +#AttenuationCorrection="MR-corrected" \ +#InjectionStart=0 +# +## Århus University Hospital +## --------------------------- +#echo "${SOURCE_FOLDER}/GeneralElectricDiscoveryPETCT-Aarhus" +#dcm2niix4pet $SOURCE_FOLDER/GeneralElectricDiscoveryPETCT-Aarhus --destination-path $DESTINATION/sub-GeneralElectricDiscoveryAarhus/pet --kwargs \ +#Manufacturer="General Electric" \ +#ManufacturersModelName="Discovery" \ +#InstitutionName="Århus University Hospital, DK" \ +#BodyPart="Phantom" \ +#Units="Bq/mL" \ +#TracerName="FDG" \ +#TracerRadionuclide="F18" \ +#InjectedRadioactivity=25.5 \ +#SpecificRadioactivity=4.5213e+03 \ +#ModeOfAdministration="infusion" \ +#AcquisitionMode="list mode" \ +#ImageDecayCorrected=True \ +#ImageDecayCorrectionTime=0 \ +#AttenuationCorrection="MR-corrected" \ +#FrameDuration=[1200] \ +#ReconFilterSize=0 \ +#ReconFilterType='none' \ +#FrameTimesStart=[0] \ +#ReconMethodParameterLabels="[none]" \ +#ReconMethodParameterUnits="[none]" \ +#ReconMethodParameterValues="[0]" +# +#echo "${SOURCE_FOLDER}/GeneralElectricSignaPETMR-Aarhus" +#dcm2niix4pet $SOURCE_FOLDER/GeneralElectricSignaPETMR-Aarhus --destination-path $DESTINATION/sub-GeneralElectricSignaAarhus/pet \ +#--kwargs \ +#Manufacturer="General Electric" \ +#ManufacturersModelName="Signa PETMR" \ +#InstitutionName="Århus University Hospital, DK" \ +#BodyPart="Phantom" \ +#Units="Bq/mL" \ +#TracerName="FDG" \ +#TracerRadionuclide="F18" \ +#InjectedRadioactivity=21 \ +#SpecificRadioactivity=3.7234e+03 \ +#ModeOfAdministration="infusion" \ +#FrameDuration=[600] \ +#FrameTimesStart=[0] \ +#AcquisitionMode="list mode" \ +#ImageDecayCorrected="true" \ +#ImageDecayCorrectionTime=0 \ +#AttenuationCorrection="MR-corrected" \ +#ReconFilterType='unknown' \ +#ReconFilterSize=1 \ +#ReconMethodParameterLabels="[none, none]" \ +#ReconMethodParameterUnits="[none, none]" \ +#ReconMethodParameterValues="[0, 0]" +# +## Johannes Gutenberg University of Mainz +## -------------------------------------- +# +## PhilipsGeminiPETMR +## -------------------------------------- +# +#echo "${SOURCE_FOLDER}/PhilipsGeminiPETMR-Unimedizin/reqCTAC" +#dcm2niix4pet $SOURCE_FOLDER/PhilipsGeminiPETMR-Unimedizin/reqCTAC --destination-path $DESTINATION/sub-PhilipsGeminiUnimedizinMainz/pet \ +#--kwargs \ +#Manufacturer="Philips Medical Systems" \ +#ManufacturersModelName="PET/CT Gemini TF16" \ +#InstitutionName="Unimedizin, Mainz, DE" \ +#BodyPart="Phantom" \ +#Units="Bq/mL" \ +#TracerName="Fallypride" \ +#TracerRadionuclide="F18" \ +#InjectedRadioactivity=114 \ +#SpecificRadioactivity=800 \ +#ModeOfAdministration="infusion" \ +#AcquisitionMode="list mode" \ +#ImageDecayCorrected=True \ +#ImageDecayCorrectionTime=0 \ +#ReconFilterType='n/a' \ +#ReconFilterSize=0 \ +#AttenuationCorrection="CTAC-SG" \ +#ScatterCorrectionMethod="SS-SIMUL" \ +#ReconstructionMethod="LOR-RAMLA" \ +#ReconMethodParameterValues="[1,1]" \ +#FrameDuration=[1798] \ +#ReconMethodParameterLabels="[none, none]" \ +#ReconMethodParameterUnits="[none, none]" \ +#FrameTimesStart=[0] \ +# +#echo "${SOURCE_FOLDER}/PhilipsGeminiPETMR-Unimedizin/reqNAC" +#dcm2niix4pet $SOURCE_FOLDER/PhilipsGeminiPETMR-Unimedizin/reqNAC --destination-path $DESTINATION/sub-PhilipsGeminiNACUnimedizinMainz/pet \ +#--kwargs \ +#Manufacturer="Philips Medical Systems" \ +#ManufacturersModelName="PET/CT Gemini TF16" \ +#InstitutionName="Unimedizin, Mainz, DE" \ +#BodyPart="Phantom" \ +#Units="Bq/mL" \ +#TracerName="Fallypride" \ +#TracerRadionuclide="F18" \ +#InjectedRadioactivity=114 \ +#SpecificRadioactivity=800 \ +#ModeOfAdministration="infusion" \ +#AcquisitionMode="list mode" \ +#ImageDecayCorrected=True \ +#ImageDecayCorrectionTime=0 \ +#ReconFilterType=None \ +#ReconFilterSize=0 \ +#AttenuationCorrection="None" \ +#ScatterCorrectionMethod="None" \ +#ReconstructionMethod="3D-RAMLA" \ +#FrameDuration=[1798] \ +#ReconMethodParameterLabels="[none, none]" \ +#ReconMethodParameterUnits="[none, none]" \ +#ReconMethodParameterValues="[1,1]" \ +#FrameTimesStart=[0] \ +# +# +## Amsterdam UMC +## --------------------------- +# +## Philips Ingenuity PET-CT +## ----------------------- +#echo "${SOURCE_FOLDER}/PhilipsIngenuityPETCT-AmsterdamUMC" +#dcm2niix4pet $SOURCE_FOLDER/PhilipsIngenuityPETCT-AmsterdamUMC --destination-path $DESTINATION/sub-PhilipsIngenuityPETCTAmsterdamUMC/pet \ +#--kwargs \ +#Manufacturer="Philips Medical Systems" \ +#ManufacturersModelName="Ingenuity TF PET/CT" \ +#InstitutionName="AmsterdamUMC,VUmc" \ +#BodyPart="Phantom" \ +#Units="Bq/mL" \ +#TracerName="Butanol" \ +#TracerRadionuclide="O15" \ +#InjectedRadioactivity=185 \ +#SpecificRadioactivity=1.9907e+04 \ +#ModeOfAdministration="infusion" \ +#AcquisitionMode="list mode" \ +#ImageDecayCorrected="true" \ +#ImageDecayCorrectionTime=0 \ +#DecayCorrectionFactor=[1] \ +#ReconFilterSize=0 \ +#ReconMethodParameterValues=[1] \ +#AttenuationCorrection="CTAC-SG" \ +#RandomsCorrectionMethod="DLYD" \ +#ScatterCorrectionMethod="SS-SIMUL" \ +#ReconstructionMethod="BLOB-OS-TF" \ +#ReconMethodParameterLabels="[none, none]" \ +#ReconMethodParameterUnits="[none, none]" \ +#ReconMethodParameterValues="[0, 0]" \ +#ReconFilterType="none" \ +# +## Philips Ingenuity PET-MRI +## ------------------------- +#echo "${SOURCE_FOLDER}/PhilipsIngenuityPETMR-AmsterdamUMC" +#dcm2niix4pet $SOURCE_FOLDER/PhilipsIngenuityPETMR-AmsterdamUMC --destination-path $DESTINATION/sub-PhilipsIngenuityPETMRAmsterdamUMC/pet \ +#--kwargs \ +#Manufacturer="Philips Medical Systems" \ +#ManufacturersModelName="Ingenuity TF PET/MR" \ +#InstitutionName="AmsterdamUMC,VUmc" \ +#BodyPart="Phantom" \ +#Units="Bq/mL" \ +#TracerName="11C-PIB" \ +#TracerRadionuclide="C11" \ +#InjectedRadioactivity=135.1 \ +#SpecificRadioactivity=1.4538e+04 \ +#ModeOfAdministration="infusion" \ +#AcquisitionMode="list mode" \ +#ImageDecayCorrected="True" \ +#ImageDecayCorrectionTime=0 \ +#ReconFilterType="None" \ +#ReconFilterSize=0 \ +#AttenuationCorrection="MRAC" \ +#RandomsCorrectionMethod="DLYD" \ +#ScatterCorrectionMethod="SS-SIMUL" \ +#ReconstructionMethod="LOR-RAMLA" \ +#ReconMethodParameterValues="[1, 1]" \ +#ReconFilterType="['n/a', 'n/a']" \ +#DecayCorrectionFactor=[1] \ +#ReconMethodParameterLabels="[none, none]" \ +#ReconMethodParameterUnits="[none, none]" \ +#ReconMethodParameterValues="[0, 0]" +# +## philipsVereosPET-CT +## ------------------- +#echo "${SOURCE_FOLDER}/PhilipsVereosPETCT-AmsterdamUMC" +#dcm2niix4pet $SOURCE_FOLDER/PhillipsVereosPETCT-AmsterdamUMC --destination-path $DESTINATION/sub-PhillipsVereosAmsterdamUMC/pet \ +#--kwargs \ +#Manufacturer="Philips Medical Systems" \ +#ManufacturersModelName="Vereos PET/CT" \ +#InstitutionName="AmsterdamUMC,VUmc" \ +#BodyPart="Phantom" \ +#Units="Bq/mL" \ +#TracerName="11C-PIB" \ +#TracerRadionuclide="C11" \ +#InjectedRadioactivity=202.5 \ +#SpecificRadioactivity=2.1791e+04 \ +#ModeOfAdministration="infusion" \ +#AcquisitionMode="list mode" \ +#ImageDecayCorrected="True" \ +#ImageDecayCorrectionTime=0 \ +#ReconFilterType="None" \ +#ReconFilterSize=0 \ +#AttenuationCorrection="CTAC-SG" \ +#ScatterCorrectionMethod="SS-SIMUL" \ +#RandomsCorrectionMethod="DLYD" \ +#ReconstructionMethod="OSEMi3s15" \ +#TimeZero="11:40:24" # National Institute of Mental Health, Bethesda # ---------------------------------------------- @@ -504,3 +485,4 @@ ReconMethodParameterUnits="['none', 'none']" \ ReconMethodParameterLabels="['subsets', 'iterations']" \ ReconFilterType="Gaussian" \ ReconFilterSize=4 + diff --git a/scripts/testphantoms b/scripts/testphantoms index 909db0ab..cdbeb5c3 100755 --- a/scripts/testphantoms +++ b/scripts/testphantoms @@ -17,13 +17,12 @@ make buildpackage make installpackage # Collect Phantoms -GDRIVE_PHANTOM_ID=$(grep GDRIVE_PHANTOM_ID pypet2bids/.env) PHANTOM_ZIP_FILE=PHANTOMS.ZIP export $GDRIVE_PHANTOM_ID if [ -f $PHANTOM_ZIP_FILE ]; then echo "$PHANTOM_ZIP_FILE exists, skipping download..." else - gdown $GDRIVE_PHANTOM_ID -O PHANTOMS.zip + wget -O PHANTOMS.zip https://openneuropet.s3.amazonaws.com/US-sourced-OpenNeuroPET-Phantoms.zip fi # cleanup old run of conversions