Merge branch 'develop' into GEFS-fix

.github/workflows/book.yml

@@ -65,7 +65,7 @@ jobs:
         git config --global user.email "[email protected]"
         git config --global user.name "GitHub Documentation Robot"
         export VERSION=$(echo $GITHUB_REF | sed 's,.*/,,' )
-        if [ "$VERSION" == "main" ]; then
+        if [ "$VERSION" = "main" ]; then
           export VERSION=latest
         fi
         cd pecan-documentation

.github/workflows/download-met-data.yml

@@ -0,0 +1,57 @@
+name : Test Data Download
+on :
+  # allow manual triggering
+  workflow_dispatch:
+
+  schedule:
+    # run Thursday 4:30 AM UTC
+  - cron: '30 4 * * 4'
+
+env:
+  R_LIBS_USER: /usr/local/lib/R/site-library
+  LC_ALL: en_US.UTF-8
+  NCPUS: 2
+  PGHOST: postgres
+  CI: true
+
+jobs:
+  met-data-download:
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+
+    services:
+      postgres:
+        image: mdillon/postgis:9.5
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+    container: 
+      image: pecan/depends:R4.1
+
+    steps:
+    # checkout source code
+    - name: work around https://github.com/actions/checkout/issues/766
+      run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+    - uses: actions/checkout@v4
+      with:
+        set-safe-directory: false
+
+    # install additional tools needed
+    - name: install utils
+      run: apt-get update && apt-get install -y postgresql-client qpdf
+    - name: install new dependencies
+      run: Rscript scripts/generate_dependencies.R && cd docker/depends && Rscript pecan.depends.R
+
+    # initialize database
+    - name: db setup
+      uses: docker://pecan/db:ci
+    - name: add models to db
+      run: ./scripts/add.models.sh
+
+    # compile PEcAn code
+    - name: build
+      run: make -j1
+
+    - name: CRUNCEP
+      run: |
+        Rscript ./tests/test_met_downloads.R --settings ./tests/met_download_settings/docker.CRUNCEP.xml

CONTRIBUTING.md

@@ -115,4 +115,4 @@ git push -u origin GH-issuenumber-title-of-issue
 - [Adding models to PEcAn](https://pecanproject.github.io/pecan-documentation/latest/adding-an-ecosystem-model.html)
 - [PEcAn configuration files](https://pecanproject.github.io/pecan-documentation/latest/pecan-xml-configuration.html)
 - [Development help](https://pecanproject.github.io/pecan-documentation/latest/developer-guide.html)
-- [PEcAn Code of Conduct](https://pecanproject.github.io/pecan-documentation/latest/contributor-covenant-code-of-conduct.html)
+- [PEcAn Code of Conduct](CODE_OF_CONDUCT.md)

base/db/R/dbfiles.R

@@ -50,9 +50,8 @@ dbfile.input.insert <- function(in.path, in.prefix, siteid, startdate, enddate,
 
 
   # setup parent part of query if specified
-  if (is.na(parentid)) {
-    parent <- ""
-  } else {
+  parent <- ""
+  if (!is.na(parentid)) {
     parent <- paste0(" AND parent_id=", parentid)
   }
 
@@ -242,13 +241,13 @@ dbfile.input.check <- function(siteid, startdate = NULL, enddate = NULL, mimetyp
   formatid <- get.id(table = "formats", colnames = c("mimetype_id", "name"), values = c(mimetypeid, formatname), con = con)
 
   if (is.null(formatid)) {
-    invisible(data.frame())
+    return (invisible(data.frame()))
   }
 
   # setup parent part of query if specified
-  if (is.na(parentid)) {
-    parent <- ""
-  } else {
+  parent <- ""
+
+  if (!is.na(parentid)) {
     parent <- paste0(" AND parent_id=", parentid)
   }
 
@@ -450,7 +449,7 @@ dbfile.posterior.check <- function(pft, mimetype, formatname, con, hostname = PE
   # find appropriate pft
   pftid <- get.id(table = "pfts", values = "name", colnames = pft, con = con)
   if (is.null(pftid)) {
-    invisible(data.frame())
+    return (invisible(data.frame()))
   }
 
   # find appropriate format
@@ -461,7 +460,7 @@ dbfile.posterior.check <- function(pft, mimetype, formatname, con, hostname = PE
   formatid <- get.id(table = "formats", colnames = c("mimetype_id", "name"), values = c(mimetypeid, formatname), con = con)
 
   if (is.null(formatid)) {
-    invisible(data.frame())
+    return (invisible(data.frame()))
   }
 
   # find appropriate posterior
@@ -473,7 +472,7 @@ dbfile.posterior.check <- function(pft, mimetype, formatname, con, hostname = PE
     con = con
   )[["id"]]
   if (is.null(posteriorid)) {
-    invisible(data.frame())
+    return (invisible(data.frame()))
   }
 
   invisible(dbfile.check(type = "Posterior", container.id = posteriorid, con = con, hostname = hostname))
@@ -639,12 +638,12 @@ dbfile.file <- function(type, id, con, hostname = PEcAn.remote::fqdn()) {
 
   if (nrow(files) > 1) {
     PEcAn.logger::logger.warn("multiple files found for", id, "returned; using the first one found")
-    invisible(file.path(files[1, "file_path"], files[1, "file_name"]))
+    return(invisible(file.path(files[1, "file_path"], files[1, "file_name"])))
   } else if (nrow(files) == 1) {
-    invisible(file.path(files[1, "file_path"], files[1, "file_name"]))
+    return(invisible(file.path(files[1, "file_path"], files[1, "file_name"])))
   } else {
     PEcAn.logger::logger.warn("no files found for ", id, "in database")
-    invisible(NA)
+    return(invisible(NA))
   }
 }
 
@@ -662,7 +661,8 @@ dbfile.id <- function(type, file, con, hostname = PEcAn.remote::fqdn()) {
   # find appropriate host
   hostid <- db.query(query = paste0("SELECT id FROM machines WHERE hostname='", hostname, "'"), con = con)[["id"]]
   if (is.null(hostid)) {
-    invisible(NA)
+    PEcAn.logger::logger.warn("hostid not found in database")
+    return (invisible(NA))
   }
 
   # find file

book_source/03_topical_pages/02_pecan_standards.Rmd

@@ -10,11 +10,11 @@ Internal PEcAn standard time follows ISO_8601 format for dates and time (https:/
 
 To aid in the conversion between PEcAn internal ISO_8601 standard and CF convention used in all met drivers and PEcAn standard output you can utilize the functions: "cf2datetime","datetime2doy","cf2doy", and for SIPNET "sipnet2datetime"
 
-### Input Standards
+## Input Standards
 
-#### Meterology Standards 
+### Meteorology Standards 
 
-##### Dimensions:
+#### Dimensions:
 
 
 |CF standard-name | units |
@@ -26,7 +26,8 @@ To aid in the conversion between PEcAn internal ISO_8601 standard and CF convent
 General Note: dates in the database should be date-time (preferably with timezone), and datetime passed around in PEcAn should be of type POSIXct.
 
 
-##### The variable names should be `standard_name`
+#### Variable names should be `standard_name`
+
 ```{r, echo=FALSE,warning=FALSE,message=FALSE}
 names<-c("<b>air_temperature</b>", "air_temperature_max", "air_temperature_min", "<b>air_pressure</b>", 
          "mole_fraction_of_carbon_dioxide_in_air", "moisture_content_of_soil_layer", "soil_temperature ",
@@ -75,15 +76,15 @@ The key is to process each type of met data (site, reanalysis, forecast, climate
 
 ### Soils and Vegetation Inputs
 
-##### Soil Data 
+#### Soil Data 
 
 See the [Soil Data] section on more into on creating a standard soil data file.
 
-##### Vegetation Data
+#### Vegetation Data
 
 See the [Vegetation Data] section on more info on creating a standard vegetation data file
 
-### Output Standards {#OutputStandards}
+## Output Standards {#OutputStandards}
 
 * created by `model2netcdf` functions
 * based on format used by [MsTMIP](http://nacp.ornl.gov/MsTMIP_variables.shtml)

book_source/03_topical_pages/92_workflow_modules.Rmd

@@ -5,7 +5,7 @@ NOTE: As of PEcAn 1.2.6 -- needs to be updated significantly
 <!--NEEDS TO BE UPDATED SIGNIFICANTLY-->
 ## Overview
 
-Workflow inputs and outputs (click to open in new page, then zoom). Code used to generate this image is provided in [qaqc/vignettes/module_output.Rmd](https://github.com/PecanProject/pecan/blob/main/qaqc/vignettes/module_output.Rmd)
+Workflow inputs and outputs (click to open in new page, then zoom). Code used to generate this image is provided in [base/qaqc/vignettes/module_output.Rmd](https://github.com/PecanProject/pecan/blob/master/base/qaqc/vignettes/module_output.Rmd)
 
 [![PEcAn Workflow](http://isda.ncsa.illinois.edu/~kooper/EBI/workflow.svg)](http://isda.ncsa.illinois.edu/~kooper/EBI/workflow.svg)
 

book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd

@@ -1,19 +1,20 @@
 ### Mac OSX {#macosx}
 
-These are specific notes for installing PEcAn on Mac OSX and will be referenced from the main [installing PEcAn](Installing-PEcAn) page. You will at least need to install the build environment and Postgres sections. If you want to access the database/PEcAn using a web browser you will need to install Apache. To access the database using the BETY interface, you will need to have Ruby installed.
+These are specific notes for installing PEcAn on Mac OSX and referenced from the [installing PEcAn](Installing-PEcAn) page. 
 
-This document also contains information on how to install the Rstudio server edition as well as any other packages that can be helpful.
+Build environment and Postgres sections are required to install and use PEcAn.
 
+Optional software includes Apache, Rails, and Rstudio. Apache is required to run the BETYdb UI and the web-based version of PEcAn - both are optional. Rails is requred to use the BETYdb web interface. Rstudio is a commonly used IDE for R.
 
 #### Install build environment
 
-```bash
-# install R
-# download from http://cran.r-project.org/bin/macosx/
+##### Option 1: Download and install
+
+R: download from http://cran.r-project.org/bin/macosx/
 
-# install gfortran 
-# download from http://cran.r-project.org/bin/macosx/tools/
+gfortran: download from http://cran.r-project.org/bin/macosx/tools/
 
+```bash
 # install OpenMPI
 curl -o openmpi-1.6.3.tar.gz http://www.open-mpi.org/software/ompi/v1.6/downloads/openmpi-1.6.3.tar.gz
 tar zxf openmpi-1.6.3.tar.gz
@@ -42,14 +43,63 @@ sudo make install
 cd ..
 ```
 
-#### Install Postgres
+##### Option 2: Homebrew
+
+```bash
+# R
+brew install --cask r
+# gfortran
+brew install gcc
+# OpenMPI
+brew install open-mpi
+# szip
+brew install szip
+# HDF5
+brew install hdf5 
+## homebrew should configure hdf5 with fortran and cxx, otherwise:
+## brew install hdf5 --with-fortran --with-cxx
+```
 
-For those on a Mac I use the following app for postgresql which has
-postgis already installed (http://postgresapp.com/)
 
-To get postgis run the following commands in psql:
+#### Install Postgres and PostGIS
+
+##### Option 1: 
+
+For MacOS, the Postgres.app provides Postgres with PostGIS
+already installed (http://postgresapp.com/).
+
+To run Postgres:
+*	Open Postgres.app.
+* In the menu bar, click the elephant icon and select “Open psql”.
+
+##### Option 2: install using homebrew:
 
 ```bash
+# optional: remove existing postgres installations with:
+# brew uninstall --force postgresql
+
+# install Postgres, fixed at v12 (officially supported by BETYdb):
+brew install postgres@12
+brew pin postgres@12
+
+# PostGIS
+brew install postgis
+
+# to run Postgres:
+brew services start postgresql
+```
+
+#### Enable PostGIS
+
+To enable PostGIS, you should start Postgres:
+
+```bash
+psql postgres
+```
+
+And then run the following commands:
+
+```sql
 ##### Enable PostGIS (includes raster)
 CREATE EXTENSION postgis;
 ##### Enable Topology
@@ -60,23 +110,42 @@ CREATE EXTENSION fuzzystrmatch;
 CREATE EXTENSION postgis_tiger_geocoder;
 ```
 
-To check your postgis run the following command again in psql: `SELECT PostGIS_full_version();`
+To check your postgis run the following command again in psql: 
+
+```sql
+SELECT PostGIS_full_version();`
+```
 
 #### Additional installs
 
 
 ##### Install JAGS
 
-Download JAGS from http://sourceforge.net/projects/mcmc-jags/files/JAGS/3.x/Mac%20OS%20X/JAGS-Mavericks-3.4.0.dmg/download
+
+##### Option 1: using homebrew
+
+```bash
+brew install jags
+```
+
+##### Option 2: Download 
+
+Download JAGS from http://sourceforge.net/projects/mcmc-jags/files/JAGS/3.x/Mac%20OS%20X/JAGS-Mavericks-3.4.0.dmg/download.
+
 
 ##### Install udunits
 
-Installing udunits-2 on MacOSX is done from source.
+##### Option 1: Install using homebrew:
+
+```bash
+brew install udunits
+```
+
+##### Option 2: Install udunits-2 on MacOSX is done from source.
 
 * download most recent [version of Udunits here](http://www.unidata.ucar.edu/downloads/udunits/index.jsp)
 * instructions for [compiling from source](http://www.unidata.ucar.edu/software/udunits/udunits-2/udunits2.html#Obtain)
 
-
 ```bash
 curl -o udunits-2.1.24.tar.gz ftp://ftp.unidata.ucar.edu/pub/udunits/udunits-2.1.24.tar.gz
 tar zxf udunits-2.1.24.tar.gz
@@ -86,7 +155,7 @@ make
 sudo make install
 ```
 
-#### Apache Configuration
+#### Apache Configuration  (Optional)
 
 Mac does not support pdo/postgresql by default. The easiest way to install is use: http://php-osx.liip.ch/
 
@@ -102,10 +171,24 @@ Alias /pecan ${PWD}/pecan/web
 EOF
 ```
 
-#### Ruby
+#### Ruby  (Optional)
+
+_Note: it is recommended that BETYdb be run using Docker because the application uses unsupported versions of Ruby and Rails._
+
+The BETYdb application requires Ruby version 2.7.7, as specified in [PecanProject/bety/.ruby-version](https://github.com/PecanProject/bety/blob/develop/.ruby-version). 
+
+```bash
+brew install rbenv
+rbenv init
+rbenv install 2.7.7 
+```
+
+#### Rstudio (Optional)
 
-The default version of ruby should work. Or use [JewelryBox](https://jewelrybox.unfiniti.com/).
+For MacOS, you can download [Rstudio Desktop](http://www.rstudio.com/).
 
-#### Rstudio Server
+Or using homebrew:
 
-For the mac you can download [Rstudio Desktop](http://www.rstudio.com/).
+```bash
+brew install --cask rstudio
+```