Libtk reorganization

.github/workflows/build-and-test.yml

@@ -24,7 +24,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Build and test
-        run: poetry run doit
+        run: |
+          poetry run doit
+          poetry run doit libtk205 --PAT "${{secrets.CI_PA_TOKEN}}"
+          poetry run doit libtk205_tests
       - name: Export libtk205 source
-        run: cmake -DPA_TOKEN="${{secrets.CI_PA_TOKEN}}" -DPROJECT_SOURCE_DIR="${{github.workspace}}/libtk205" -P "${{github.workspace}}/libtk205/cmake/update_lib_repo.cmake"
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10'
+        run: cmake -DPA_TOKEN="${{secrets.CI_PA_TOKEN}}" -DPROJECT_SOURCE_DIR="${{github.workspace}}" -Drepo_name=libtk205 -Dupload_repo=https://github.com/open205/libtk205 -P "${{github.workspace}}/cmake/update_lib_repo_ci.cmake"
 

.gitignore

@@ -10,4 +10,5 @@ env/
 .vscode/
 .pytest_cache/
 sandbox/
-.doit.db*
+.doit.db*
+libtk205/

.gitmodules

@@ -1,9 +1,3 @@
 [submodule "schema-205"]
 	path = schema-205
 	url = https://github.com/open205/schema-205
-[submodule "libtk205/vendor/json"]
-	path = libtk205/vendor/json
-	url = https://github.com/nlohmann/json.git
-[submodule "libtk205/vendor/btwxt"]
-	path = libtk205/vendor/btwxt
-	url = https://github.com/bigladder/btwxt.git

CMakeLists.txt

@@ -17,12 +17,37 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
     "MinSizeRel" "RelWithDebInfo")
 endif()
 
-option( BUILD_TK205_TESTING "Build libtk205 testing targets" ON )
+# Run libtk205 generator
 
-# Set up testing/coverage
-if (BUILD_TK205_TESTING)
-  set(TEST205_INPUT_EXAMPLES_DIR "${PROJECT_SOURCE_DIR}/schema-205") 
-  enable_testing()
-endif()
+file(GLOB RS_schemas "${PROJECT_SOURCE_DIR}/schema-205/schema-source/*.schema.yaml")
+
+foreach(schema IN LISTS RS_schemas)
+  string(REGEX REPLACE "${PROJECT_SOURCE_DIR}/schema-205/schema-source/(.*).schema.yaml" "\\1" schema_name "${schema}")
+  string(TOLOWER ${schema_name} schema_name)
+  list(APPEND rs_headers "${PROJECT_SOURCE_DIR}/schema-205/build/include/${schema_name}.h")
+  list(APPEND rs_src "${PROJECT_SOURCE_DIR}/schema-205/build/cpp/${schema_name}.cpp")
+  if (schema_name MATCHES "rs.*")
+    list(APPEND factory_headers "${PROJECT_SOURCE_DIR}/schema-205/build/include/${schema_name}_factory.h")
+    list(APPEND factory_src "${PROJECT_SOURCE_DIR}/schema-205/build/cpp/${schema_name}_factory.cpp")
+  endif()
+endforeach()
+
+if(BUILD_LIBTK205)
+  execute_process(
+    COMMAND ${CMAKE_COMMAND} -DPA_TOKEN=${PA_TOKEN} -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR} -Drepo_name=libtk205 -Dupload_repo=https://github.com/open205/libtk205 -P "cmake/get_lib_repo_local.cmake"
+  )
+  add_subdirectory(libtk205)
 
-add_subdirectory(libtk205)
+  add_custom_target(libtk205_generator ALL
+                    DEPENDS ${rs_headers} ${rs_src} ${factory_headers} ${factory_src})
+
+  # Runs if OUTPUT is missing, or if OUTPUT exists BUT file DEPENDencies have changed
+  # If OUTPUT exists and DEPENDS files haven't changed, the command is not run.
+  # If no target depends on its outputs, the command is not run (see libtk205_generator custom target)
+  add_custom_command(OUTPUT ${rs_headers} ${rs_src} ${factory_headers} ${factory_src}
+                    COMMAND poetry run doit cpp
+                    COMMAND ${CMAKE_COMMAND} -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR} -Drepo_name=libtk205 -P "../cmake/update_lib_repo_local.cmake"
+                    DEPENDS ${RS_schemas}
+                    COMMENT "Generate libtk205 files from YAML schema"
+                    WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}/schema-205")
+endif()

cmake/get_lib_repo_local.cmake

@@ -0,0 +1,61 @@
+# repo_name is unique to the project
+if(${PA_TOKEN})
+   set(authenticated_repo "https://${PA_TOKEN}:x-oauth-basic@${upload_repo}") # PA_TOKEN is set by GitHub Actions
+else()
+   set(authenticated_repo ${upload_repo})
+endif()
+
+message(STATUS ${authenticated_repo})
+
+set(clone_dir "${PROJECT_SOURCE_DIR}/${repo_name}")
+
+find_package(Git QUIET)
+
+if(GIT_FOUND)
+   if(NOT EXISTS "${clone_dir}")
+      execute_process(COMMAND ${GIT_EXECUTABLE} clone ${authenticated_repo}
+                      WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                      RESULT_VARIABLE GIT_SUBMOD_RESULT)
+      if(NOT GIT_SUBMOD_RESULT EQUAL "0")
+         message(FATAL_ERROR "${GIT_EXECUTABLE} clone ${authenticated_repo} failed with ${GIT_SUBMOD_RESULT}.")
+      endif()
+   # else() pull or fetch?
+   endif()
+else()
+   message(FATAL_ERROR "git not found!")
+endif()
+
+# git branch name query
+execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD
+                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                OUTPUT_VARIABLE current_git_branch
+                OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+message(STATUS "Current local branch is ${current_git_branch}.")
+
+execute_process(COMMAND ${GIT_EXECUTABLE} log -1 --pretty=%B
+                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                OUTPUT_VARIABLE last_commit_msg
+                OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+message(STATUS "Last commit message on local branch was \"${last_commit_msg}\"")
+
+execute_process(COMMAND ${GIT_EXECUTABLE} ls-remote --heads --exit-code ${authenticated_repo} ${current_git_branch}
+                WORKING_DIRECTORY ${clone_dir}
+                RESULT_VARIABLE exit_code
+                OUTPUT_VARIABLE output_repo_branches
+)
+if(exit_code EQUAL "0") # Successful communication with remote (but matching refs status unknown)
+   message(STATUS "Checking out branch ${current_git_branch} from ${repo_name} remote.")
+   execute_process(COMMAND ${GIT_EXECUTABLE} checkout ${current_git_branch}
+                   WORKING_DIRECTORY ${clone_dir}
+   )
+elseif(exit_code EQUAL "2") # No matching refs in remote
+   message(STATUS "Branch ${current_git_branch} must be created in upload remote.")
+   execute_process(COMMAND ${GIT_EXECUTABLE} checkout -b ${current_git_branch}
+                   WORKING_DIRECTORY ${clone_dir}
+   )
+   execute_process(COMMAND ${GIT_EXECUTABLE} push -u origin ${current_git_branch}
+                   WORKING_DIRECTORY ${clone_dir}
+   )
+endif()

cmake/update_lib_repo_ci.cmake

@@ -0,0 +1,44 @@
+# repo_name is unique to the project
+set(authenticated_repo "https://${PA_TOKEN}:x-oauth-basic@${upload_repo}") # PA_TOKEN is set by GitHub Actions
+set(clone_dir "${PROJECT_SOURCE_DIR}/${repo_name}")
+
+find_package(Git QUIET)
+
+if(NOT GIT_FOUND)
+   message(FATAL_ERROR "git not found!")
+endif()
+
+message(STATUS "Working directory: ${PROJECT_SOURCE_DIR}.")
+# Collect relevant info from toolkit-205:
+execute_process(COMMAND ${GIT_EXECUTABLE} log -1 --pretty=%B
+                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                OUTPUT_VARIABLE last_commit_msg
+                OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+message(STATUS "Last commit message on ${PROJECT_NAME} branch was \"${last_commit_msg}\"")
+
+message(STATUS "Working directory: ${clone_dir}.")
+# libtk205 remote side:
+# git config
+execute_process(COMMAND ${GIT_EXECUTABLE} config user.name "Actions CI"
+                WORKING_DIRECTORY ${clone_dir}
+)
+execute_process(COMMAND ${GIT_EXECUTABLE} config user.email "ci.bigladdersoftware.com"
+                WORKING_DIRECTORY ${clone_dir}
+)
+# git add
+execute_process(COMMAND ${GIT_EXECUTABLE} add include src test examples
+                WORKING_DIRECTORY ${clone_dir}
+)
+# git status (see if files changed)
+execute_process(COMMAND ${GIT_EXECUTABLE} status
+                WORKING_DIRECTORY ${clone_dir}
+)
+# git commit
+execute_process(COMMAND ${GIT_EXECUTABLE} commit -m "\"${last_commit_msg}\""
+                WORKING_DIRECTORY ${clone_dir}
+)
+# git push
+execute_process(COMMAND ${GIT_EXECUTABLE} push ${authenticated_repo}
+                WORKING_DIRECTORY ${clone_dir}
+)

cmake/update_lib_repo_local.cmake

@@ -0,0 +1,38 @@
+set(clone_dir "${PROJECT_SOURCE_DIR}/${repo_name}")
+
+set(include_dest_dir "${clone_dir}/include/${repo_name}")
+set(src_dest_dir "${clone_dir}/src")
+
+file(MAKE_DIRECTORY "${include_dest_dir}")
+file(MAKE_DIRECTORY "${src_dest_dir}")
+
+# Move/copy generated files
+file(GLOB generated_rs_headers "${PROJECT_SOURCE_DIR}/schema-205/build/include/*.h")
+foreach(filepath ${generated_rs_headers})
+  get_filename_component(filename ${filepath} NAME)
+  message(STATUS "Moving ${filepath} to ${include_dest_dir}/${filename}.")
+  file(RENAME ${filepath} ${include_dest_dir}/${filename})
+endforeach(filepath)
+
+file(COPY "${PROJECT_SOURCE_DIR}/../schema-205/schema205/libtk205_fixed_src/include/"
+     DESTINATION "${include_dest_dir}"
+     FILES_MATCHING PATTERN *.h)
+
+file(GLOB generated_rs_src "${PROJECT_SOURCE_DIR}/schema-205/build/cpp/*.cpp")
+     foreach(filepath ${generated_rs_src})
+       get_filename_component(filename ${filepath} NAME)
+       message(STATUS "Moving ${filepath} to ${src_dest_dir}/${filename}.")
+       file(RENAME ${filepath} ${src_dest_dir}/${filename})
+     endforeach(filepath)
+
+file(COPY "${PROJECT_SOURCE_DIR}/../schema-205/schema205/libtk205_fixed_src/src/"
+     DESTINATION "${src_dest_dir}"
+     FILES_MATCHING PATTERN *.cpp)
+
+file(COPY "${PROJECT_SOURCE_DIR}/schema-205/examples"
+     DESTINATION "${clone_dir}")
+
+# git status (see if files changed)
+execute_process(COMMAND ${GIT_EXECUTABLE} status
+        WORKING_DIRECTORY ${clone_dir}
+)

dodo.py

@@ -1,5 +1,6 @@
 import os
 import tk205
+import subprocess
 from doit.tools import create_folder
 
 BUILD_PATH = "build"
@@ -9,10 +10,12 @@
 EXAMPLES_OUTPUT_PATH = os.path.join(BUILD_PATH,"examples")
 TEMPLATE_OUTPUT_PATH = os.path.join(BUILD_PATH,"templates")
 TEMPLATE_CONFIG = os.path.join('config','templates.json')
-LIB_BUILD_PATH = os.path.join(BUILD_PATH,"libtk205")
+LIB_BUILD_PATH = BUILD_PATH
 TK205_SOURCE_PATH = 'tk205'
 SCHEMA205_SOURCE_PATH = os.path.join("schema-205","schema205")
 
+DOIT_CONFIG = {'default_tasks': ['build_schema', 'cbor', 'yaml', 'xlsx', 'json', 'templates', 'test', 'web']}
+
 def task_build_schema():
   '''Build the schema'''
   return {
@@ -171,20 +174,26 @@ def task_web():
 
 def task_libtk205():
   '''Build libtk205'''
+  def configure_build(PAT):
+    subprocess.run(['cmake', f'-B {LIB_BUILD_PATH}', f'-DPA_TOKEN={PAT}', '-DBUILD_LIBTK205=ON', '-Dlibtk205_BUILD_TESTING=ON'])
   return {
     'task_dep': ['build_schema'],
+    'params':[{'name':'PAT',
+               'long': 'PAT',
+               'default': ''}],
     'actions': [
       (create_folder, [LIB_BUILD_PATH]),
-      f'cmake -B {LIB_BUILD_PATH}',
-      f'cmake --build {LIB_BUILD_PATH} --config Release'
+      (configure_build, ),
+      'cmake --build build --config Release'
       ],
+    'clean': ['doit -d schema-205 clean cpp'],
   }
 
 def task_libtk205_tests():
   '''Performs unit tests and example file validation tests'''
   return {
     'task_dep': ['libtk205'],
     'actions': [
-      f'cd {LIB_BUILD_PATH} && ctest',
+      f'cd {LIB_BUILD_PATH}/libtk205 && ctest',
       ],
   }