Named Constants and template support added to standard library

Allowing for significant reduction in the size of the source for the standard library
Templates can be nested and replace multiple variables simultaneously
Templates can be expanded and diff in cmake stage - allowing for easy validation of appropriate template expansion
Leaves the libraries folder untouched for now, to facilitate easier hands on experimentation with the new grammar

Also includes documentation building tools - which have been the proving ground for the new template grammar.
Builds specification docs to a temporary folder (for now) from a new template location - again for easy validation that
the expansion is correct.
The documentation work was critical to ensuring this initial template system is fully featured enough to
be useful, and uncovered a number of discrepancies with the human edited specification. This is one of the
strong advantages of using a template expansion based system.

Author: ld-kerley

.gitignore

@@ -1,3 +1,8 @@
 build
 dist
 .DS_Store
+.cache
+
+# temporary addition while we're iterating on building the specification from source
+documents/SpecificationBuild
+

CMakeLists.txt

@@ -36,7 +36,9 @@ project(MaterialX VERSION ${MATERIALX_LIBRARY_VERSION})
 option(MATERIALX_BUILD_PYTHON "Build the MaterialX Python package from C++ bindings. Requires Python 3.9 or greater." OFF)
 option(MATERIALX_BUILD_VIEWER "Build the MaterialX Viewer." OFF)
 option(MATERIALX_BUILD_GRAPH_EDITOR "Build the MaterialX Graph Editor." OFF)
-option(MATERIALX_BUILD_DOCS "Create HTML documentation using Doxygen. Requires that Doxygen be installed." OFF)
+option(MATERIALX_BUILD_DOCS "(deprecated) use MATERIALX_BUILD_DOXYGEN_DOCS." OFF)
+option(MATERIALX_BUILD_DOXYGEN_DOCS "Create HTML documentation using Doxygen. Requires that Doxygen be installed." OFF)
+option(MATERIALX_BUILD_SPECIFICATION_DOCS "Create Specification documentation, using the data library templates." OFF)
 
 option(MATERIALX_BUILD_GEN_GLSL "Build the GLSL shader generator back-end." ON)
 option(MATERIALX_BUILD_GEN_OSL "Build the OSL shader generator back-end." ON)
@@ -51,6 +53,8 @@ option(MATERIALX_BUILD_BENCHMARK_TESTS "Build benchmark tests." OFF)
 
 option(MATERIALX_BUILD_SHARED_LIBS "Build MaterialX libraries as shared rather than static." OFF)
 option(MATERIALX_BUILD_DATA_LIBRARY "Build generated products from the MaterialX data library." OFF)
+option(MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS "Process the data library files at build time to expand any template elements." ON)
+option(MATERIALX_BUILD_BAKE_NAMED_VALUES "Process the data library files at build time to bake out the named values." ON)
 option(MATERIALX_BUILD_MONOLITHIC "Build a single monolithic MaterialX library." OFF)
 option(MATERIALX_BUILD_USE_CCACHE "Enable the use of ccache to speed up build time, if present." ON)
 option(MATERIALX_PYTHON_LTO "Enable link-time optimizations for MaterialX Python." ON)
@@ -107,6 +111,11 @@ if (MATERIALX_BUILD_JS)
     set(MATERIALX_BUILD_TESTS OFF)
 endif()
 
+# to maintain legacy cmake argument
+if (MATERIALX_BUILD_DOCS)
+    set(MATERIALX_BUILD_DOXYGEN_DOCS ON)
+endif()
+
 # All hardware shading languages currently depend on the GLSL shader generator.
 if(MATERIALX_BUILD_GEN_MSL)
     set(MATERIALX_BUILD_GEN_GLSL ON)
@@ -437,6 +446,16 @@ else()
     set(BUILD_SHARED_LIBS "OFF")
 endif()
 
+# If we're baking the named "Constant:" entries - then we have to also expand any
+# template elements
+if (MATERIALX_BUILD_BAKE_NAMED_VALUES)
+    set(MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS ON)
+endif()
+
+if (MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS OR MATERIALX_BUILD_BAKE_NAMED_VALUES)
+    set(MATERIALX_BUILD_DATA_LIBRARY ON)
+endif()
+
 # Build a monolithic target - needs to be added before the other build targets that may be included.
 if (MATERIALX_BUILD_MONOLITHIC)
     set(MATERIALX_MONOLITHIC_TARGET MaterialX)
@@ -446,6 +465,32 @@ endif()
 # Add core subdirectories
 add_subdirectory(source/MaterialXCore)
 add_subdirectory(source/MaterialXFormat)
+if (MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS OR MATERIALX_BUILD_BAKE_NAMED_VALUES)
+    if (CMAKE_CROSSCOMPILING)
+        set(_MaterialXBuildLibrary "${CMAKE_BINARY_DIR}/BUILD_TOOLS/MaterialXBuildLibrary")
+
+        add_custom_command(
+                OUTPUT ${_MaterialXBuildLibrary}
+                COMMAND ${CMAKE_COMMAND}
+                -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
+                -DCMAKE_RUNTIME_OUTPUT_DIRECTORY:PATH=${CMAKE_BINARY_DIR}/BUILD_TOOLS
+                -DMATERIALX_MAJOR_VERSION=${MATERIALX_MAJOR_VERSION}
+                -DMATERIALX_MINOR_VERSION=${MATERIALX_MINOR_VERSION}
+                -DMATERIALX_BUILD_VERSION=${MATERIALX_BUILD_VERSION}
+                -DMATERIALX_NAMESPACE=${MATERIALX_NAMESPACE}
+                -DMATERIALX_BUILD_BAKE_NAMED_VALUES=ON
+                -B"BUILD_TOOLS"
+                -H"${CMAKE_SOURCE_DIR}/source/MaterialXBuildTools"
+                COMMAND ${CMAKE_COMMAND} --build BUILD_TOOLS
+                DEPENDS MaterialXCore MaterialXFormat
+        )
+        add_custom_target(MaterialXBuildLibrary ALL
+        DEPENDS ${_MaterialXBuildLibrary})
+    else()
+        set(_MaterialXBuildLibrary MaterialXBuildLibrary)
+        add_subdirectory(source/MaterialXBuildTools/buildLibrary)
+    endif()
+endif()
 
 # Add shader generation subdirectories
 add_subdirectory(source/MaterialXGenShader)
@@ -467,6 +512,7 @@ if(MATERIALX_BUILD_GEN_GLSL OR MATERIALX_BUILD_GEN_OSL OR MATERIALX_BUILD_GEN_MD
         add_subdirectory(source/MaterialXGenMsl)
     endif()
     add_subdirectory(libraries)
+    add_subdirectory(templates)
 endif()
 
 # Add rendering and viewer subdirectories
@@ -511,8 +557,11 @@ if(MATERIALX_BUILD_PYTHON)
     add_subdirectory(python)
 endif()
 
-if(MATERIALX_BUILD_DOCS)
+if(MATERIALX_BUILD_DOXYGEN_DOCS OR MATERIALX_BUILD_SPECIFICATION_DOCS)
     add_subdirectory(documents)
+    if (MATERIALX_BUILD_SPECIFICATION_DOCS)
+        add_subdirectory(source/MaterialXBuildTools/buildDocs)
+    endif()
 endif()
 
 if(MATERIALX_BUILD_JS)

documents/CMakeLists.txt

@@ -1,45 +1,81 @@
-set(DOXYGEN_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(DOXYGEN_HTML_OUTPUT_DIR ${DOXYGEN_OUTPUT_DIR}/html)
-set(DOXYGEN_INPUT_LIST ${CMAKE_CURRENT_BINARY_DIR}/MainPage.md)
-
-set(MATERIALX_DOXYGEN_SOURCE_FOLDERS
-    ${PROJECT_SOURCE_DIR}/source/MaterialXCore
-    ${PROJECT_SOURCE_DIR}/source/MaterialXFormat
-    ${PROJECT_SOURCE_DIR}/source/MaterialXGenShader
-    ${PROJECT_SOURCE_DIR}/source/MaterialXGenShader/Nodes
-    ${PROJECT_SOURCE_DIR}/source/MaterialXGenGlsl
-    ${PROJECT_SOURCE_DIR}/source/MaterialXGenGlsl/Nodes
-    ${PROJECT_SOURCE_DIR}/source/MaterialXGenOsl
-    ${PROJECT_SOURCE_DIR}/source/MaterialXGenMdl
-    ${PROJECT_SOURCE_DIR}/source/MaterialXRender
-    ${PROJECT_SOURCE_DIR}/source/MaterialXRenderHw
-    ${PROJECT_SOURCE_DIR}/source/MaterialXRenderGlsl
-    ${PROJECT_SOURCE_DIR}/source/MaterialXRenderOsl)
-
-find_package(Doxygen REQUIRED)
-
-foreach(FOLDER ${MATERIALX_DOXYGEN_SOURCE_FOLDERS})
-    file(GLOB FOLDER_HEADERS ${FOLDER}/*.h)
-    list(APPEND DOXYGEN_INPUT_LIST ${FOLDER_HEADERS})
-endforeach()
-
-string (REPLACE ";" " " DOXYGEN_INPUT_STR "${DOXYGEN_INPUT_LIST}")
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
-
-add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/MainPage.md
-                   COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/DeveloperGuide/MainPage.md ${CMAKE_CURRENT_BINARY_DIR}
-                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-                   DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/DeveloperGuide/MainPage.md)
-
-add_custom_command(OUTPUT ${DOXYGEN_HTML_OUTPUT_DIR}/index.html
-                   COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/DoxygenAwesome ${CMAKE_CURRENT_BINARY_DIR}
-                   COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/Images ${CMAKE_CURRENT_BINARY_DIR}
-                   COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
-                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-                   DEPENDS ${DOXYGEN_INPUT_LIST} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
-                   COMMENT "Generating HTML documentation: ${DOXYGEN_HTML_OUTPUT_DIR}/index.html")
-add_custom_target(MaterialXDocs ALL DEPENDS ${DOXYGEN_HTML_OUTPUT_DIR}/index.html)
-
-install(DIRECTORY ${DOXYGEN_HTML_OUTPUT_DIR}
-        DESTINATION "documents" MESSAGE_NEVER)
+
+if (MATERIALX_BUILD_DOXYGEN_DOCS)
+
+    set(DOXYGEN_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
+    set(DOXYGEN_HTML_OUTPUT_DIR ${DOXYGEN_OUTPUT_DIR}/html)
+    set(DOXYGEN_INPUT_LIST ${CMAKE_CURRENT_BINARY_DIR}/MainPage.md)
+
+    set(MATERIALX_DOXYGEN_SOURCE_FOLDERS
+        ${PROJECT_SOURCE_DIR}/source/MaterialXCore
+        ${PROJECT_SOURCE_DIR}/source/MaterialXFormat
+        ${PROJECT_SOURCE_DIR}/source/MaterialXGenShader
+        ${PROJECT_SOURCE_DIR}/source/MaterialXGenShader/Nodes
+        ${PROJECT_SOURCE_DIR}/source/MaterialXGenGlsl
+        ${PROJECT_SOURCE_DIR}/source/MaterialXGenGlsl/Nodes
+        ${PROJECT_SOURCE_DIR}/source/MaterialXGenOsl
+        ${PROJECT_SOURCE_DIR}/source/MaterialXGenMdl
+        ${PROJECT_SOURCE_DIR}/source/MaterialXRender
+        ${PROJECT_SOURCE_DIR}/source/MaterialXRenderHw
+        ${PROJECT_SOURCE_DIR}/source/MaterialXRenderGlsl
+        ${PROJECT_SOURCE_DIR}/source/MaterialXRenderOsl)
+
+    find_package(Doxygen REQUIRED)
+
+    foreach(FOLDER ${MATERIALX_DOXYGEN_SOURCE_FOLDERS})
+        file(GLOB FOLDER_HEADERS ${FOLDER}/*.h)
+        list(APPEND DOXYGEN_INPUT_LIST ${FOLDER_HEADERS})
+    endforeach()
+
+    string (REPLACE ";" " " DOXYGEN_INPUT_STR "${DOXYGEN_INPUT_LIST}")
+
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
+
+    add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/MainPage.md
+                       COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/DeveloperGuide/MainPage.md ${CMAKE_CURRENT_BINARY_DIR}
+                       WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                       DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/DeveloperGuide/MainPage.md)
+
+    add_custom_command(OUTPUT ${DOXYGEN_HTML_OUTPUT_DIR}/index.html
+                       COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/DoxygenAwesome ${CMAKE_CURRENT_BINARY_DIR}
+                       COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/Images ${CMAKE_CURRENT_BINARY_DIR}
+                       COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
+                       WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                       DEPENDS ${DOXYGEN_INPUT_LIST} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
+                       COMMENT "Generating HTML documentation: ${DOXYGEN_HTML_OUTPUT_DIR}/index.html")
+    add_custom_target(MaterialXDocs ALL DEPENDS ${DOXYGEN_HTML_OUTPUT_DIR}/index.html)
+
+    install(DIRECTORY ${DOXYGEN_HTML_OUTPUT_DIR}
+            DESTINATION "documents" MESSAGE_NEVER)
+endif()
+
+
+if (MATERIALX_BUILD_SPECIFICATION_DOCS)
+    set(MATERIALX_DATA_LIBRARY_TEMPLATE_DIR ${PROJECT_SOURCE_DIR}/templates)
+    set(MATERIALX_SPECIFICATION_TEMPLATE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/SpecificationTemplates)
+    #Ulitmately we may decide to move this directly to ${CMAKE_CURRENT_SOURCE_DIR}/Specification
+    set(MATERIALX_SPECIFICATION_DEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/SpecificationBuild) 
+
+    set(DOCS_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/nodes)
+    set(NODE_MD_MANIFEST ${DOCS_OUTPUT_DIR}/_manifest.txt)
+
+    file(GLOB_RECURSE MATERIALX_DATA_LIBRARY_MTLX_SOURCE_FILES
+            LIST_DIRECTORIES false
+            ${MATERIALX_DATA_LIBRARY_TEMPLATE_DIR}/*.mtlx)
+
+    file(GLOB_RECURSE MATERIALX_MD_SOURCE_FILES
+            LIST_DIRECTORIES false
+            ${MATERIALX_SPECIFICATION_TEMPLATE_SOURCE_DIR}/*.template.md)
+
+    add_custom_command(
+            OUTPUT ${NODE_MD_MANIFEST}
+            COMMAND ${CMAKE_COMMAND} -E make_directory ${DOCS_OUTPUT_DIR}
+            COMMAND ${CMAKE_COMMAND} -E make_directory ${MATERIALX_SPECIFICATION_DEST_DIR}
+            COMMAND MaterialXBuildDocs --sourceLibraryRoot ${MATERIALX_DATA_LIBRARY_TEMPLATE_DIR} --destDocRoot ${DOCS_OUTPUT_DIR} --sourceMDRoot  ${MATERIALX_SPECIFICATION_TEMPLATE_SOURCE_DIR} --destMDRoot  ${MATERIALX_SPECIFICATION_DEST_DIR} --manifestFile ${NODE_MD_MANIFEST}
+            DEPENDS ${MATERIALX_DATA_LIBRARY_MTLX_SOURCE_FILES} ${MATERIALX_MD_SOURCE_FILES} MaterialXBuildDocs
+    )
+
+    add_custom_target(MaterialXBuildSpecificationDocs ALL
+            DEPENDS ${NODE_MD_MANIFEST})
+
+endif()
+