fix(test): fix broken AsciiDoc imports, refactor data source builder

The query files inserted within the AsciiDoc templates appeared to be
HTML files. Add a `file_extension` parameter to produce whatever file
extension is needed, currently either `.html` or `.adoc`. Update/add
tests accordingly so such accidents can be captured in the future.

Author: schivmeister

dqgen/services/asciidoc_templates_generator.py

@@ -5,9 +5,8 @@
 # Author: Generated for AsciiDoc template support
 import logging
 import pathlib
-from distutils.dir_util import copy_tree
+from shutil import copytree, copyfile
 from pathlib import Path
-from shutil import copyfile
 
 import numpy as np
 import pandas as pd
@@ -16,7 +15,7 @@
 from dqgen.services import INSTANCE_OPERATIONS, PROPERTIES_OPERATIONS, REIFIED_PROPERTIES_OPERATIONS, ASCII_DOC_TEMPLATES, \
     PATH_TO_ASCIIDOC_STATIC_FOLDER, TEMPLATE_AND_ASCIIDOC_FILE_NAME_MAPPING
 from dqgen.services.asciidoc_generator import AsciiDocGenerator
-from dqgen.services.html_templates_data_source_builder import build_datasource_for_html_template, camel_case_to_words
+from dqgen.services.templates_data_source_builder import build_datasource_for_template, camel_case_to_words
 from dqgen.services.validate_application_profile import validate_application_profile
 
 
@@ -98,7 +97,7 @@ def generate_asciidoc_template(processed_csv_file: pd.DataFrame, asciidoc_output
     :return:
     """
 
-    data_source = build_datasource_for_html_template(processed_csv_file=processed_csv_file)
+    data_source = build_datasource_for_template(processed_csv_file=processed_csv_file, file_extension='adoc')
     build_template = template.stream(data_source=data_source)
     build_template.dump(asciidoc_output_folder_path + "/" + file_name)
 
@@ -134,5 +133,6 @@ def generate_asciidoc_templates_from_csv(ap_file_path: pathlib.Path, output_base
         generate_asciidoc_template(processed_csv_file=processed_csv_file,
                                asciidoc_output_folder_path=str(asciidoc_output), template=template, file_name=file_name)
 
-    copy_tree(PATH_TO_ASCIIDOC_STATIC_FOLDER, str(asciidoc_output))
+    # copy static files into the generated asciidoc output directory
+    copytree(PATH_TO_ASCIIDOC_STATIC_FOLDER, str(asciidoc_output), dirs_exist_ok=True)
 

dqgen/services/html_templates_generator.py

@@ -6,9 +6,8 @@
 # Email: costezki.eugen@gmail.com
 import logging
 import pathlib
-from distutils.dir_util import copy_tree
+from shutil import copytree, copyfile
 from pathlib import Path
-from shutil import copyfile
 
 import numpy as np
 import pandas as pd
@@ -18,7 +17,7 @@
 from dqgen.services import INSTANCE_OPERATIONS, PROPERTIES_OPERATIONS, REIFIED_PROPERTIES_OPERATIONS, HTML_TEMPLATES, \
     PATH_TO_STATIC_FOLDER, TEMPLATE_AND_HTML_FILE_NAME_MAPPING
 from dqgen.services.html_generator import HtmlGenerator
-from dqgen.services.html_templates_data_source_builder import build_datasource_for_html_template, camel_case_to_words
+from dqgen.services.templates_data_source_builder import build_datasource_for_template, camel_case_to_words
 from dqgen.services.validate_application_profile import validate_application_profile
 
 
@@ -100,7 +99,7 @@ def generate_html_template(processed_csv_file: pd.DataFrame, html_output_folder_
     :return:
     """
 
-    data_source = build_datasource_for_html_template(processed_csv_file=processed_csv_file)
+    data_source = build_datasource_for_template(processed_csv_file=processed_csv_file)
     build_template = template.stream(data_source=data_source)
     build_template.dump(html_output_folder_path + "/" + file_name)
 
@@ -136,4 +135,5 @@ def generate_html_templates_from_csv(ap_file_path: pathlib.Path, output_base_dir
         generate_html_template(processed_csv_file=processed_csv_file,
                                html_output_folder_path=str(html_output), template=template, file_name=file_name)
 
-    copy_tree(PATH_TO_STATIC_FOLDER, str(html_output))
+    # copy static files into the generated html output directory
+    copytree(PATH_TO_STATIC_FOLDER, str(html_output), dirs_exist_ok=True)

…es/html_templates_data_source_builder.py …ervices/templates_data_source_builder.pydqgen/services/html_templates_data_source_builder.py renamed to dqgen/services/templates_data_source_builder.py dqgen/services/html_templates_data_source_builder.py renamed to dqgen/services/templates_data_source_builder.py

@@ -16,20 +16,21 @@ def camel_case_to_words(name: str):
     return ' '.join(words)
 
 
-def generate_file_data(cls, class_folder_name, operation, prop=None, obj_prop=None):
+def generate_file_data(cls, class_folder_name, operation, prop=None, obj_prop=None, file_extension="html"):
     """
     This method will return a query file path and a count query file name
     :param cls: 
     :param class_folder_name: 
     :param operation: 
     :param prop: 
     :param obj_prop: 
+    :param file_extension: extension to use for the generated files (html/adoc)
     :return: 
     """
     file_path = make_file_path(output_folder_path=class_folder_name,
                                file_name=make_file_name(operation=operation,
                                                         cls=cls,
-                                                        file_extension="html",
+                                                        file_extension=file_extension,
                                                         prop=prop, obj_prop=obj_prop))
     count_file_name = make_file_name(operation="count_" + operation,
                                      cls=cls,
@@ -39,7 +40,7 @@ def generate_file_data(cls, class_folder_name, operation, prop=None, obj_prop=No
 
 
 def iterate_operations(operation_list, file_paths, count_queries, cls, class_folder_name, prop=None,
-                       obj_prop=None):
+                       obj_prop=None, file_extension="html"):
     """
     This method will iterate through a list of operations. It will put one query file path in the file paths list
      and one count query name in the count query list
@@ -50,22 +51,25 @@ def iterate_operations(operation_list, file_paths, count_queries, cls, class_fol
     :param class_folder_name: 
     :param prop: 
     :param obj_prop: 
+    :param file_extension: extension to use for generated file paths
     :return: 
     """
     for operation in operation_list:
         file_path, count_file_name = generate_file_data(cls=cls, class_folder_name=class_folder_name,
-                                                        operation=operation, prop=prop, obj_prop=obj_prop)
+                                                        operation=operation, prop=prop, obj_prop=obj_prop,
+                                                        file_extension=file_extension)
         file_paths.append(file_path)
         count_queries.append(count_file_name)
 
 
-def add_instance_changes(data_source, cls, class_name, class_folder_name):
+def add_instance_changes(data_source, cls, class_name, class_folder_name, file_extension="html"):
     """
     This method will build the necessary data at the class level and it will add it to a data source dictionary
     :param data_source: 
     :param cls: 
     :param class_name: 
     :param class_folder_name: 
+    :param file_extension: extension to use for generated file paths
     :return: 
     """
     if "instance_changes" not in data_source[cls].keys():
@@ -75,7 +79,8 @@ def add_instance_changes(data_source, cls, class_name, class_folder_name):
         instance_count_queries = []
 
         iterate_operations(operation_list=INSTANCE_OPERATIONS, file_paths=instance_file_paths,
-                           count_queries=instance_count_queries, cls=cls, class_folder_name=class_folder_name)
+                           count_queries=instance_count_queries, cls=cls, class_folder_name=class_folder_name,
+                           file_extension=file_extension)
 
         data_source[cls]["instance_changes"].update(
             {"files": instance_file_paths, "count_queries": instance_count_queries})
@@ -104,10 +109,11 @@ def add_prop_group_details(data_source, prop_group_value, cls, prop_name, prop_f
             {prop_name: {"files": count_prop_file_paths, "label": prop_name}})
 
 
-def build_datasource_for_html_template(processed_csv_file: pd.DataFrame) -> dict:
+def build_datasource_for_template(processed_csv_file: pd.DataFrame, file_extension: str = "html") -> dict:
     """
     This method will build a data source dictionary from a given application profile dataframe
     :param processed_csv_file: 
+    :param file_extension: extension to use for generated file paths (default: html)
     :return: 
     """
     data_source = {}
@@ -121,7 +127,7 @@ def build_datasource_for_html_template(processed_csv_file: pd.DataFrame) -> dict
             data_source[row["class"]] = {"label": camel_case_to_words(class_name).title(), "prop_groups": {}}
 
         add_instance_changes(data_source=data_source, cls=row["class"], class_name=class_name,
-                             class_folder_name=class_folder_name)
+                             class_folder_name=class_folder_name, file_extension=file_extension)
 
         prop_file_paths = []
         count_queries = []
@@ -130,16 +136,20 @@ def build_datasource_for_html_template(processed_csv_file: pd.DataFrame) -> dict
             prop_name = row["property"]
             iterate_operations(operation_list=PROPERTIES_OPERATIONS, file_paths=prop_file_paths,
                                count_queries=count_queries, cls=row["class"],
-                               class_folder_name=f'{class_folder_name}/{prop_group_folder}', prop=row["property"])
+                               class_folder_name=f'{class_folder_name}/{prop_group_folder}', prop=row["property"],
+                               file_extension=file_extension)
         else:
             prop_name = row["property"] + "/" + row["object property"]
             iterate_operations(operation_list=REIFIED_PROPERTIES_OPERATIONS, file_paths=prop_file_paths,
                                count_queries=count_queries, cls=row["class"],
                                class_folder_name=f'{class_folder_name}/{prop_group_folder}', prop=row["property"],
-                               obj_prop=row["object property"])
+                               obj_prop=row["object property"], file_extension=file_extension)
 
         add_prop_group_details(data_source=data_source, prop_group_value=row["property group"], cls=row["class"],
                                prop_name=prop_name, prop_file_paths=prop_file_paths,
                                count_prop_file_paths=count_queries)
 
     return data_source
+
+# Backwards compatibility alias
+build_datasource_for_html_template = build_datasource_for_template

…ts/unit/test_html_data_source_builder.py tests/unit/test_data_source_builder.pytests/unit/test_html_data_source_builder.py renamed to tests/unit/test_data_source_builder.py tests/unit/test_html_data_source_builder.py renamed to tests/unit/test_data_source_builder.py

@@ -1,7 +1,7 @@
 import pathlib
 
 from dqgen.adapters.ap_reader import read_ap_from_csv
-from dqgen.services.html_templates_data_source_builder import camel_case_to_words, build_datasource_for_html_template, \
+from dqgen.services.templates_data_source_builder import camel_case_to_words, build_datasource_for_template, \
     generate_file_data, iterate_operations, add_instance_changes, add_prop_group_details
 
 
@@ -50,11 +50,47 @@ def test_add_prop_group_details():
                                                                                "preferred labels"].keys())
 
 
-def test_build_datasource_for_html_template():
+def test_build_datasource_for_hmtl_template():
     path_to_csv_file = pathlib.Path(__file__).parent.parent / "test_data" / "aps" / "src_ap_mod.csv"
     df = read_ap_from_csv(path_to_csv_file)
-    data_source = build_datasource_for_html_template(processed_csv_file=df)
+    data_source = build_datasource_for_template(processed_csv_file=df)
     print(data_source)
     assert isinstance(data_source, dict)
     assert 'skos:Concept', 'skos:Collection' in data_source.keys()
     assert "label", "prop_groups" in data_source["skos:Concept"].keys()
+
+    # check at least one class-level instance file uses .html
+    instance_files = data_source.get("skos:Concept", {}).get("instance_changes", {}).get("files", [])
+    assert any(f.endswith('.html') for f in instance_files)
+
+    # check a property-level file uses .html
+    prop_groups = data_source.get("skos:Concept", {}).get("prop_groups", {})
+    found = False
+    for group in prop_groups.values():
+        paths = group.get("query_template_file_paths", [])
+        if any(p.endswith('.html') for p in paths):
+            found = True
+            break
+    assert found, "No .html file paths found in property groups"
+
+
+def test_build_datasource_for_adoc_template():
+    # Ensure the data source builder emits .adoc file paths when requested
+    path_to_csv_file = pathlib.Path(__file__).parent.parent / "test_data" / "aps" / "src_ap_mod.csv"
+    df = read_ap_from_csv(path_to_csv_file)
+    data_source = build_datasource_for_template(processed_csv_file=df, file_extension="adoc")
+
+    # check at least one class-level instance file uses .adoc
+    instance_files = data_source.get("skos:Concept", {}).get("instance_changes", {}).get("files", [])
+    assert any(f.endswith('.adoc') for f in instance_files)
+
+    # check a property-level file uses .adoc
+    prop_groups = data_source.get("skos:Concept", {}).get("prop_groups", {})
+    # find any prop group entry and check its query template file paths
+    found = False
+    for group in prop_groups.values():
+        paths = group.get("query_template_file_paths", [])
+        if any(p.endswith('.adoc') for p in paths):
+            found = True
+            break
+    assert found, "No .adoc file paths found in property groups"