Skip to content

Commit 785ae79

Browse files
schivmeisterschivmeister
authored
Merge pull request #36 from meaningfy-ws/feature/DIF1-32_asciidoc-generation
Support for AsciiDoc template generation
2 parents 8b7930e + 8cd8a2a commit 785ae79

File tree

9 files changed

+307
-18
lines changed

9 files changed

+307
-18
lines changed

‎Makefile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,10 @@ generate_queries:
3232
# example: make generate_html_templates ap=dqgen/resources/aps/owl-core.csv output=./output
3333
generate_html_templates:
3434
@ python -m dqgen.entrypoints.cli.generate_html_template $(ap) $(output)
35+
36+
#-----------------------------------------------------------------------------
37+
# Generator commands
38+
#-----------------------------------------------------------------------------
39+
# example: make generate_asciidoc_templates ap=dqgen/resources/aps/owl-core.csv output=./output
40+
generate_asciidoc_templates:
41+
@ python -m dqgen.entrypoints.cli.generate_asciidoc_template $(ap) $(output)

‎dqgen/entrypoints/cli/generate_asciidoc_template.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/usr/bin/python3
2+
3+
# Date: 2024
4+
# Author: Generated for AsciiDoc template support
5+
import pathlib
6+
7+
import click
8+
9+
from dqgen.services.asciidoc_templates_generator import generate_asciidoc_templates_from_csv
10+
11+
12+
@click.command()
13+
@click.argument("file_path", type=click.Path(exists=True, dir_okay=False))
14+
@click.argument("output_folder", type=click.Path(dir_okay=True, file_okay=False))
15+
def generate_asciidoc_templates(file_path, output_folder):
16+
generate_asciidoc_templates_from_csv(pathlib.Path(file_path), pathlib.Path(output_folder))
17+
18+
19+
if __name__ == '__main__':
20+
generate_asciidoc_templates()
21+

‎dqgen/services/__init__.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@
1313
from jinja2 import Environment, PackageLoader
1414

1515
HTML_TEMPLATES = Environment(loader=PackageLoader("dqgen.resources", "html_templates"))
16+
ASCII_DOC_TEMPLATES = Environment(loader=PackageLoader("dqgen.resources", "asciidoc_templates"))
1617
QUERIES_TEMPLATES = Environment(loader=PackageLoader("dqgen.resources", "query_templates"))
1718
PATH_TO_STATIC_FOLDER = pathlib.Path(__file__).parent.parent / "resources" / "html_templates" / "static"
19+
PATH_TO_ASCIIDOC_STATIC_FOLDER = pathlib.Path(__file__).parent.parent / "resources" / "asciidoc_templates" / "static"
1820

1921
CLASSES_OPERATION_TEMPLATE_MAPPING = {
2022
"added_instance": QUERIES_TEMPLATES.get_template("instance_additions.rq"),
@@ -82,6 +84,9 @@
8284
TEMPLATE_AND_HTML_FILE_NAME_MAPPING = {"main.html": HTML_TEMPLATES.get_template("main.jinja2"),
8385
"statistics.html": HTML_TEMPLATES.get_template("statistics.jinja2")}
8486

87+
TEMPLATE_AND_ASCIIDOC_FILE_NAME_MAPPING = {"main.adoc": ASCII_DOC_TEMPLATES.get_template("main.jinja2"),
88+
"statistics.adoc": ASCII_DOC_TEMPLATES.get_template("statistics.jinja2")}
89+
8590
MULTI_LANGUAGES = ["en", "fr", "de", "es"]
8691
SINGLE_LANGUAGE = ["en"]
8792

‎dqgen/services/asciidoc_generator.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
from jinja2 import Template
2+
3+
from dqgen.adapters import template_builder
4+
from dqgen.services import QUERY_FALLBACK_LANGUAGES
5+
from dqgen.services.base_generator import BaseGenerator
6+
7+
8+
class AsciiDocGenerator(BaseGenerator):
9+
"""
10+
This class will generate an AsciiDoc template file from an AsciiDoc meta-template
11+
"""
12+
def __init__(self, cls: str, operation: str, output_folder_path: str, template: Template, prop: str = None,
13+
object_property: str = None, new_version_graph: str = None, old_version_graph: str = None,
14+
version_history_graph: str = None, languages: list = QUERY_FALLBACK_LANGUAGES, class_name: str = "", prop_name: str = "",
15+
obj_prop_name: str = ""):
16+
super().__init__(cls, operation, output_folder_path, template, prop, object_property, new_version_graph,
17+
old_version_graph, version_history_graph, languages)
18+
self.file_extension = "adoc"
19+
self.class_name = class_name
20+
self.prop_name = prop_name
21+
self.obj_prop_name = obj_prop_name
22+
23+
def build_template(self):
24+
"""
25+
This method builds a desired AsciiDoc template from the a meta-template
26+
:return: AsciiDoc template
27+
"""
28+
query_file = self.build_file_name(file_extension='rq')
29+
operation = self.operation.split("_")[0]
30+
return template_builder.build_html_template(jinja2_template=self.template, query_file=query_file,
31+
operation=operation, cls=self.cls, prop=self.prop,
32+
obj_prop=self.object_property, class_name=self.class_name,
33+
prop_name=self.prop_name,
34+
obj_prop_name=self.obj_prop_name)
35+

‎dqgen/services/asciidoc_templates_generator.py‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
#!/usr/bin/python3
2+
3+
# asciidoc_templates_generator.py
4+
# Date: 2024
5+
# Author: Generated for AsciiDoc template support
6+
import logging
7+
import pathlib
8+
from shutil import copytree, copyfile
9+
from pathlib import Path
10+
11+
import numpy as np
12+
import pandas as pd
13+
from dqgen.adapters.ap_reader import read_ap_from_csv
14+
15+
from dqgen.services import INSTANCE_OPERATIONS, PROPERTIES_OPERATIONS, REIFIED_PROPERTIES_OPERATIONS, ASCII_DOC_TEMPLATES, \
16+
PATH_TO_ASCIIDOC_STATIC_FOLDER, TEMPLATE_AND_ASCIIDOC_FILE_NAME_MAPPING
17+
from dqgen.services.asciidoc_generator import AsciiDocGenerator
18+
from dqgen.services.templates_data_source_builder import build_datasource_for_template, camel_case_to_words
19+
from dqgen.services.validate_application_profile import validate_application_profile
20+
21+
22+
def generate_class_level_asciidoc_templates(processed_csv_file: pd.DataFrame, asciidoc_output_folder_path):
23+
"""
24+
generate AsciiDoc templates for each class in the configuration CSV.
25+
"""
26+
27+
for cls in processed_csv_file["class"].unique():
28+
for operation in INSTANCE_OPERATIONS:
29+
class_name = cls.split(":")[1]
30+
class_folder_name = class_name.lower()
31+
output_folder_path = asciidoc_output_folder_path + "/" + class_folder_name
32+
pathlib.Path(output_folder_path).mkdir(parents=True, exist_ok=True)
33+
AsciiDocGenerator(cls=cls, operation=operation,
34+
class_name=camel_case_to_words(class_name).title(),
35+
output_folder_path=output_folder_path,
36+
template=ASCII_DOC_TEMPLATES.get_template("instance.jinja2")).to_file()
37+
logging.info("Generated instance AsciiDoc templates ...")
38+
39+
40+
def generate_property_level_asciidoc_templates(processed_csv_file: pd.DataFrame, asciidoc_output_folder_path):
41+
"""
42+
generate AsciiDoc template for data properties and their values for each instance in the configuration CSV
43+
"""
44+
for index, row in processed_csv_file.iterrows():
45+
46+
if not row["object property"]:
47+
for operation in PROPERTIES_OPERATIONS:
48+
class_folder_name = row["class"].split(":")[1].lower()
49+
if row["property group"] and row["property group"] is not np.NaN:
50+
property_group_folder = row["property group"].replace(" ", "_")
51+
output_folder_path = asciidoc_output_folder_path + "/" + class_folder_name + "/" + property_group_folder
52+
else:
53+
output_folder_path = asciidoc_output_folder_path + "/" + class_folder_name
54+
pathlib.Path(output_folder_path).mkdir(parents=True, exist_ok=True)
55+
AsciiDocGenerator(cls=row["class"],
56+
prop=row["property"],
57+
prop_name=camel_case_to_words(row["property"].split(":")[1]).lower(),
58+
operation=operation,
59+
output_folder_path=output_folder_path,
60+
template=ASCII_DOC_TEMPLATES.get_template("property.jinja2")).to_file()
61+
62+
logging.info("Generated property AsciiDoc templates ...")
63+
64+
65+
def generate_reified_property_level_asciidoc_templates(processed_csv_file: pd.DataFrame, asciidoc_output_folder_path):
66+
"""
67+
generate AsciiDoc template of reified structures for each instance in the configuration CSV
68+
"""
69+
for index, row in processed_csv_file.iterrows():
70+
if row["object property"]:
71+
for operation in REIFIED_PROPERTIES_OPERATIONS:
72+
class_folder_name = row["class"].split(":")[1].lower()
73+
if row["property group"] and row["property group"] is not np.NaN:
74+
property_group_folder = row["property group"].replace(" ", "_")
75+
output_folder_path = asciidoc_output_folder_path + "/" + class_folder_name + "/" + property_group_folder
76+
else:
77+
output_folder_path = asciidoc_output_folder_path + "/" + class_folder_name
78+
pathlib.Path(output_folder_path).mkdir(parents=True, exist_ok=True)
79+
AsciiDocGenerator(cls=row["class"],
80+
prop=row["property"],
81+
object_property=row["object property"],
82+
prop_name=camel_case_to_words(row["property"].split(":")[1]).lower(),
83+
operation=operation,
84+
output_folder_path=output_folder_path,
85+
template=ASCII_DOC_TEMPLATES.get_template("reified_property.jinja2")).to_file()
86+
87+
logging.info("Generated reified property AsciiDoc templates ...")
88+
89+
90+
def generate_asciidoc_template(processed_csv_file: pd.DataFrame, asciidoc_output_folder_path, template, file_name):
91+
"""
92+
Builds an AsciiDoc page and puts into a specified folder
93+
:param file_name:
94+
:param template:
95+
:param processed_csv_file:
96+
:param asciidoc_output_folder_path:
97+
:return:
98+
"""
99+
100+
data_source = build_datasource_for_template(processed_csv_file=processed_csv_file, file_extension='adoc')
101+
build_template = template.stream(data_source=data_source)
102+
build_template.dump(asciidoc_output_folder_path + "/" + file_name)
103+
104+
105+
def copy_files_from_static_folder(file_list: list, destination_folder: str):
106+
"""
107+
Copy the files from the static folder to a specified destination
108+
:param file_list:
109+
:param destination_folder:
110+
"""
111+
for file in file_list:
112+
file_name = file.name
113+
copyfile(file, destination_folder + "/" + file_name)
114+
115+
116+
def generate_asciidoc_templates_from_csv(ap_file_path: pathlib.Path, output_base_dir: pathlib.Path):
117+
"""
118+
generates a set of AsciiDoc templates from the configuration CSV
119+
"""
120+
processed_csv_file = read_ap_from_csv(ap_file_path)
121+
validate_application_profile(application_profile_df=processed_csv_file)
122+
output = Path(output_base_dir) / ap_file_path.stem
123+
asciidoc_output = output / "asciidoc"
124+
asciidoc_output.mkdir(parents=True, exist_ok=True)
125+
126+
generate_class_level_asciidoc_templates(processed_csv_file=processed_csv_file, asciidoc_output_folder_path=str(asciidoc_output))
127+
generate_property_level_asciidoc_templates(processed_csv_file=processed_csv_file,
128+
asciidoc_output_folder_path=str(asciidoc_output))
129+
generate_reified_property_level_asciidoc_templates(processed_csv_file=processed_csv_file,
130+
asciidoc_output_folder_path=str(asciidoc_output))
131+
132+
for file_name, template in TEMPLATE_AND_ASCIIDOC_FILE_NAME_MAPPING.items():
133+
generate_asciidoc_template(processed_csv_file=processed_csv_file,
134+
asciidoc_output_folder_path=str(asciidoc_output), template=template, file_name=file_name)
135+
136+
# copy static files into the generated asciidoc output directory
137+
copytree(PATH_TO_ASCIIDOC_STATIC_FOLDER, str(asciidoc_output), dirs_exist_ok=True)
138+

‎dqgen/services/html_templates_generator.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,8 @@
66
# Email: costezki.eugen@gmail.com
77
import logging
88
import pathlib
9-
from distutils.dir_util import copy_tree
9+
from shutil import copytree, copyfile
1010
from pathlib import Path
11-
from shutil import copyfile
1211

1312
import numpy as np
1413
import pandas as pd
@@ -18,7 +17,7 @@
1817
from dqgen.services import INSTANCE_OPERATIONS, PROPERTIES_OPERATIONS, REIFIED_PROPERTIES_OPERATIONS, HTML_TEMPLATES, \
1918
PATH_TO_STATIC_FOLDER, TEMPLATE_AND_HTML_FILE_NAME_MAPPING
2019
from dqgen.services.html_generator import HtmlGenerator
21-
from dqgen.services.html_templates_data_source_builder import build_datasource_for_html_template, camel_case_to_words
20+
from dqgen.services.templates_data_source_builder import build_datasource_for_template, camel_case_to_words
2221
from dqgen.services.validate_application_profile import validate_application_profile
2322

2423

@@ -100,7 +99,7 @@ def generate_html_template(processed_csv_file: pd.DataFrame, html_output_folder_
10099
:return:
101100
"""
102101

103-
data_source = build_datasource_for_html_template(processed_csv_file=processed_csv_file)
102+
data_source = build_datasource_for_template(processed_csv_file=processed_csv_file)
104103
build_template = template.stream(data_source=data_source)
105104
build_template.dump(html_output_folder_path + "/" + file_name)
106105

@@ -136,4 +135,5 @@ def generate_html_templates_from_csv(ap_file_path: pathlib.Path, output_base_dir
136135
generate_html_template(processed_csv_file=processed_csv_file,
137136
html_output_folder_path=str(html_output), template=template, file_name=file_name)
138137

139-
copy_tree(PATH_TO_STATIC_FOLDER, str(html_output))
138+
# copy static files into the generated html output directory
139+
copytree(PATH_TO_STATIC_FOLDER, str(html_output), dirs_exist_ok=True)

dqgen/services/html_templates_data_source_builder.py renamed to dqgen/services/templates_data_source_builder.py

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,20 +16,21 @@ def camel_case_to_words(name: str):
1616
return ' '.join(words)
1717

1818

19-
def generate_file_data(cls, class_folder_name, operation, prop=None, obj_prop=None):
19+
def generate_file_data(cls, class_folder_name, operation, prop=None, obj_prop=None, file_extension="html"):
2020
"""
2121
This method will return a query file path and a count query file name
2222
:param cls:
2323
:param class_folder_name:
2424
:param operation:
2525
:param prop:
2626
:param obj_prop:
27+
:param file_extension: extension to use for the generated files (html/adoc)
2728
:return:
2829
"""
2930
file_path = make_file_path(output_folder_path=class_folder_name,
3031
file_name=make_file_name(operation=operation,
3132
cls=cls,
32-
file_extension="html",
33+
file_extension=file_extension,
3334
prop=prop, obj_prop=obj_prop))
3435
count_file_name = make_file_name(operation="count_" + operation,
3536
cls=cls,
@@ -39,7 +40,7 @@ def generate_file_data(cls, class_folder_name, operation, prop=None, obj_prop=No
3940

4041

4142
def iterate_operations(operation_list, file_paths, count_queries, cls, class_folder_name, prop=None,
42-
obj_prop=None):
43+
obj_prop=None, file_extension="html"):
4344
"""
4445
This method will iterate through a list of operations. It will put one query file path in the file paths list
4546
and one count query name in the count query list
@@ -50,22 +51,25 @@ def iterate_operations(operation_list, file_paths, count_queries, cls, class_fol
5051
:param class_folder_name:
5152
:param prop:
5253
:param obj_prop:
54+
:param file_extension: extension to use for generated file paths
5355
:return:
5456
"""
5557
for operation in operation_list:
5658
file_path, count_file_name = generate_file_data(cls=cls, class_folder_name=class_folder_name,
57-
operation=operation, prop=prop, obj_prop=obj_prop)
59+
operation=operation, prop=prop, obj_prop=obj_prop,
60+
file_extension=file_extension)
5861
file_paths.append(file_path)
5962
count_queries.append(count_file_name)
6063

6164

62-
def add_instance_changes(data_source, cls, class_name, class_folder_name):
65+
def add_instance_changes(data_source, cls, class_name, class_folder_name, file_extension="html"):
6366
"""
6467
This method will build the necessary data at the class level and it will add it to a data source dictionary
6568
:param data_source:
6669
:param cls:
6770
:param class_name:
6871
:param class_folder_name:
72+
:param file_extension: extension to use for generated file paths
6973
:return:
7074
"""
7175
if "instance_changes" not in data_source[cls].keys():
@@ -75,7 +79,8 @@ def add_instance_changes(data_source, cls, class_name, class_folder_name):
7579
instance_count_queries = []
7680

7781
iterate_operations(operation_list=INSTANCE_OPERATIONS, file_paths=instance_file_paths,
78-
count_queries=instance_count_queries, cls=cls, class_folder_name=class_folder_name)
82+
count_queries=instance_count_queries, cls=cls, class_folder_name=class_folder_name,
83+
file_extension=file_extension)
7984

8085
data_source[cls]["instance_changes"].update(
8186
{"files": instance_file_paths, "count_queries": instance_count_queries})
@@ -104,10 +109,11 @@ def add_prop_group_details(data_source, prop_group_value, cls, prop_name, prop_f
104109
{prop_name: {"files": count_prop_file_paths, "label": prop_name}})
105110

106111

107-
def build_datasource_for_html_template(processed_csv_file: pd.DataFrame) -> dict:
112+
def build_datasource_for_template(processed_csv_file: pd.DataFrame, file_extension: str = "html") -> dict:
108113
"""
109114
This method will build a data source dictionary from a given application profile dataframe
110115
:param processed_csv_file:
116+
:param file_extension: extension to use for generated file paths (default: html)
111117
:return:
112118
"""
113119
data_source = {}
@@ -121,7 +127,7 @@ def build_datasource_for_html_template(processed_csv_file: pd.DataFrame) -> dict
121127
data_source[row["class"]] = {"label": camel_case_to_words(class_name).title(), "prop_groups": {}}
122128

123129
add_instance_changes(data_source=data_source, cls=row["class"], class_name=class_name,
124-
class_folder_name=class_folder_name)
130+
class_folder_name=class_folder_name, file_extension=file_extension)
125131

126132
prop_file_paths = []
127133
count_queries = []
@@ -130,16 +136,20 @@ def build_datasource_for_html_template(processed_csv_file: pd.DataFrame) -> dict
130136
prop_name = row["property"]
131137
iterate_operations(operation_list=PROPERTIES_OPERATIONS, file_paths=prop_file_paths,
132138
count_queries=count_queries, cls=row["class"],
133-
class_folder_name=f'{class_folder_name}/{prop_group_folder}', prop=row["property"])
139+
class_folder_name=f'{class_folder_name}/{prop_group_folder}', prop=row["property"],
140+
file_extension=file_extension)
134141
else:
135142
prop_name = row["property"] + "/" + row["object property"]
136143
iterate_operations(operation_list=REIFIED_PROPERTIES_OPERATIONS, file_paths=prop_file_paths,
137144
count_queries=count_queries, cls=row["class"],
138145
class_folder_name=f'{class_folder_name}/{prop_group_folder}', prop=row["property"],
139-
obj_prop=row["object property"])
146+
obj_prop=row["object property"], file_extension=file_extension)
140147

141148
add_prop_group_details(data_source=data_source, prop_group_value=row["property group"], cls=row["class"],
142149
prop_name=prop_name, prop_file_paths=prop_file_paths,
143150
count_prop_file_paths=count_queries)
144151

145152
return data_source
153+
154+
# Backwards compatibility alias
155+
build_datasource_for_html_template = build_datasource_for_template

0 commit comments

Comments
 (0)