Filtering for copyleft schemas

.github/workflows/pythonpublish.yml

@@ -1,33 +1,30 @@
-# This workflows will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
-name: Upload Python Package
+name: Upload Python Package to PyPI when a Release is Created
 
 on:
   release:
-    types: [published]
+    types: [created]
 
 jobs:
-  deploy:
-
+  pypi-publish:
+    name: Publish release to PyPI
     runs-on: ubuntu-latest
-
+    environment:
+      name: pypi
+      url: https://pypi.org/project/polyfile-weave
+    permissions:
+      id-token: write
     steps:
-    - uses: actions/checkout@v4.2.2
-      with:
-        submodules: recursive
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel
+      - name: Build package
+        run: |
+          python setup.py sdist bdist_wheel  # Could also be python -m build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/tests.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest] # windows-latest, macos-latest,
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     runs-on: ${{ matrix.os }}
 

compile_kaitai_parsers.py

@@ -12,6 +12,14 @@
 KAITAI_FORMAT_LIBRARY: Path = POLYFILE_DIR / "kaitai_struct_formats"
 KAITAI_PARSERS_DIR: Path = POLYFILE_DIR / "polyfile" / "kaitai" / "parsers"
 MANIFEST_PATH: Path = KAITAI_PARSERS_DIR / "manifest.json"
+EXCLUDE = [
+    './kaitai_struct_formats/firmware/broadcom_trx.ksy',
+    './kaitai_struct_formats/filesystem/vdi.ksy',
+    './kaitai_struct_formats/filesystem/lvm2.ksy',
+    './kaitai_struct_formats/image/pif.ksy',
+    './kaitai_struct_formats/scientific/nt_mdt/nt_mdt.ksy'
+]
+
 
 
 # Make sure the ktaitai_struct_formats submodlue is cloned:
@@ -48,6 +56,9 @@ def mtime(path: Path) -> datetime:
 
 
 def rebuild(force: bool = False):
+    # Get the list of copyleft-licensed files to exclude
+    excluded_paths = {Path(f).absolute() for f in EXCLUDE}
+
     # Remove the manifest file to force a rebuild:
     if force or not MANIFEST_PATH.exists():
         if MANIFEST_PATH.exists():
@@ -57,6 +68,9 @@ def rebuild(force: bool = False):
         # see if any of the files are out of date and need to be recompiled
         newest_definition: Optional[datetime] = None
         for definition in KAITAI_FORMAT_LIBRARY.glob("**/*.ksy"):
+            # Skip excluded files
+            if definition.absolute() in excluded_paths:
+                continue
             modtime = mtime(definition)
             if newest_definition is None or newest_definition < modtime:
                 newest_definition = modtime
@@ -71,7 +85,15 @@ def rebuild(force: bool = False):
             sys.stderr.write("Error: You must have kaitai-struct-compiler installed\nSee https://kaitai.io/#download\n")
             sys.exit(1)
 
-        num_files = sum(1 for _ in KAITAI_FORMAT_LIBRARY.glob("**/*.ksy"))
+        # Count non-excluded files
+        all_ksy_files = list(KAITAI_FORMAT_LIBRARY.glob("**/*.ksy"))
+        ksy_files_to_compile = [f for f in all_ksy_files if f.absolute() not in excluded_paths]
+        num_excluded = len(all_ksy_files) - len(ksy_files_to_compile)
+
+        if num_excluded > 0:
+            print(f"Excluding {num_excluded} copyleft-licensed KSY files from compilation")
+
+        num_files = len(ksy_files_to_compile)
 
         try:
             from tqdm import tqdm
@@ -99,7 +121,7 @@ def update(self, n: int):
             with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count()) as executor:
                 futures_to_path: Dict[concurrent.futures.Future, Path] = {
                     executor.submit(compile_ksy, file): file
-                    for file in KAITAI_FORMAT_LIBRARY.glob("**/*.ksy")
+                    for file in ksy_files_to_compile
                 }
                 for future in concurrent.futures.as_completed(futures_to_path):
                     t.update(1)