update translate script, add new py script to resolve bugs

Author: inoribea

.gitignore

@@ -40,9 +40,7 @@ Thumbs.db
 *.sqlite3
 
 # translation files
-untranslated_fr.tmp
-untranslated_ja.tmp
-untranslated_zh.tmp
+*.tmp
 
 locales/fr/LC_MESSAGES/messages.po.bak
 locales/ja/LC_MESSAGES/messages.po.bak

README.md

@@ -17,18 +17,19 @@ A practical command-line interface (CLI) for selected [astroquery](https://astro
 - **Gaia**: Cone search
 - **IRSA**: Infrared Science Archive queries
 - **Heasarc**: HEASARC Archive queries
-- **JPLSBDB**: JPL Small-Body Database queries
+- **JPL**: JPL Small-Body Database queries
 - **MAST**: Mikulski Archive for Space Telescopes queries
-- **NASA-ADS**: NASA Astrophysics Data System literature search and BibTeX retrieval, allows simple commands to search for "latest papers" or "highly cited reviews".
+- **ADS**: NASA Astrophysics Data System literature search and BibTeX retrieval, allows simple commands to search for "latest papers" or "highly cited reviews".
 - **NED**: NASA/IPAC Extragalactic Database name resolution
 - **NIST**: National Institute of Standards and Technology Atomic Spectra Database queries
+- **Exoplanet**: NASA Exoplanet Archive queries
 - **SDSS**: Sloan Digital Sky Survey queries
 - **ESO**: European Southern Observatory queries
 - **SIMBAD**: SIMBAD Astronomical Database basic query
 - **Splatalogue**: Molecular line queries
 - **VizieR**: VizieR Catalogue Database catalog search, basic query
 
-_Some modules and commands are not fully implemented. Please refer to `aqc --help` for the latest status._
+_Some modules and commands are not fully implemented. Aliases are available for some modules (e.g., `sim` for `simbad`, `viz` for `vizier`, `spl` for `splatalogue`, `hea` for `heasarc`, `exo` for `exoplanet`). Please refer to `aqc --help` for the latest status._
 
 ---
 
@@ -126,7 +127,16 @@ aqc --field simbad
 
 ### Updating Translations
 
-Helper scripts in the `locales/` directory assist with extracting, updating, and compiling translation files. See script comments for details.
+Helper scripts in the `locales/` directory assist with extracting, updating, and compiling translation files. The general workflow is as follows:
+
+1.  **Extract untranslated entries**: Run `locales/extract-untranslated.sh`. This script generates `untranslated_pot.tmp` (for new entries in `messages.pot`) and `untranslated_<lang>.tmp` files (for untranslated entries in language-specific `.po` files).
+2.  **Translate `untranslated_pot.tmp`**: Manually translate the entries in `locales/untranslated_pot.tmp`. These are new `msgid` entries that need to be added to all language files.
+3.  **Merge translations**: After translating `untranslated_pot.tmp`, merge these translations into the respective `untranslated_<lang>.tmp` files. This step typically involves copying the translated `msgstr` from `untranslated_pot.tmp` to the corresponding entries in `untranslated_<lang>.tmp`.
+4.  **Update `.po` files**: Run `locales/update-po.sh` to incorporate the translated entries from the `untranslated_<lang>.tmp` files into the `messages.po` files for each language.
+5.  **Check for updates**: Run `locales/check-update.sh` to ensure all translation files are consistent and up-to-date.
+6.  **Compile `.mo` files**: After updating `.po` files, compile them into `.mo` files using `locales/compile-mo.sh` (or similar command if not explicitly provided as a script).
+
+Refer to the comments within each script in the `locales/` directory for more detailed instructions.
 
 ---
 

locales/extract-untranslated.sh

@@ -9,166 +9,28 @@ PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
 # Full path to locales directory
 LOCALES_DIR="$SCRIPT_DIR" 
 DOMAIN="messages"
-POT_FILE="$LOCALES_DIR/$DOMAIN.pot" # This needs to be an absolute path
 
-# AWK script to extract and clean msgid strings, handling multi-line and unescaping
-# This script is written to a temporary file to avoid issues with 'read -r -d'
-AWK_EXTRACT_MSGID_SCRIPT_PATH="$LOCALES_DIR/awk_extract_msgid.awk"
-cat << 'EOF_AWK_EXTRACT_MSGID' > "$AWK_EXTRACT_MSGID_SCRIPT_PATH"
-BEGIN {
-    current_msgid_raw = "";
-    in_msgid_block = 0;
-}
-
-/^msgid / {
-    if (in_msgid_block) {
-        cleaned_msgid = current_msgid_raw;
-        sub(/^msgid /, "", cleaned_msgid);
-        if (length(cleaned_msgid) > 0 && substr(cleaned_msgid, 1, 1) == "\"" && substr(cleaned_msgid, length(cleaned_msgid), 1) == "\"") {
-            cleaned_msgid = substr(cleaned_msgid, 2, length(cleaned_msgid) - 2);
-        }
-        gsub(/\n"/, "\n", cleaned_msgid);
-        gsub(/\\"/, "\"", cleaned_msgid);
-        gsub(/\\n/, "\n", cleaned_msgid);
-        print cleaned_msgid;
-    }
-    current_msgid_raw = $0;
-    in_msgid_block = 1;
-    next;
-}
-
-/^msgstr / {
-    cleaned_msgid = current_msgid_raw;
-    sub(/^msgid /, "", cleaned_msgid);
-    if (length(cleaned_msgid) > 0 && substr(cleaned_msgid, 1, 1) == "\"" && substr(cleaned_msgid, length(cleaned_msgid), 1) == "\"") {
-        cleaned_msgid = substr(cleaned_msgid, 2, length(cleaned_msgid) - 2);
-    }
-    gsub(/\n"/, "\n", cleaned_msgid);
-    gsub(/\\"/, "\"", cleaned_msgid);
-    gsub(/\\n/, "\n", cleaned_msgid);
-    print cleaned_msgid;
+echo "Extracting untranslated and missing entries using Python..."
+for file in "$LOCALES_DIR"/$DOMAIN.pot "$LOCALES_DIR"/*/LC_MESSAGES/$DOMAIN.po; do
+    [ -f "$file" ] || continue
 
-    current_msgid_raw = "";
-    in_msgid_block = 0;
-    next;
-}
-
-/^"/ {
-    if (in_msgid_block) {
-        current_msgid_raw = current_msgid_raw "\n" $0;
-    }
-    next;
-}
-
-/^#/ {
-    next;
-}
-
-/^$/ {
-    if (in_msgid_block) {
-        cleaned_msgid = current_msgid_raw;
-        sub(/^msgid /, "", cleaned_msgid);
-        if (length(cleaned_msgid) > 0 && substr(cleaned_msgid, 1, 1) == "\"" && substr(cleaned_msgid, length(cleaned_msgid), 1) == "\"") {
-            cleaned_msgid = substr(cleaned_msgid, 2, length(cleaned_msgid) - 2);
-        }
-        gsub(/\n"/, "\n", cleaned_msgid);
-        gsub(/\\"/, "\"", cleaned_msgid);
-        gsub(/\\n/, "\n", cleaned_msgid);
-        print cleaned_msgid;
-    }
-    current_msgid_raw = "";
-    in_msgid_block = 0;
-    next;
-}
-
-END {
-    if (in_msgid_block && current_msgid_raw != "") {
-        cleaned_msgid = current_msgid_raw;
-        sub(/^msgid /, "", cleaned_msgid);
-        if (length(cleaned_msgid) > 0 && substr(cleaned_msgid, 1, 1) == "\"" && substr(cleaned_msgid, length(cleaned_msgid), 1) == "\"") {
-            cleaned_msgid = substr(cleaned_msgid, 2, length(cleaned_msgid) - 2);
-        }
-        gsub(/\n"/, "\n", cleaned_msgid);
-        gsub(/\\"/, "\"", cleaned_msgid);
-        gsub(/\\n/, "\n", cleaned_msgid);
-        print cleaned_msgid;
-    }
-}
-EOF_AWK_EXTRACT_MSGID
-
-echo "Extracting untranslated and missing entries..."
-for po in "$LOCALES_DIR"/*/LC_MESSAGES/$DOMAIN.po; do
-    [ -f "$po" ] || continue
-    lang=$(basename "$(dirname "$(dirname "$po")")")
-    tmpfile="$LOCALES_DIR/untranslated_${lang}.tmp"
-    tmpfile_pot_msgids="$LOCALES_DIR/all_pot_msgids.tmp"
-    tmpfile_po_translated_msgids="$LOCALES_DIR/po_translated_msgids_${lang}.tmp"
+    filename=$(basename "$file")
+    if [ "$filename" = "$DOMAIN.pot" ]; then
+        lang="pot"
+        tmpfile="$LOCALES_DIR/untranslated_${lang}.tmp"
+    else
+        lang=$(basename "$(dirname "$(dirname "$file")")")
+        tmpfile="$LOCALES_DIR/untranslated_${lang}.tmp"
+    fi
 
-    # Clear tmp files
+    # Clear tmp file
     : > "$tmpfile"
-    : > "$tmpfile_pot_msgids"
-    : > "$tmpfile_po_translated_msgids"
-
-    echo "--- Processing language: $lang ---"
 
-    # 1. Extract all msgids from the .pot file
-    echo "Step 1: Extracting all msgids from $POT_FILE..."
-    awk -f "$AWK_EXTRACT_MSGID_SCRIPT_PATH" "$POT_FILE" | sort -u > "$tmpfile_pot_msgids"
-    echo "Step 1 Complete: All msgids from $POT_FILE extracted to $tmpfile_pot_msgids"
+    echo "--- Processing file: $file (Language: $lang) ---"
 
-    # 2. Extract all *translated* msgids from the .po file
-    echo "Step 2: Extracting translated msgids from $po..."
-    grep -P -A 1 '^msgid ' "$po" | awk '
-        BEGIN {
-            current_msgid_block = "";
-            in_msgid_section = 0;
-        }
-        /^msgid / {
-            current_msgid_block = $0;
-            in_msgid_section = 1;
-            next;
-        }
-        /^msgstr "[^"]+"$/ { # msgstr is not empty
-            if (in_msgid_section) {
-                print current_msgid_block; # Print the msgid block
-            }
-            in_msgid_section = 0;
-            current_msgid_block = "";
-            next;
-        }
-        /^"/ { # Continuation lines for msgid
-            if (in_msgid_section) {
-                current_msgid_block = current_msgid_block "\n" $0;
-            }
-            next;
-        }
-        /^#/ { next; } # Ignore comments
-        /^$/ { # Empty line, end of entry
-            in_msgid_section = 0;
-            current_msgid_block = "";
-            next;
-        }
-        END {
-            # Handle case where last entry is a translated msgid
-            if (in_msgid_section && current_msgid_block != "") {
-                # This case is tricky, as we only print if msgstr is non-empty.
-                # The grep -A 1 should handle this by providing the msgstr line.
-                # So, no need to print here, as it would have been printed by the /^msgstr/ block.
-            }
-        }
-    ' | awk -f "$AWK_EXTRACT_MSGID_SCRIPT_PATH" - | sort -u > "$tmpfile_po_translated_msgids"
-    echo "Step 2 Complete: Translated msgids from $po extracted to $tmpfile_po_translated_msgids"
+    # Use the Python script to extract untranslated msgid,msgstr pairs
+    echo "Extracting untranslated entries from $file to $tmpfile..."
+    poetry run python "$LOCALES_DIR/extract_untranslated.py" "$file" "$tmpfile"
+    echo "Complete: Untranslated entries written to: $tmpfile"
 
-    # 3. Compare the two lists to find untranslated/missing entries
-    echo "Step 3: Comparing msgids to find untranslated and missing entries..."
-    comm -23 "$tmpfile_pot_msgids" "$tmpfile_po_translated_msgids" > "$tmpfile"
-    echo "Step 3 Complete: Untranslated and missing entries written to: $tmpfile"
-
-    # Clean up temporary files for the current language
-    echo "Cleaning up temporary files for language $lang..."
-    rm "$tmpfile_pot_msgids" "$tmpfile_po_translated_msgids"
 done
-
-# Clean up the AWK script file after all languages are processed
-echo "Cleaning up AWK script file..."
-rm "$AWK_EXTRACT_MSGID_SCRIPT_PATH"

locales/extract_untranslated.py

@@ -0,0 +1,41 @@
+import polib
+import os
+import sys
+
+def extract_untranslated(po_file_path, output_file_path):
+    """
+    Extracts untranslated or fuzzy entries from a .po file and writes them to an output file.
+    """
+    try:
+        po = polib.pofile(po_file_path)
+    except Exception as e:
+        print(f"Error reading PO file {po_file_path}: {e}", file=sys.stderr)
+        return
+
+    untranslated_entries = []
+    for entry in po:
+        # An entry is considered untranslated if msgstr is empty or identical to msgid
+        # and it's not a fuzzy translation.
+        if not entry.obsolete and (not entry.msgstr or entry.msgstr == entry.msgid or entry.fuzzy):
+            untranslated_entries.append(entry.msgid)
+
+    # Sort and write unique entries to the output file
+    untranslated_entries = sorted(list(set(untranslated_entries)))
+
+    try:
+        with open(output_file_path, 'w', encoding='utf-8') as f:
+            for entry in untranslated_entries:
+                # polib handles unescaping, so we just write the msgid directly
+                f.write(f"{entry}|||\n")
+        print(f"Complete: Untranslated entries written to: {output_file_path}")
+    except Exception as e:
+        print(f"Error writing to output file {output_file_path}: {e}", file=sys.stderr)
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: python extract_untranslated_py.py <po_file_path> <output_file_path>", file=sys.stderr)
+        sys.exit(1)
+
+    po_file = sys.argv[1]
+    output_file = sys.argv[2]
+    extract_untranslated(po_file, output_file)

pyproject.toml

@@ -27,6 +27,7 @@ pyvo = "^1.4.1"
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.2"
 snakeviz = "^2.2.0"
+polib = "^1.2.0"
 
 [build-system]
 requires = ["poetry-core"]