fetch-ucd

#!/bin/sh
#
# Fetch Unicode Character Database files for Lingenic-Text.
#
# Downloads all UCD, UCA, IDNA, and emoji data files required by the library
# and its conformance tests.  Files are placed in the ucd/ directory.
#
# Usage:
#   ./scripts/fetch-ucd.sh              # Unicode 17.0 (default)
#   ./scripts/fetch-ucd.sh 16.0.0       # specific version
#
# Requires: curl or wget

set -e

VERSION="${1:-17.0.0}"
MAJOR_MINOR=$(echo "$VERSION" | sed 's/\.[0-9]*$//')

UCD_BASE="https://www.unicode.org/Public/${VERSION}/ucd"
UCA_BASE="https://www.unicode.org/Public/UCA/${VERSION}"
UCA_FALLBACK="https://www.unicode.org/Public/UCA/latest"
IDNA_BASE="https://www.unicode.org/Public/${VERSION}/idna"
EMOJI_BASE="https://www.unicode.org/Public/emoji/${MAJOR_MINOR}"
EMOJI_FALLBACK="https://www.unicode.org/Public/emoji/latest"

SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
ROOT_DIR=$(cd "$SCRIPT_DIR/.." && pwd)
OUT_DIR="$ROOT_DIR/ucd"

# ---------------------------------------------------------------------------
#  Detect download tool
# ---------------------------------------------------------------------------

if command -v curl >/dev/null 2>&1; then
    fetch() { curl -fSL --retry 3 -o "$1" "$2"; }
elif command -v wget >/dev/null 2>&1; then
    fetch() { wget -q -O "$1" "$2"; }
else
    echo "error: curl or wget required" >&2
    exit 1
fi

# ---------------------------------------------------------------------------
#  Download helper — skips files that already exist with correct size
# ---------------------------------------------------------------------------

download() {
    _url="$1"
    _dest="$2"
    _fallback="${3:-}"

    _name=$(basename "$_dest")
    if [ -f "$_dest" ]; then
        printf "  %-45s  skip (exists)\n" "$_name"
        return
    fi

    printf "  %-45s  " "$_name"
    if fetch "$_dest" "$_url" 2>/dev/null; then
        _size=$(wc -c < "$_dest" | tr -d ' ')
        printf "%s bytes\n" "$_size"
    elif [ -n "$_fallback" ] && fetch "$_dest" "$_fallback" 2>/dev/null; then
        _size=$(wc -c < "$_dest" | tr -d ' ')
        printf "%s bytes (fallback)\n" "$_size"
    else
        printf "FAILED\n"
        rm -f "$_dest"
        FAILURES=$((FAILURES + 1))
    fi
}

# ---------------------------------------------------------------------------
#  Main
# ---------------------------------------------------------------------------

FAILURES=0

echo "Lingenic-Text UCD fetch — Unicode ${VERSION}"
echo "Target: ${OUT_DIR}"
echo ""

mkdir -p "$OUT_DIR"
mkdir -p "$OUT_DIR/CollationTest"

# --- Core UCD files (used at initialization) ---

echo "Core UCD files:"

download "$UCD_BASE/UnicodeData.txt"                    "$OUT_DIR/UnicodeData.txt"
download "$UCD_BASE/Scripts.txt"                        "$OUT_DIR/Scripts.txt"
download "$UCD_BASE/ScriptExtensions.txt"               "$OUT_DIR/ScriptExtensions.txt"
download "$UCD_BASE/PropertyValueAliases.txt"           "$OUT_DIR/PropertyValueAliases.txt"
download "$UCD_BASE/SpecialCasing.txt"                  "$OUT_DIR/SpecialCasing.txt"
download "$UCD_BASE/CaseFolding.txt"                    "$OUT_DIR/CaseFolding.txt"
download "$UCD_BASE/DerivedNormalizationProps.txt"      "$OUT_DIR/DerivedNormalizationProps.txt"
download "$UCD_BASE/DerivedCoreProperties.txt"          "$OUT_DIR/DerivedCoreProperties.txt"
download "$UCD_BASE/EastAsianWidth.txt"                 "$OUT_DIR/EastAsianWidth.txt"
download "$UCD_BASE/LineBreak.txt"                      "$OUT_DIR/LineBreak.txt"
download "$UCD_BASE/BidiBrackets.txt"                   "$OUT_DIR/BidiBrackets.txt"

echo ""
echo "Auxiliary UCD files:"

download "$UCD_BASE/auxiliary/GraphemeBreakProperty.txt"   "$OUT_DIR/GraphemeBreakProperty.txt"
download "$UCD_BASE/auxiliary/WordBreakProperty.txt"       "$OUT_DIR/WordBreakProperty.txt"
download "$UCD_BASE/auxiliary/SentenceBreakProperty.txt"   "$OUT_DIR/SentenceBreakProperty.txt"

echo ""
echo "Extracted UCD files:"

download "$UCD_BASE/extracted/DerivedGeneralCategory.txt"  "$OUT_DIR/DerivedGeneralCategory.txt"
download "$UCD_BASE/extracted/DerivedBidiClass.txt"        "$OUT_DIR/DerivedBidiClass.txt"
download "$UCD_BASE/extracted/DerivedJoiningType.txt"      "$OUT_DIR/DerivedJoiningType.txt"

# --- Emoji ---

echo ""
echo "Emoji data files:"

download "$UCD_BASE/emoji/emoji-data.txt"               "$OUT_DIR/emoji-data.txt"
download "$EMOJI_BASE/emoji-test.txt"                    "$OUT_DIR/emoji-test.txt" \
         "$EMOJI_FALLBACK/emoji-test.txt"

# --- UCA (Collation) ---

echo ""
echo "UCA (Collation) files:"

download "$UCA_BASE/allkeys.txt"                        "$OUT_DIR/allkeys.txt" \
         "$UCA_FALLBACK/allkeys.txt"

# CollationTest — download zip and extract
_ct_zip="$OUT_DIR/CollationTest.zip"
if [ -f "$OUT_DIR/CollationTest/CollationTest_NON_IGNORABLE.txt" ]; then
    printf "  %-45s  skip (exists)\n" "CollationTest/"
else
    printf "  %-45s  " "CollationTest.zip"
    if fetch "$_ct_zip" "$UCA_BASE/CollationTest.zip" 2>/dev/null \
       || fetch "$_ct_zip" "$UCA_FALLBACK/CollationTest.zip" 2>/dev/null; then
        _size=$(wc -c < "$_ct_zip" | tr -d ' ')
        printf "%s bytes\n" "$_size"

        # Extract .txt files into CollationTest/
        if command -v unzip >/dev/null 2>&1; then
            unzip -oqj "$_ct_zip" "*.txt" -d "$OUT_DIR/CollationTest" 2>/dev/null || true
        elif command -v python3 >/dev/null 2>&1; then
            python3 -c "
import zipfile, os, sys
zf = zipfile.ZipFile('$_ct_zip')
dest = '$OUT_DIR/CollationTest'
for info in zf.infolist():
    if info.filename.endswith('.txt'):
        info.filename = os.path.basename(info.filename)
        zf.extract(info, dest)
"
        else
            echo "  warning: cannot extract zip (no unzip or python3)" >&2
        fi
        rm -f "$_ct_zip"
        echo "  CollationTest/ extracted"
    else
        printf "FAILED\n"
        rm -f "$_ct_zip"
        FAILURES=$((FAILURES + 1))
    fi
fi

# --- IDNA ---

echo ""
echo "IDNA files:"

download "$IDNA_BASE/IdnaMappingTable.txt"              "$OUT_DIR/IdnaMappingTable.txt"

# --- Conformance test files ---

echo ""
echo "Conformance test data:"

download "$UCD_BASE/auxiliary/GraphemeBreakTest.txt"     "$OUT_DIR/GraphemeBreakTest.txt"
download "$UCD_BASE/auxiliary/WordBreakTest.txt"         "$OUT_DIR/WordBreakTest.txt"
download "$UCD_BASE/auxiliary/SentenceBreakTest.txt"     "$OUT_DIR/SentenceBreakTest.txt"
download "$UCD_BASE/auxiliary/LineBreakTest.txt"         "$OUT_DIR/LineBreakTest.txt"
download "$UCD_BASE/NormalizationTest.txt"              "$OUT_DIR/NormalizationTest.txt"
download "$UCD_BASE/BidiCharacterTest.txt"              "$OUT_DIR/BidiCharacterTest.txt"
download "$IDNA_BASE/IdnaTestV2.txt"                    "$OUT_DIR/IdnaTestV2.txt"

# --- Summary ---

echo ""

if [ "$FAILURES" -gt 0 ]; then
    echo "Done with ${FAILURES} failure(s)."
    echo ""
    echo "If versioned URLs fail, the UCA or emoji directories may not"
    echo "be published yet for ${VERSION}.  Try:"
    echo "  UCA:   https://www.unicode.org/Public/UCA/latest/"
    echo "  Emoji: https://www.unicode.org/Public/emoji/latest/"
    exit 1
else
    _count=$(find "$OUT_DIR" -name '*.txt' -type f | wc -l | tr -d ' ')
    _size=$(du -sh "$OUT_DIR" | cut -f1 | tr -d ' ')
    echo "Done. ${_count} files, ${_size} in ${OUT_DIR}"
fi