xml - curl 从 Smartling 下载所有文件

Question

我想编写一个简单的解决方案来备份我们目前在 smarting 上拥有的所有字符串文件。这包括任何语言环境文件。

目前我还有许多其他问题，但主要目标是下载所有文件。

单个文件的当前 curl 是:

curl -d "apiKey=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx
&fileUri=%2Fpath%2Fto%2Ffolder%2Fstrings.xml
&projectId=1234567890" "https://api.smartling.com/v1/file/get/"

这将为我提供一个 strings.xml 文件。我可以使用 &locale=DE 进一步定义语言环境，以获得包含翻译的相同文件。

我可以使用通配符下载所有 *.xml 文件吗？

Answer 1

Smartling 有一个与 Smrtling API 交互的 shell 脚本，可以执行此类操作(以及更多)。假设您想下载所有已翻译的文件以及已发布的翻译，该命令类似于:

./download-smartling-files.sh -t published -a xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx -p 1234567890 -l "nl-NL ru-RU"

请务必使用您为 Smartling 项目配置的语言环境更新 locales -l 参数。

如果您想下载所有文件，包括任何和所有待处理的翻译以及已发布的翻译，那么该命令将类似于

./download-smartling-files.sh -t pending -a xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx -p 1234567890 -l "nl-NL ru-RU"

这两个命令都将与 Smartling API 交互以获取当前上传到 Smartling 的所有文件的列表，然后遍历它们，将翻译后的文件下载到 translated/[locale] 目录

您可以通过脚本执行许多其他操作，包括下载所有翻译的 TMX 文件。要了解如何执行此操作以及其他操作，只需运行

./download-smartling-files.sh

另一种选择是使用 Smartling Maven Plugin 。这通常在将本地化整合到您的构建过程中并且您希望在构建和部署事件期间上传和/或下载文件时使用。

这里是 download-smartling-files.sh 脚本

#!/bin/bash
# author: Eric Negron (enegron@smartling.com)
# last updated February 20, 2014

shopt -s nullglob


function delete_files () {

    # print the files first to make damn sure they know what they are deleting
    COUNT=1
    for CURRENT_FILE in ${uris[@]}
    do
        echo $'\n'$COUNT" "$CURRENT_FILE
        ((COUNT++))
        done

    echo "Are you SURE you want to delete all "${#uris[@]}" files?"

    # confirm by getting the user to enter 'YES' to input, and if YES then delete these files
    COUNT=1
    read delete_confirmation
    if [ $delete_confirmation != "YES" ]; then
        echo "sorry, must answer YES to delete. exiting"
    exit

    else
        for CURRENT_FILE in ${uris[@]}
        do
            echo $'\n'$COUNT" "$CURRENT_FILE
            curl -sS -d "apiKey=$SL_APIKEY&projectId=$SL_PROJECT&fileUri=$CURRENT_FILE" "https://$SERVER_URL/v1/file/delete"
            ((COUNT++))
        done
    fi
}

function usage {
cat << EOF
    This script uses the Smartling API
    OPTIONS:
    -h      Show this message
    -t (REQUIRED) download type. Acceptable types: original | published | p100 | pseudo | pending | TMX | DELETE
    -a (REQUIRED) API key
    -p (REQUIRED) project ID
    -u (OPTIONAL) URI mask
    -l (OPTIONAL) locales list - a qouted array of Smartling Locales. E.G. "es-ES ja-JP"
    -x (OPTIONAL) TMX download type: full (default) | published - only valid with -t TMX
    -d (OPTIONAL) when downloading translations or TMX set this option to false to append filename with locale instead of creating locale folders. Not valid with -t original option.
    -E (OPTIONAL) DELETE files -E must be true and type = DELETE and must use -u MASK option
EOF
}

# hardcode to regular API, not sandbox
SERVER_URL=api.smartling.com

#opt t
DL_TYPE=

#opt a
SL_APIKEY=

#opt p
SL_PROJECT=

#opt u
URI_MASK=

#opt u
LOCALES=

#opt x
TMX_FLAG="full"

#opt d
USE_LOCALE_DIR="true"

#opt E
DELETE_CONFIRM="false"

while getopts "ht:a:p:u:l:x:d:E:" OPTION
do
    case $OPTION in
        h)
            usage
            exit 1
            ;;
        t)
            DL_TYPE=$OPTARG
            ;;
        a)
            SL_APIKEY=$OPTARG
            ;;
        p)
            SL_PROJECT=$OPTARG
            ;;
        u)
            URI_MASK=$OPTARG
            ;;
        l)
            LOCALES=($OPTARG)
            ;;
        x)
            TMX_FLAG=$OPTARG
            ;;
        d)
            USE_LOCALE_DIR=$OPTARG
            ;;
        E)
            DELETE_CONFIRM=$OPTARG
            ;;

    esac
done

# make sure the required paramaters are set to something
# TODO check if getops handles this natively in some way
if [ "$DL_TYPE" == "" ] || [ "$SL_APIKEY" == "" ] || [ "$SL_PROJECT" == "" ]; then
    usage
    exit
fi

# make sure download type is valid
if [ "$DL_TYPE" != "original" ]\
 && [ "$DL_TYPE" != "published" ]\
 && [ "$DL_TYPE" != "p100" ]\
 && [ "$DL_TYPE" != "pseudo" ]\
 && [ "$DL_TYPE" != "pending" ]\
 && [ "$DL_TYPE" != "TMX" ]\
 && [ "$DL_TYPE" != "DELETE" ]; then
echo "INVALID TYPE. Acceptable types: original | published | p100 | pseudo | pending | TMX.   Exiting."
exit
fi

# if user has specified mask confirm it - note mask means nothing to TMX
if [ "$URI_MASK" != "" ]; then
    if [ $DL_TYPE == "TMX" ]; then
        echo "Mask option has no effect when downloading TMX. Ignoring."
        URI_MASK=""
    else
        echo "mask was set!"$'\n'
    fi
fi

IFS=$'\n'

#make sure there are no API configuration errors by doing a file/list call just to check for error conditions
errors=($(curl -sS -d "apiKey=$SL_APIKEY&projectId=$SL_PROJECT&uriMask=$URI_MASK" "https://$SERVER_URL/v1/file/list" |  grep -Eo "ERROR"))
error_count=${#errors[@]}$'\n'
if [ $error_count -gt 0 ]; then
    echo "error with either the API key or Project ID - verify your values."
    exit
fi

# figure out how many total files there - because if more than standard 500 returned in list, then need to paginate to build the list of uris
# need to move this check since if downloading TMX it's possible there are no files.
total_files=($(curl -sS -d "apiKey=$SL_APIKEY&projectId=$SL_PROJECT&uriMask=$URI_MASK" "https://$SERVER_URL/v1/file/list" |  grep -Eo "fileCount\":[0-9]+" | sed -e 's/fileCount\"://g'))


# The /file/list API has a limit of 500 items, if the total needed is more, need to make multiple calls to build the full URI list

listLimit=500

# figure out how many passes in batches of 500 needed

let "passes=$total_files/$listLimit"

# if more than 1 batch needed, then make as many passes as needed to build the list

pass=0

while [ $pass -le $passes ]; do

let "passOffset=$pass*$listLimit"

    uris+=($(curl -sS -d "apiKey=$SL_APIKEY&projectId=$SL_PROJECT&uriMask=$URI_MASK&offset=$passOffset" "https://$SERVER_URL/v1/file/list" |  grep -Eo "fileUri\":\"[^\"]*" | sed -e 's/fileUri":"//g'))
    ((pass++))

done

# exit if requesting to download files, but nothing matched at all, otherwise show the count of files that matched and the names
# TODO: refactor since TMX handling is different with getopts

file_count=${#uris[@]}
if [ $file_count = "0" ] && [ $DL_TYPE != "TMX" ]; then echo "nothing to operate on, exiting!"; exit; fi

# if deleting check all the parameters set and if so pass of URI mask list to delete function
if [ "$DL_TYPE" == "DELETE" ]; then
    if [ "$DELETE_CONFIRM" == "true" ] && [ "$URI_MASK" != "" ]; then
        delete_files $uris
        exit
    else
        echo "Must set -u to a uri mask value and must set -E true to confirm deletion. Exiting"
        exit
    fi
fi

# if not downloading TMX - then list the files we found
echo "total files to download "$total_files$'\n'
if [ $DL_TYPE != "TMX" ]; then echo "URIs: "${uris[@]}$'\n'; fi


#BEGIN ORIGINALS
if [ "$DL_TYPE" == "original" ] ; then
    echo "downloading originals"
    echo "files to download: "$file_count$'\n'

# create the originals folder if it doesn't exist
    if [ ! -d "originals" ]; then
        echo "making originals folder"$'\n'
        mkdir originals
    fi

    COUNT=0
    # go through the list of files to download (in each language)
    for CURRENT_FILE in ${uris[@]}
    do
        echo $CURRENT_FILE
        ((COUNT++))
        echo $COUNT

        # since the URI includes full URI including default smartling prefix '/files/' or any other prefix specified - strip that and just use the last part after the last /
        # e.g. if the URI is /files/filename.ext then BF_NAME will be filename.ext - this what we use for the local file name (in the folder)
        BF_NAME=${CURRENT_FILE##*/}
        # This is curl call that downloads the originals. Since no LOCALE is set that is the behavior of /file/get

        # becuase we are flattening the URIs to basename it's possible filenames could be duplicated - so to avoid this check if the file already exist and if it does use count to append name to create unique
        # TODO USE COUNT
        if [ -e originals/$BF_NAME ]; then
            BF_NAME=$COUNT.$BF_NAME
            echo "Updated BF_NAME"
        fi

        curl -sS -d "apiKey=$SL_APIKEY&projectId=$SL_PROJECT&fileUri=$CURRENT_FILE" "https://$SERVER_URL/v1/file/get" > originals/$BF_NAME
        echo "downloaded URI: "$CURRENT_FILE" to: originals/"$BF_NAME$'\n'

    done
exit
fi
#end of originals


# if not getting originals - then getting translations OR TMX - so get locales and then use that to download all
# get the list of locales for this project.  Similar to above grep but the locale key value then sed for just the value

# if user has not set the locales array in the call, then get the full list via API
# TODO - validate the locales are good - otherwise user will just get empty files
if [ "$LOCALES" = "" ]; then
    LOCALES=($(curl -sS -d "apiKey=$SL_APIKEY&projectId=$SL_PROJECT" "https://$SERVER_URL/v1/project/locale/list" |  grep -Eo "locale\":\"[^\"]*" | sed -e 's/locale":"//g'))
fi

echo "locales: "${#LOCALES[@]}$'\n'
# count of the locales
echo ${LOCALES[@]}$'\n'

#script puts the translated' versions in 'translated folder' with sub-folders for locales
if [ ! -d "translated" ]; then
    echo "making translated folder"$'\n'
    mkdir translated
fi

if [ $USE_LOCALE_DIR == "true" ]; then
#if [ "$USE_LOCALE_DIR" == "true" ]; then
    # make the subfolders in the translated folder if they don't exist
    for CURRENT_LOCALE in ${LOCALES[@]}
        do
        if [ ! -d translated/$CURRENT_LOCALE ]; then
            echo "making "$CURRENT_LOCALE" folder" $'\n'
            mkdir translated/$CURRENT_LOCALE
        fi
    done
fi

#TODO need to refactor TMX / MASK / PUBLISHED / FULL
# Download TMX
# Downloading TMX uses a different API than downloading tranlated files
if [ "$DL_TYPE" == "TMX" ] ; then
    echo "downloading TMX "$TMX_FLAG
    for CURRENT_LOCALE in ${LOCALES[@]}
    do
        # if using folders - set the folder locale
        # else using file-names so set the filename locale
        if [ "$USE_LOCALE_DIR" == "true" ]; then
            LOCALE_DIR=$CURRENT_LOCALE"/"
            FILE_LOCALE="TMX-"
        else
            FILE_LOCALE=$CURRENT_LOCALE"-"
            LOCALE_DIR=""
        fi

        curl -sS -d "apiKey=$SL_APIKEY&projectId=$SL_PROJECT&locale=$CURRENT_LOCALE&format=TMX&dataSet=$TMX_FLAG" "https://$SERVER_URL/v1/translations/download" > translated/$LOCALE_DIR$FILE_LOCALE"TMX.xml"
        echo "downloaded " $CURRENT_LOCALE "TMX" $'\n'
    done

    exit
fi


# downloading translations of files
# by this point the file list has already been filtered and the and the locale list has been created
# go through the list of files to download (in each language)

COUNT=0
P100=0

# since I use DL_TYPE as actual parameter to API call need to set this back to actual "published" before entering loop to download
if [ $DL_TYPE == "p100" ]; then
    DL_TYPE="published"
    P100=1
fi


for CURRENT_FILE in ${uris[@]}
do
    echo $CURRENT_FILE
    ((COUNT++))
    echo $COUNT
    # loop through all the locales for each file

    for CURRENT_LOCALE in ${LOCALES[@]}
    do

    # if using folders - set the folder locale
    # else using file-names so set the filename locale
    if [ $USE_LOCALE_DIR == "true" ]; then
        LOCALE_DIR=$CURRENT_LOCALE"/"
        FILE_LOCALE=""
    else
        FILE_LOCALE=$CURRENT_LOCALE"-"
        LOCALE_DIR=""
    fi

        # since the URI includes full URI including default smartling prefix '/files/' or any other prefix specified - strip that and just use the last part after the last /
        # e.g. if the URI is /files/filename.ext then BF_NAME will be filename.ext - this what we use for the local file name (in the locale folder)
        BF_NAME=${CURRENT_FILE##*/}

        EXT=${BF_NAME##*.}
        # rename .pot files to .po - we really should be checking the header from the /file/get API call but this is just simpler
        if [ "$EXT" == "pot" ]; then
            filename=${BF_NAME%.*}
            BF_NAME=$filename".po"
        echo "renamed pot to po"
        fi

        # because we are flattening the URIs to basename it's possible filenames could be duplicated - so to avoid this check if the file already exist and if it does use COUNT to append name to create unique
        if [ -e translated/$LOCALE_DIR$FILE_LOCALE$BF_NAME ]; then
            BF_NAME=$COUNT.$BF_NAME
            echo "Updated BF_NAME"
        fi

# Here if user is doing 100% published need to test and if it's not then skip it
# probably should do this earlier in this loops to avoid other tests but thare not needed if we are just going to skip the file

if [ $P100 == 1 ]; then

#    echo "requested 100 percent only"

    # use api status call and pull out the part of the response that has the string count and completed string count
    completeStatus=($(curl -sS -d "apiKey=$SL_APIKEY&projectId=$SL_PROJECT&fileUri=$CURRENT_FILE&locale=$CURRENT_LOCALE" "https://$SERVER_URL/v1/file/status" | grep -Eo "stringCount\":[0-9]+.*\"completedStringCount\":[0-9]+" ))

    # strip those out as separate variables to test
    stringCount=($(echo $completeStatus | sed -E 's/stringCount\":([0-9]+).*/\1/g'))
    completedStringCount=($(echo $completeStatus | sed -E 's/.*completedStringCount\":([0-9]+).*/\1/g'))
#echo $stringCount","$completedStringCount

    # set download to 1 only if 100%
    if [ $stringCount == $completedStringCount ]; then
        download=1
        echo $CURRENT_FILE" in "$CURRENT_LOCALE" is 100% complete!"$'\n'

    else
        download=0
        echo "100% complete requested. Skipping URI: "$CURRENT_FILE" in "$CURRENT_LOCALE" total:"$stringCount", completed:"$completedStringCount$'\n'
    fi

else
    # didn't ask for 100% so always download
    download=1

fi


if [ $download == 1 ]; then
        # This is the actual curl command that downloads the given file for given locale, and state as specified when called
        curl -sS -d "apiKey=$SL_APIKEY&projectId=$SL_PROJECT&fileUri=$CURRENT_FILE&locale=$CURRENT_LOCALE&retrievalType=$DL_TYPE" "https://$SERVER_URL/v1/file/get" > translated/$LOCALE_DIR$FILE_LOCALE$BF_NAME
        echo "downloaded URI: "$CURRENT_FILE" to: translated/"$LOCALE_DIR$FILE_LOCALE$BF_NAME$'\n'
fi


    done

done

exit