openbilibili/.rider/unit_test.sh

#!/bin/bash

declare -a dirs=(dao)
declare -a packages
declare -a projects
declare mergeUser
declare commitUser
declare mergeID

#init env
function Init(){
    if [ ! -d "${CI_PROJECT_DIR}/../src" ];then
        mkdir ${CI_PROJECT_DIR}/../src
    fi
    ln -fs ${CI_PROJECT_DIR} ${CI_PROJECT_DIR}/../src
    export GOPATH=${CI_PROJECT_DIR}/..
}

function GetPackages(){
    reg="library/(.*)/(.*).go"
    for dir in ${dirs[@]}
    do
        reg+="|app/(service|interface|admin|job)/main/(.*)/${dir}/(.*).go"
    done
    files=`.rider/changefiles.sh ${suffix} | grep  -E "${reg}"`
    if [[ "${files}" = "" ]]; then
        echo "shell.GetPackages: no change files"
        exit 0
    fi
    for file in ${files}
    do
        # if [[ "${file}" =~ "library"* || "${file}" =~ "/mock" ]]; then
        if [[ "${file}" =~ "/mock" ]]; then
            continue
        fi
        package="go-common/$(dirname ${file})"
        if [[ ${packages} =~ ${package} ]]; then
            continue
        fi
        packages+=${package}" "
        packageSplit=(${package//\// })
        project=$(printf "%s/" ${packageSplit[@]:0:6})
        if [[ ${projects} =~ ${project} || ${project} =~ "/library" ]]; then
            continue
        fi
        projects+=${project%*/}" "
    done
    if [[ ${packages} = "" || ${projects} = "" ]]; then
        echo "shell.GetPackages no change packages"
        exit 0
    fi
}
# GetUserInfo get userinfo by gitlab result.
function GetUserInfo(){
    gitMergeRequestUrl="${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/commits/${CI_COMMIT_SHA}/merge_requests?private_token=${CI_PRIVATE_TOKEN}"
    gitCommitUrl="${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/commits/${CI_COMMIT_SHA}/statuses?private_token=${CI_PRIVATE_TOKEN}"
    mergeJson=$(curl -s ${gitMergeRequestUrl})
    commitJson=$(curl -s ${gitCommitUrl})
    if [[ "${mergeJson}" = "[]" ]] || [[ "${commitJson}" = "[]" ]]; then
        echo "Test not run, maybe you should try create a merge request first!"
        exit 0       
    fi
    mergeID=$(echo ${mergeJson} | jq -r '.[0].iid')
    mergeUser=$(echo ${mergeJson} | jq -r '.[0].author.username')
    commitUser=$(echo ${commitJson} | jq -r '.[0].author.username')
}

# Magic ignore method Check()
function Magic(){
    url="http://git.bilibili.co/api/v4/projects/${CI_PROJECT_ID}/merge_requests/${mergeID}/notes?private_token=${CI_PRIVATE_TOKEN}"
    json=$(curl -s ${url})
    admin="haoguanwei,chenjianrong,hedan,fengshanshan,zhaobingqing"
    len=$(echo ${json} | jq '.|length')
    for i in $(seq 0 $len)
    do
        comment=$(echo ${json} | jq -r ".[$i].body")
        user=$(echo ${json} | jq -r ".[$i].author.username")
        if [[ ${comment} = "+skiput" && ${admin} =~ ${user} ]]; then
             exit 0
        fi
    done
}

# Check determine whether the standard is up to standard
#$1: commit_id
function Check(){
    curl -s "${CI_UATSVEN_URL}/x/admin/apm/ut/git/report?project_id=${CI_PROJECT_ID}&merge_id=${mergeID}&commit_id=${CI_COMMIT_SHA}"
    checkURL="${CI_UATSVEN_URL}/x/admin/apm/ut/check?commit_id=${CI_COMMIT_SHA}"
    json=$(curl -s ${checkURL})
    code=$(echo ${json} | jq -r '.code')
    if [[ ${code} -ne 0 ]]; then
        echo -e "curl ${checkURL} response(${json})"
        exit 1
    fi
    package=$(echo ${json} | jq -r '.data.package')
    coverage=$(echo ${json} | jq -r '.data.coverage')
    passRate=$(echo ${json} | jq -r '.data.pass_rate')
    standard=$(echo ${json} | jq -r '.data.standard')
    increase=$(echo ${json} | jq -r '.data.increase')
    tyrant=$(echo ${json} | jq -r '.data.tyrant')
    lastCID=$(echo ${json} | jq -r '.data.last_cid')
    if ${tyrant}; then
        echo -e "\t续命失败!\n\t大佬，本次执行结果未达标哦(灬ꈍ ꈍ灬)，请再次优化ut重新提交🆙"
        echo -e "\t---------------------------------------------------------------------"
        printf "\t%-14s %-14s %-14s %-14s\n" "本次覆盖率(%)" "本次通过率(%)" "本次增长量(%)" 执行pkg
        printf "\t%-13.2f %-13.2f %-13.2f %-12s\n" ${coverage} ${passRate} ${increase} ${package}
        echo -e "\t(达标标准：覆盖率>=${standard} && 通过率=100% && 同比当前package历史最高覆盖率的增长率>=0)"
        echo -e "\t---------------------------------------------------------------------"
        echo -e "本次执行详细结果查询地址请访问：http://sven.bilibili.co/#/ut?merge_id=${mergeID}&&pn=1&ps=20"
        exit 1
    else
        echo -e "\t恭喜你，续命成功，可以请求合并MR了!"
    fi
}

function ReadDir(){
    # get go-common/app all dir path
    gopath=${GOPATH%..}
    PathDirs=`find ${gopath}app -maxdepth 3 -type d`
    value=""
    for dir in ${PathDirs}
    do
        if [[ -d "$dir" ]];then
            for file in `find ${dir} -maxdepth 1 -type f |grep "OWNERS"`
            do
                owner=""
                substr=${dir#*"go-common"}
                while read line
                do
                    if [[ "${line}" = "#"* ]] || [[ "${line}" = "" ]] || [[ "${line}" = "approvers:" ]];then
                        continue
                    elif [[ "${line}" = "labels:"* ]];then
                        break
                    else
                        owner+="${line:1},"
                    fi
                done < ${file}
                value+="{\"path\":\"go-common${substr}\",\"owner\":\"${owner%,}\"},"
            done
        fi
    done
    # delete "," at the end of value
    value=${value%,}
    echo "[${value}]" > path.out
}

# UTLint check the *_test.go files in the pkg
# $1: pkg
function UTLint()
{   
    cd $GOPATH
    declare -i numCase=0
    declare -i numAssertion=0
    files=$(ls $1 | grep -E "(.*)_test\.go")
    if [[ ${#files} -eq 0 ]];then
        echo "RunPKGUT.UTLint no *_test.go files in pkg($1)"
        exit 1
    fi
    for file in ${files}
    do
        numCase+=`grep -c -E "^func Test(.+)\(t \*testing\.T\) \{$" $1/${file}`
        numAssertion+=`grep -c -E "^(.*)So\((.+)\)$" $1/${file}`
    done
    if [[ ${numCase} -eq 0 || ${numAssertion} -eq 0 ]];then
        echo -e "RunPKGUT.UTLint no test case or assertion in pkg($1)"
        exit 1
    fi
}

# BazelTest execute bazel coverage and go tool
# $1: pkg
function BazelTest(){
    cd $GOPATH/go-common
    pkg=${1//go-common//}":go_default_test"
    path=${1//go-common\//}

    bazel coverage --config=ci --instrumentation_filter="//${path}[:],-//${path}/mock[/:]" --test_env=DEPLOY_ENV=uat --test_timeout=60 --test_env=APP_ID=bazel.test --test_output=all --cache_test_results=auto --test_arg=-convey-json ${pkg} > result.out
    if [[ ! -s result.out ]]; then 
        echo "==================================WARNING!======================================"
        echo "No test case found,请完善如下路径测试用例： ${pkg} "
        exit 1
    else
        echo $?
        cp $GOPATH/go-common/bazel-out/k8-fastbuild/testlogs/${path}/go_default_test/coverage.dat ./
        go tool cover -html=coverage.dat -o cover.html
    fi
}

# BazelTest execute bazel coverage for All files
# $1: pkg(go-common/app/admin/main/xxx/dao)
function BazelTestAll(){
    cd $GOPATH/go-common
    pkg=${1//go-common//}"/..."
    path=${1//go-common\//}
    echo "RunProjUT.BazelTestAll(${1}) pkg(${pkg}) path(${path}) pwd($(pwd))"
    bazel coverage --config=ci --instrumentation_filter="//${path}[/:],-//${path}/mock[/:]" --test_env=DEPLOY_ENV=uat --test_timeout=60 --test_env=APP_ID=bazel.test --test_output=all --cache_test_results=auto --test_arg=-convey-json ${pkg} > result.out
    find bazel-out/k8-fastbuild/testlogs/${path} -name "coverage.dat" | xargs cat | sort -nr | rev | uniq -s 1 | rev > coverage.dat
    coverage=$(cat coverage.dat | awk '{sum += $2;covSum += $2 * $3;} END {print covSum/sum*100}')
    sed -if "s/coverage: .*%/coverage: ${coverage}%/g" result.out
    go tool cover -html=coverage.dat -o cover.html
}

# upload data to apm
# $1: file result.out path
function Upload () {
    if [[ ! -f "result.out" ]] || [[ ! -f "cover.html" ]] || [[ ! -f "coverage.dat" ]]; then
        echo "==================================WARNING!======================================"
        echo "No test found!~ 请完善如下路径测试用例： ${1} "
        exit 1
    fi
    url="${CI_UATSVEN_URL}/x/admin/apm/ut/upload?merge_id=${mergeID}&username=${mergeUser}&author=${commitUser}&commit_id=${CI_COMMIT_SHA}&pkg=${1}"
    json=$(curl -s ${url} -H "Content-type: multipart/form-data" -F "html_file=@cover.html" -F "report_file=@result.out" -F "data_file=@coverage.dat")
    if [[ "${json}" = "" ]]; then
        echo "RunPKGUT.Upload curl ${url} fail"
        exit 1
    fi
    msg=$(echo ${json} | jq -r '.message')
    data=$(echo ${json} | jq -r '.data')
    code=$(echo ${json} | jq -r '.code')
    if [[ ${code} -ne 0 ]]; then
        echo "=============================================================================="
        echo -e "RunPKGUT.Upload Response. message(${msg})"
        echo -e "RunPKGUT.Upload Response. data(${data})\n\n"
        echo -e "RunPKGUT.Upload Upload Fail! status(${code})"
        exit ${code}
    fi
}

function RunPKGUT(){
    for package in ${packages}
    do
        echo "RunPKGUT.UTLint Start. pkg(${package})"
        UTLint ${package}
        echo "RunPKGUT.BazelTest Start. pkg(${package})"
        BazelTest ${package}
        echo "RunPKGUT.Upload Start. pkg(${package})"
        Upload ${package}
    done
    return 0
}

function RunProjUT(){
    for project in ${projects}
    do
        echo "RunProjUT.BazelTestAll Start. project(${project})"
        BazelTestAll ${project}
        echo "RunProjUT.Upload BazelTest Start. project(${project})"
        Upload ${project%/*}
    done
}

function UploadApp(){
    ReadDir
    url="${CI_UATSVEN_URL}/x/admin/apm/ut/upload/app" 
    curl -s ${url} -H "Content-type: multipart/form-data" -F "path_file=@path.out" > /dev/null
    echo "UploadApp() UpPath has finshed."
}

# run
Init
GetPackages
GetUserInfo
Magic
RunPKGUT
RunProjUT
UploadApp
Check