build.sh 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368
  1. #!/bin/bash
  2. # Run all the different permutations of all the tests and other things
  3. # This helps ensure that nothing gets broken.
  4. _tests() {
  5. local vet="" # TODO: make it off
  6. local gover=$( ${gocmd} version | cut -f 3 -d ' ' )
  7. [[ $( ${gocmd} version ) == *"gccgo"* ]] && zcover=0
  8. [[ $( ${gocmd} version ) == *"gollvm"* ]] && zcover=0
  9. case $gover in
  10. go1.[7-9]*|go1.1[0-9]*|go2.*|devel*) true ;;
  11. *) return 1
  12. esac
  13. # note that codecgen requires fastpath, so you cannot do "codecgen codec.notfastpath"
  14. # we test the following permutations wnich all execute different code paths as below.
  15. echo "TestCodecSuite: (fastpath/unsafe), (!fastpath/unsafe), (fastpath/!unsafe), (!fastpath/!unsafe), (codecgen/unsafe)"
  16. local echo=1
  17. local nc=2 # count
  18. local cpus="1,$(nproc)"
  19. # if using the race detector, then set nc to
  20. if [[ " ${zargs[@]} " =~ "-race" ]]; then
  21. cpus="$(nproc)"
  22. fi
  23. local a=( "" "codec.notfastpath" "codec.safe" "codec.notfastpath codec.safe" "codecgen" )
  24. local b=()
  25. local c=()
  26. for i in "${a[@]}"
  27. do
  28. local i2=${i:-default}
  29. [[ "$zwait" == "1" ]] && echo ">>>> TAGS: 'alltests $i'; RUN: 'TestCodecSuite'"
  30. [[ "$zcover" == "1" ]] && c=( -coverprofile "${i2// /-}.cov.out" )
  31. true &&
  32. ${gocmd} vet -printfuncs "errorf" "$@" &&
  33. if [[ "$echo" == 1 ]]; then set -o xtrace; fi &&
  34. ${gocmd} test ${zargs[*]} ${ztestargs[*]} -vet "$vet" -tags "alltests $i" -count $nc -cpu $cpus -run "TestCodecSuite" "${c[@]}" "$@" &
  35. if [[ "$echo" == 1 ]]; then set +o xtrace; fi
  36. b+=("${i2// /-}.cov.out")
  37. [[ "$zwait" == "1" ]] && wait
  38. # if [[ "$?" != 0 ]]; then return 1; fi
  39. done
  40. if [[ "$zextra" == "1" ]]; then
  41. [[ "$zwait" == "1" ]] && echo ">>>> TAGS: 'codec.notfastpath x'; RUN: 'Test.*X$'"
  42. [[ "$zcover" == "1" ]] && c=( -coverprofile "x.cov.out" )
  43. ${gocmd} test ${zargs[*]} ${ztestargs[*]} -vet "$vet" -tags "codec.notfastpath x" -count $nc -run 'Test.*X$' "${c[@]}" &
  44. b+=("x.cov.out")
  45. [[ "$zwait" == "1" ]] && wait
  46. fi
  47. wait
  48. # go tool cover is not supported for gccgo, gollvm, other non-standard go compilers
  49. [[ "$zcover" == "1" ]] &&
  50. command -v gocovmerge &&
  51. gocovmerge "${b[@]}" > __merge.cov.out &&
  52. ${gocmd} tool cover -html=__merge.cov.out
  53. }
  54. # is a generation needed?
  55. _ng() {
  56. local a="$1"
  57. if [[ ! -e "$a" ]]; then echo 1; return; fi
  58. for i in `ls -1 *.go.tmpl gen.go values_test.go`
  59. do
  60. if [[ "$a" -ot "$i" ]]; then echo 1; return; fi
  61. done
  62. }
  63. _prependbt() {
  64. cat > ${2} <<EOF
  65. // +build generated
  66. EOF
  67. cat ${1} >> ${2}
  68. rm -f ${1}
  69. }
  70. # _build generates fast-path.go and gen-helper.go.
  71. _build() {
  72. if ! [[ "${zforce}" || $(_ng "fast-path.generated.go") || $(_ng "gen-helper.generated.go") || $(_ng "gen.generated.go") ]]; then return 0; fi
  73. if [ "${zbak}" ]; then
  74. _zts=`date '+%m%d%Y_%H%M%S'`
  75. _gg=".generated.go"
  76. [ -e "gen-helper${_gg}" ] && mv gen-helper${_gg} gen-helper${_gg}__${_zts}.bak
  77. [ -e "fast-path${_gg}" ] && mv fast-path${_gg} fast-path${_gg}__${_zts}.bak
  78. [ -e "gen${_gg}" ] && mv gen${_gg} gen${_gg}__${_zts}.bak
  79. fi
  80. rm -f gen-helper.generated.go fast-path.generated.go gen.generated.go \
  81. *safe.generated.go *_generated_test.go *.generated_ffjson_expose.go
  82. cat > gen.generated.go <<EOF
  83. // +build codecgen.exec
  84. // Copyright (c) 2012-2020 Ugorji Nwoke. All rights reserved.
  85. // Use of this source code is governed by a MIT license found in the LICENSE file.
  86. package codec
  87. // DO NOT EDIT. THIS FILE IS AUTO-GENERATED FROM gen-dec-(map|array).go.tmpl
  88. const genDecMapTmpl = \`
  89. EOF
  90. cat >> gen.generated.go < gen-dec-map.go.tmpl
  91. cat >> gen.generated.go <<EOF
  92. \`
  93. const genDecListTmpl = \`
  94. EOF
  95. cat >> gen.generated.go < gen-dec-array.go.tmpl
  96. cat >> gen.generated.go <<EOF
  97. \`
  98. const genEncChanTmpl = \`
  99. EOF
  100. cat >> gen.generated.go < gen-enc-chan.go.tmpl
  101. cat >> gen.generated.go <<EOF
  102. \`
  103. EOF
  104. cat > gen-from-tmpl.codec.generated.go <<EOF
  105. package codec
  106. func GenRunTmpl2Go(in, out string) { genRunTmpl2Go(in, out) }
  107. func GenRunSortTmpl2Go(in, out string) { genRunSortTmpl2Go(in, out) }
  108. EOF
  109. # stub xxxRv and xxxRvSlice creation, before you create it
  110. cat > gen-from-tmpl.sort-slice-stubs.generated.go <<EOF
  111. // +build codecgen.sort_slice
  112. package codec
  113. import "reflect"
  114. import "time"
  115. EOF
  116. for i in string bool uint64 int64 float64 bytes time; do
  117. local i2=$i
  118. case $i in
  119. 'time' ) i2="time.Time";;
  120. 'bytes' ) i2="[]byte";;
  121. esac
  122. cat >> gen-from-tmpl.sort-slice-stubs.generated.go <<EOF
  123. type ${i}Rv struct { v ${i2}; r reflect.Value }
  124. type ${i}RvSlice []${i}Rv
  125. func (${i}RvSlice) Len() int { return 0 }
  126. func (${i}RvSlice) Less(i, j int) bool { return false }
  127. func (${i}RvSlice) Swap(i, j int) {}
  128. type ${i}Intf struct { v ${i2}; i interface{} }
  129. type ${i}IntfSlice []${i}Intf
  130. func (${i}IntfSlice) Len() int { return 0 }
  131. func (${i}IntfSlice) Less(i, j int) bool { return false }
  132. func (${i}IntfSlice) Swap(i, j int) {}
  133. EOF
  134. done
  135. sed -e 's+// __DO_NOT_REMOVE__NEEDED_FOR_REPLACING__IMPORT_PATH__FOR_CODEC_BENCH__+import . "github.com/ugorji/go/codec"+' \
  136. shared_test.go > bench/shared_test.go
  137. # explicitly return 0 if this passes, else return 1
  138. local btags="codec.notfastpath codec.safe codecgen.exec"
  139. rm -f sort-slice.generated.go fast-path.generated.go gen-helper.generated.go mammoth_generated_test.go mammoth2_generated_test.go
  140. cat > gen-from-tmpl.sort-slice.generated.go <<EOF
  141. // +build ignore
  142. package main
  143. import "${zpkg}"
  144. func main() {
  145. codec.GenRunSortTmpl2Go("sort-slice.go.tmpl", "sort-slice.generated.go")
  146. }
  147. EOF
  148. ${gocmd} run -tags "$btags codecgen.sort_slice" gen-from-tmpl.sort-slice.generated.go || return 1
  149. rm -f gen-from-tmpl.sort-slice.generated.go
  150. cat > gen-from-tmpl.generated.go <<EOF
  151. // +build ignore
  152. package main
  153. import "${zpkg}"
  154. func main() {
  155. codec.GenRunTmpl2Go("fast-path.go.tmpl", "fast-path.generated.go")
  156. codec.GenRunTmpl2Go("gen-helper.go.tmpl", "gen-helper.generated.go")
  157. codec.GenRunTmpl2Go("mammoth-test.go.tmpl", "mammoth_generated_test.go")
  158. codec.GenRunTmpl2Go("mammoth2-test.go.tmpl", "mammoth2_generated_test.go")
  159. }
  160. EOF
  161. ${gocmd} run -tags "$btags" gen-from-tmpl.generated.go || return 1
  162. rm -f gen-from-tmpl.generated.go
  163. rm -f gen-from-tmpl.*generated.go
  164. return 0
  165. }
  166. _codegenerators() {
  167. local c5="_generated_test.go"
  168. local c7="$PWD/codecgen"
  169. local c8="$c7/__codecgen"
  170. local c9="codecgen-scratch.go"
  171. if ! [[ $zforce || $(_ng "values_codecgen${c5}") ]]; then return 0; fi
  172. # Note: ensure you run the codecgen for this codebase/directory i.e. ./codecgen/codecgen
  173. true &&
  174. echo "codecgen ... " &&
  175. if [[ $zforce || ! -f "$c8" || "$c7/gen.go" -nt "$c8" ]]; then
  176. echo "rebuilding codecgen ... " && ( cd codecgen && ${gocmd} build -o $c8 ${zargs[*]} . )
  177. fi &&
  178. $c8 -rt 'codecgen' -t 'codecgen generated' -o "values_codecgen${c5}" -d 19780 "$zfin" "$zfin2" &&
  179. cp mammoth2_generated_test.go $c9 &&
  180. $c8 -t 'codecgen,!codec.notfastpath,!codec.notmammoth generated,!codec.notfastpath,!codec.notmammoth' -o "mammoth2_codecgen${c5}" -d 19781 "mammoth2_generated_test.go" &&
  181. rm -f $c9 &&
  182. echo "generators done!"
  183. }
  184. _prebuild() {
  185. echo "prebuild: zforce: $zforce"
  186. local d="$PWD"
  187. local zfin="test_values.generated.go"
  188. local zfin2="test_values_flex.generated.go"
  189. local zpkg="github.com/ugorji/go/codec"
  190. local returncode=1
  191. # zpkg=${d##*/src/}
  192. # zgobase=${d%%/src/*}
  193. # rm -f *_generated_test.go
  194. rm -f codecgen-*.go &&
  195. _build &&
  196. cp $d/values_test.go $d/$zfin &&
  197. cp $d/values_flex_test.go $d/$zfin2 &&
  198. _codegenerators &&
  199. if [[ "$(type -t _codegenerators_external )" = "function" ]]; then _codegenerators_external ; fi &&
  200. if [[ $zforce ]]; then ${gocmd} install ${zargs[*]} .; fi &&
  201. returncode=0 &&
  202. echo "prebuild done successfully"
  203. rm -f $d/$zfin $d/$zfin2
  204. return $returncode
  205. # unset zfin zfin2 zpkg
  206. }
  207. _make() {
  208. local makeforce=${zforce}
  209. zforce=1
  210. (cd codecgen && ${gocmd} install ${zargs[*]} .) && _prebuild && ${gocmd} install ${zargs[*]} .
  211. zforce=${makeforce}
  212. }
  213. _clean() {
  214. rm -f \
  215. gen-from-tmpl.*generated.go \
  216. codecgen-*.go \
  217. test_values.generated.go test_values_flex.generated.go
  218. }
  219. _release() {
  220. local reply
  221. read -p "Pre-release validation takes a few minutes and MUST be run from within GOPATH/src. Confirm y/n? " -n 1 -r reply
  222. echo
  223. if [[ ! $reply =~ ^[Yy]$ ]]; then return 1; fi
  224. # expects GOROOT, GOROOT_BOOTSTRAP to have been set.
  225. if [[ -z "${GOROOT// }" || -z "${GOROOT_BOOTSTRAP// }" ]]; then return 1; fi
  226. # (cd $GOROOT && git checkout -f master && git pull && git reset --hard)
  227. (cd $GOROOT && git pull)
  228. local f=`pwd`/make.release.out
  229. cat > $f <<EOF
  230. ========== `date` ===========
  231. EOF
  232. # # go 1.6 and below kept giving memory errors on Mac OS X during SDK build or go run execution,
  233. # # that is fine, as we only explicitly test the last 3 releases and tip (2 years).
  234. local makeforce=${zforce}
  235. zforce=1
  236. for i in 1.10 1.11 1.12 master
  237. do
  238. echo "*********** $i ***********" >>$f
  239. if [[ "$i" != "master" ]]; then i="release-branch.go$i"; fi
  240. (false ||
  241. (echo "===== BUILDING GO SDK for branch: $i ... =====" &&
  242. cd $GOROOT &&
  243. git checkout -f $i && git reset --hard && git clean -f . &&
  244. cd src && ./make.bash >>$f 2>&1 && sleep 1 ) ) &&
  245. echo "===== GO SDK BUILD DONE =====" &&
  246. _prebuild &&
  247. echo "===== PREBUILD DONE with exit: $? =====" &&
  248. _tests "$@"
  249. if [[ "$?" != 0 ]]; then return 1; fi
  250. done
  251. zforce=${makeforce}
  252. echo "++++++++ RELEASE TEST SUITES ALL PASSED ++++++++"
  253. }
  254. _usage() {
  255. # hidden args:
  256. # -pf [p=prebuild (f=force)]
  257. cat <<EOF
  258. primary usage: $0
  259. -[tesow m n l d] -> [t=tests (e=extra, s=short, o=cover, w=wait), m=make, n=inlining diagnostics, l=mid-stack inlining, d=race detector]
  260. -v -> v=verbose
  261. EOF
  262. if [[ "$(type -t _usage_run)" = "function" ]]; then _usage_run ; fi
  263. }
  264. _main() {
  265. if [[ -z "$1" ]]; then _usage; return 1; fi
  266. local x # determines the main action to run in this build
  267. local zforce # force
  268. local zcover # generate cover profile and show in browser when done
  269. local zwait # run tests in sequence, not parallel ie wait for one to finish before starting another
  270. local zextra # means run extra (python based tests, etc) during testing
  271. local ztestargs=()
  272. local zargs=()
  273. local zverbose=()
  274. local zbenchflags=""
  275. local gocmd=${MYGOCMD:-go}
  276. OPTIND=1
  277. while getopts ":cetmnrgpfvldsowkxyzb:" flag
  278. do
  279. case "x$flag" in
  280. 'xo') zcover=1 ;;
  281. 'xe') zextra=1 ;;
  282. 'xw') zwait=1 ;;
  283. 'xf') zforce=1 ;;
  284. 'xs') ztestargs+=("-short") ;;
  285. 'xv') zverbose+=(1) ;;
  286. 'xl') zargs+=("-gcflags"); zargs+=("-l=4") ;;
  287. 'xn') zargs+=("-gcflags"); zargs+=("-m=2") ;;
  288. 'xd') zargs+=("-race") ;;
  289. 'xb') x='b'; zbenchflags=${OPTARG} ;;
  290. x\?) _usage; return 1 ;;
  291. *) x=$flag ;;
  292. esac
  293. done
  294. shift $((OPTIND-1))
  295. # echo ">>>> _main: extra args: $@"
  296. case "x$x" in
  297. 'xt') _tests "$@" ;;
  298. 'xm') _make "$@" ;;
  299. 'xr') _release "$@" ;;
  300. 'xg') _go ;;
  301. 'xp') _prebuild "$@" ;;
  302. 'xc') _clean "$@" ;;
  303. 'xx') _analyze_checks "$@" ;;
  304. 'xy') _analyze_debug_types "$@" ;;
  305. 'xz') _analyze_do_inlining_and_more "$@" ;;
  306. 'xk') _go_compiler_validation_suite ;;
  307. 'xb') _bench "$@" ;;
  308. esac
  309. # unset zforce zargs zbenchflags
  310. }
  311. [ "." = `dirname $0` ] && _main "$@"