ci.sh 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279
  1. #!/usr/bin/env bash
  2. stderr() {
  3. echo "$@" 1>&2
  4. }
  5. usage() {
  6. b=$(basename "$0")
  7. echo $b: ERROR: "$@" 1>&2
  8. cat 1>&2 <<EOF
  9. DESCRIPTION
  10. $(basename "$0") is the script to run continuous integration commands for
  11. go-toml on unix.
  12. Requires Go and Git to be available in the PATH. Expects to be ran from the
  13. root of go-toml's Git repository.
  14. USAGE
  15. $b COMMAND [OPTIONS...]
  16. COMMANDS
  17. benchmark [OPTIONS...] [BRANCH]
  18. Run benchmarks.
  19. ARGUMENTS
  20. BRANCH Optional. Defines which Git branch to use when running
  21. benchmarks.
  22. OPTIONS
  23. -d Compare benchmarks of HEAD with BRANCH using benchstats. In
  24. this form the BRANCH argument is required.
  25. -a Compare benchmarks of HEAD against go-toml v1 and
  26. BurntSushi/toml.
  27. -html When used with -a, emits the output as HTML, ready to be
  28. embedded in the README.
  29. coverage [OPTIONS...] [BRANCH]
  30. Generates code coverage.
  31. ARGUMENTS
  32. BRANCH Optional. Defines which Git branch to use when reporting
  33. coverage. Defaults to HEAD.
  34. OPTIONS
  35. -d Compare coverage of HEAD with the one of BRANCH. In this form,
  36. the BRANCH argument is required. Exit code is non-zero when
  37. coverage percentage decreased.
  38. EOF
  39. exit 1
  40. }
  41. cover() {
  42. branch="${1}"
  43. dir="$(mktemp -d)"
  44. stderr "Executing coverage for ${branch} at ${dir}"
  45. if [ "${branch}" = "HEAD" ]; then
  46. cp -r . "${dir}/"
  47. else
  48. git worktree add "$dir" "$branch"
  49. fi
  50. pushd "$dir"
  51. go test -covermode=atomic -coverpkg=./... -coverprofile=coverage.out.tmp ./...
  52. cat coverage.out.tmp | grep -v testsuite | grep -v tomltestgen | grep -v gotoml-test-decoder > coverage.out
  53. go tool cover -func=coverage.out
  54. popd
  55. if [ "${branch}" != "HEAD" ]; then
  56. git worktree remove --force "$dir"
  57. fi
  58. }
  59. coverage() {
  60. case "$1" in
  61. -d)
  62. shift
  63. target="${1?Need to provide a target branch argument}"
  64. output_dir="$(mktemp -d)"
  65. target_out="${output_dir}/target.txt"
  66. head_out="${output_dir}/head.txt"
  67. cover "${target}" > "${target_out}"
  68. cover "HEAD" > "${head_out}"
  69. cat "${target_out}"
  70. cat "${head_out}"
  71. echo ""
  72. target_pct="$(tail -n2 ${target_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%.*/\1/')"
  73. head_pct="$(tail -n2 ${head_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%/\1/')"
  74. echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%"
  75. delta_pct=$(echo "$head_pct - $target_pct" | bc -l)
  76. echo "Delta: ${delta_pct}"
  77. if [[ $delta_pct = \-* ]]; then
  78. echo "Regression!";
  79. target_diff="${output_dir}/target.diff.txt"
  80. head_diff="${output_dir}/head.diff.txt"
  81. cat "${target_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${target_diff}"
  82. cat "${head_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${head_diff}"
  83. diff --side-by-side --suppress-common-lines "${target_diff}" "${head_diff}"
  84. return 1
  85. fi
  86. return 0
  87. ;;
  88. esac
  89. cover "${1-HEAD}"
  90. }
  91. bench() {
  92. branch="${1}"
  93. out="${2}"
  94. replace="${3}"
  95. dir="$(mktemp -d)"
  96. stderr "Executing benchmark for ${branch} at ${dir}"
  97. if [ "${branch}" = "HEAD" ]; then
  98. cp -r . "${dir}/"
  99. else
  100. git worktree add "$dir" "$branch"
  101. fi
  102. pushd "$dir"
  103. if [ "${replace}" != "" ]; then
  104. find ./benchmark/ -iname '*.go' -exec sed -i -E "s|github.com/pelletier/go-toml/v2|${replace}|g" {} \;
  105. go get "${replace}"
  106. fi
  107. export GOMAXPROCS=2
  108. nice -n -19 taskset --cpu-list 0,1 go test '-bench=^Benchmark(Un)?[mM]arshal' -count=5 -run=Nothing ./... | tee "${out}"
  109. popd
  110. if [ "${branch}" != "HEAD" ]; then
  111. git worktree remove --force "$dir"
  112. fi
  113. }
  114. fmktemp() {
  115. if mktemp --version|grep GNU >/dev/null; then
  116. mktemp --suffix=-$1;
  117. else
  118. mktemp -t $1;
  119. fi
  120. }
  121. benchstathtml() {
  122. python3 - $1 <<'EOF'
  123. import sys
  124. lines = []
  125. stop = False
  126. with open(sys.argv[1]) as f:
  127. for line in f.readlines():
  128. line = line.strip()
  129. if line == "":
  130. stop = True
  131. if not stop:
  132. lines.append(line.split(','))
  133. results = []
  134. for line in reversed(lines[1:]):
  135. v2 = float(line[1])
  136. results.append([
  137. line[0].replace("-32", ""),
  138. "%.1fx" % (float(line[3])/v2), # v1
  139. "%.1fx" % (float(line[5])/v2), # bs
  140. ])
  141. # move geomean to the end
  142. results.append(results[0])
  143. del results[0]
  144. def printtable(data):
  145. print("""
  146. <table>
  147. <thead>
  148. <tr><th>Benchmark</th><th>go-toml v1</th><th>BurntSushi/toml</th></tr>
  149. </thead>
  150. <tbody>""")
  151. for r in data:
  152. print(" <tr><td>{}</td><td>{}</td><td>{}</td></tr>".format(*r))
  153. print(""" </tbody>
  154. </table>""")
  155. def match(x):
  156. return "ReferenceFile" in x[0] or "HugoFrontMatter" in x[0]
  157. above = [x for x in results if match(x)]
  158. below = [x for x in results if not match(x)]
  159. printtable(above)
  160. print("<details><summary>See more</summary>")
  161. print("""<p>The table above has the results of the most common use-cases. The table below
  162. contains the results of all benchmarks, including unrealistic ones. It is
  163. provided for completeness.</p>""")
  164. printtable(below)
  165. print('<p>This table can be generated with <code>./ci.sh benchmark -a -html</code>.</p>')
  166. print("</details>")
  167. EOF
  168. }
  169. benchmark() {
  170. case "$1" in
  171. -d)
  172. shift
  173. target="${1?Need to provide a target branch argument}"
  174. old=`fmktemp ${target}`
  175. bench "${target}" "${old}"
  176. new=`fmktemp HEAD`
  177. bench HEAD "${new}"
  178. benchstat "${old}" "${new}"
  179. return 0
  180. ;;
  181. -a)
  182. shift
  183. v2stats=`fmktemp go-toml-v2`
  184. bench HEAD "${v2stats}" "github.com/pelletier/go-toml/v2"
  185. v1stats=`fmktemp go-toml-v1`
  186. bench HEAD "${v1stats}" "github.com/pelletier/go-toml"
  187. bsstats=`fmktemp bs-toml`
  188. bench HEAD "${bsstats}" "github.com/BurntSushi/toml"
  189. cp "${v2stats}" go-toml-v2.txt
  190. cp "${v1stats}" go-toml-v1.txt
  191. cp "${bsstats}" bs-toml.txt
  192. if [ "$1" = "-html" ]; then
  193. tmpcsv=`fmktemp csv`
  194. benchstat -csv -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt > $tmpcsv
  195. benchstathtml $tmpcsv
  196. else
  197. benchstat -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt
  198. fi
  199. rm -f go-toml-v2.txt go-toml-v1.txt bs-toml.txt
  200. return $?
  201. esac
  202. bench "${1-HEAD}" `mktemp`
  203. }
  204. case "$1" in
  205. coverage) shift; coverage $@;;
  206. benchmark) shift; benchmark $@;;
  207. *) usage "bad argument $1";;
  208. esac