sampler.go 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516
  1. // Copyright (c) 2017 Uber Technologies, Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package jaeger
  15. import (
  16. "fmt"
  17. "math"
  18. "strings"
  19. "sync"
  20. "github.com/uber/jaeger-client-go/thrift-gen/sampling"
  21. "github.com/uber/jaeger-client-go/utils"
  22. )
  23. const (
  24. defaultMaxOperations = 2000
  25. )
  26. // Sampler decides whether a new trace should be sampled or not.
  27. type Sampler interface {
  28. // IsSampled decides whether a trace with given `id` and `operation`
  29. // should be sampled. This function will also return the tags that
  30. // can be used to identify the type of sampling that was applied to
  31. // the root span. Most simple samplers would return two tags,
  32. // sampler.type and sampler.param, similar to those used in the Configuration
  33. IsSampled(id TraceID, operation string) (sampled bool, tags []Tag)
  34. // Close does a clean shutdown of the sampler, stopping any background
  35. // go-routines it may have started.
  36. Close()
  37. // Equal checks if the `other` sampler is functionally equivalent
  38. // to this sampler.
  39. // TODO (breaking change) remove this function. See PerOperationSampler.Equals for explanation.
  40. Equal(other Sampler) bool
  41. }
  42. // -----------------------
  43. // ConstSampler is a sampler that always makes the same decision.
  44. type ConstSampler struct {
  45. legacySamplerV1Base
  46. Decision bool
  47. tags []Tag
  48. }
  49. // NewConstSampler creates a ConstSampler.
  50. func NewConstSampler(sample bool) *ConstSampler {
  51. tags := []Tag{
  52. {key: SamplerTypeTagKey, value: SamplerTypeConst},
  53. {key: SamplerParamTagKey, value: sample},
  54. }
  55. s := &ConstSampler{
  56. Decision: sample,
  57. tags: tags,
  58. }
  59. s.delegate = s.IsSampled
  60. return s
  61. }
  62. // IsSampled implements IsSampled() of Sampler.
  63. func (s *ConstSampler) IsSampled(id TraceID, operation string) (bool, []Tag) {
  64. return s.Decision, s.tags
  65. }
  66. // Close implements Close() of Sampler.
  67. func (s *ConstSampler) Close() {
  68. // nothing to do
  69. }
  70. // Equal implements Equal() of Sampler.
  71. func (s *ConstSampler) Equal(other Sampler) bool {
  72. if o, ok := other.(*ConstSampler); ok {
  73. return s.Decision == o.Decision
  74. }
  75. return false
  76. }
  77. // String is used to log sampler details.
  78. func (s *ConstSampler) String() string {
  79. return fmt.Sprintf("ConstSampler(decision=%t)", s.Decision)
  80. }
  81. // -----------------------
  82. // ProbabilisticSampler is a sampler that randomly samples a certain percentage
  83. // of traces.
  84. type ProbabilisticSampler struct {
  85. legacySamplerV1Base
  86. samplingRate float64
  87. samplingBoundary uint64
  88. tags []Tag
  89. }
  90. const maxRandomNumber = ^(uint64(1) << 63) // i.e. 0x7fffffffffffffff
  91. // NewProbabilisticSampler creates a sampler that randomly samples a certain percentage of traces specified by the
  92. // samplingRate, in the range between 0.0 and 1.0.
  93. //
  94. // It relies on the fact that new trace IDs are 63bit random numbers themselves, thus making the sampling decision
  95. // without generating a new random number, but simply calculating if traceID < (samplingRate * 2^63).
  96. // TODO remove the error from this function for next major release
  97. func NewProbabilisticSampler(samplingRate float64) (*ProbabilisticSampler, error) {
  98. if samplingRate < 0.0 || samplingRate > 1.0 {
  99. return nil, fmt.Errorf("Sampling Rate must be between 0.0 and 1.0, received %f", samplingRate)
  100. }
  101. return newProbabilisticSampler(samplingRate), nil
  102. }
  103. func newProbabilisticSampler(samplingRate float64) *ProbabilisticSampler {
  104. s := new(ProbabilisticSampler)
  105. s.delegate = s.IsSampled
  106. return s.init(samplingRate)
  107. }
  108. func (s *ProbabilisticSampler) init(samplingRate float64) *ProbabilisticSampler {
  109. s.samplingRate = math.Max(0.0, math.Min(samplingRate, 1.0))
  110. s.samplingBoundary = uint64(float64(maxRandomNumber) * s.samplingRate)
  111. s.tags = []Tag{
  112. {key: SamplerTypeTagKey, value: SamplerTypeProbabilistic},
  113. {key: SamplerParamTagKey, value: s.samplingRate},
  114. }
  115. return s
  116. }
  117. // SamplingRate returns the sampling probability this sampled was constructed with.
  118. func (s *ProbabilisticSampler) SamplingRate() float64 {
  119. return s.samplingRate
  120. }
  121. // IsSampled implements IsSampled() of Sampler.
  122. func (s *ProbabilisticSampler) IsSampled(id TraceID, operation string) (bool, []Tag) {
  123. return s.samplingBoundary >= id.Low&maxRandomNumber, s.tags
  124. }
  125. // Close implements Close() of Sampler.
  126. func (s *ProbabilisticSampler) Close() {
  127. // nothing to do
  128. }
  129. // Equal implements Equal() of Sampler.
  130. func (s *ProbabilisticSampler) Equal(other Sampler) bool {
  131. if o, ok := other.(*ProbabilisticSampler); ok {
  132. return s.samplingBoundary == o.samplingBoundary
  133. }
  134. return false
  135. }
  136. // Update modifies in-place the sampling rate. Locking must be done externally.
  137. func (s *ProbabilisticSampler) Update(samplingRate float64) error {
  138. if samplingRate < 0.0 || samplingRate > 1.0 {
  139. return fmt.Errorf("Sampling Rate must be between 0.0 and 1.0, received %f", samplingRate)
  140. }
  141. s.init(samplingRate)
  142. return nil
  143. }
  144. // String is used to log sampler details.
  145. func (s *ProbabilisticSampler) String() string {
  146. return fmt.Sprintf("ProbabilisticSampler(samplingRate=%v)", s.samplingRate)
  147. }
  148. // -----------------------
  149. // RateLimitingSampler samples at most maxTracesPerSecond. The distribution of sampled traces follows
  150. // burstiness of the service, i.e. a service with uniformly distributed requests will have those
  151. // requests sampled uniformly as well, but if requests are bursty, especially sub-second, then a
  152. // number of sequential requests can be sampled each second.
  153. type RateLimitingSampler struct {
  154. legacySamplerV1Base
  155. maxTracesPerSecond float64
  156. rateLimiter *utils.ReconfigurableRateLimiter
  157. tags []Tag
  158. }
  159. // NewRateLimitingSampler creates new RateLimitingSampler.
  160. func NewRateLimitingSampler(maxTracesPerSecond float64) *RateLimitingSampler {
  161. s := new(RateLimitingSampler)
  162. s.delegate = s.IsSampled
  163. return s.init(maxTracesPerSecond)
  164. }
  165. func (s *RateLimitingSampler) init(maxTracesPerSecond float64) *RateLimitingSampler {
  166. if s.rateLimiter == nil {
  167. s.rateLimiter = utils.NewRateLimiter(maxTracesPerSecond, math.Max(maxTracesPerSecond, 1.0))
  168. } else {
  169. s.rateLimiter.Update(maxTracesPerSecond, math.Max(maxTracesPerSecond, 1.0))
  170. }
  171. s.maxTracesPerSecond = maxTracesPerSecond
  172. s.tags = []Tag{
  173. {key: SamplerTypeTagKey, value: SamplerTypeRateLimiting},
  174. {key: SamplerParamTagKey, value: maxTracesPerSecond},
  175. }
  176. return s
  177. }
  178. // IsSampled implements IsSampled() of Sampler.
  179. func (s *RateLimitingSampler) IsSampled(id TraceID, operation string) (bool, []Tag) {
  180. return s.rateLimiter.CheckCredit(1.0), s.tags
  181. }
  182. // Update reconfigures the rate limiter, while preserving its accumulated balance.
  183. // Locking must be done externally.
  184. func (s *RateLimitingSampler) Update(maxTracesPerSecond float64) {
  185. if s.maxTracesPerSecond != maxTracesPerSecond {
  186. s.init(maxTracesPerSecond)
  187. }
  188. }
  189. // Close does nothing.
  190. func (s *RateLimitingSampler) Close() {
  191. // nothing to do
  192. }
  193. // Equal compares with another sampler.
  194. func (s *RateLimitingSampler) Equal(other Sampler) bool {
  195. if o, ok := other.(*RateLimitingSampler); ok {
  196. return s.maxTracesPerSecond == o.maxTracesPerSecond
  197. }
  198. return false
  199. }
  200. // String is used to log sampler details.
  201. func (s *RateLimitingSampler) String() string {
  202. return fmt.Sprintf("RateLimitingSampler(maxTracesPerSecond=%v)", s.maxTracesPerSecond)
  203. }
  204. // -----------------------
  205. // GuaranteedThroughputProbabilisticSampler is a sampler that leverages both ProbabilisticSampler and
  206. // RateLimitingSampler. The RateLimitingSampler is used as a guaranteed lower bound sampler such that
  207. // every operation is sampled at least once in a time interval defined by the lowerBound. ie a lowerBound
  208. // of 1.0 / (60 * 10) will sample an operation at least once every 10 minutes.
  209. //
  210. // The ProbabilisticSampler is given higher priority when tags are emitted, ie. if IsSampled() for both
  211. // samplers return true, the tags for ProbabilisticSampler will be used.
  212. type GuaranteedThroughputProbabilisticSampler struct {
  213. probabilisticSampler *ProbabilisticSampler
  214. lowerBoundSampler *RateLimitingSampler
  215. tags []Tag
  216. samplingRate float64
  217. lowerBound float64
  218. }
  219. // NewGuaranteedThroughputProbabilisticSampler returns a delegating sampler that applies both
  220. // ProbabilisticSampler and RateLimitingSampler.
  221. func NewGuaranteedThroughputProbabilisticSampler(
  222. lowerBound, samplingRate float64,
  223. ) (*GuaranteedThroughputProbabilisticSampler, error) {
  224. return newGuaranteedThroughputProbabilisticSampler(lowerBound, samplingRate), nil
  225. }
  226. func newGuaranteedThroughputProbabilisticSampler(lowerBound, samplingRate float64) *GuaranteedThroughputProbabilisticSampler {
  227. s := &GuaranteedThroughputProbabilisticSampler{
  228. lowerBoundSampler: NewRateLimitingSampler(lowerBound),
  229. lowerBound: lowerBound,
  230. }
  231. s.setProbabilisticSampler(samplingRate)
  232. return s
  233. }
  234. func (s *GuaranteedThroughputProbabilisticSampler) setProbabilisticSampler(samplingRate float64) {
  235. if s.probabilisticSampler == nil {
  236. s.probabilisticSampler = newProbabilisticSampler(samplingRate)
  237. } else if s.samplingRate != samplingRate {
  238. s.probabilisticSampler.init(samplingRate)
  239. }
  240. // since we don't validate samplingRate, sampler may have clamped it to [0, 1] interval
  241. samplingRate = s.probabilisticSampler.SamplingRate()
  242. if s.samplingRate != samplingRate || s.tags == nil {
  243. s.samplingRate = s.probabilisticSampler.SamplingRate()
  244. s.tags = []Tag{
  245. {key: SamplerTypeTagKey, value: SamplerTypeLowerBound},
  246. {key: SamplerParamTagKey, value: s.samplingRate},
  247. }
  248. }
  249. }
  250. // IsSampled implements IsSampled() of Sampler.
  251. func (s *GuaranteedThroughputProbabilisticSampler) IsSampled(id TraceID, operation string) (bool, []Tag) {
  252. if sampled, tags := s.probabilisticSampler.IsSampled(id, operation); sampled {
  253. s.lowerBoundSampler.IsSampled(id, operation)
  254. return true, tags
  255. }
  256. sampled, _ := s.lowerBoundSampler.IsSampled(id, operation)
  257. return sampled, s.tags
  258. }
  259. // Close implements Close() of Sampler.
  260. func (s *GuaranteedThroughputProbabilisticSampler) Close() {
  261. s.probabilisticSampler.Close()
  262. s.lowerBoundSampler.Close()
  263. }
  264. // Equal implements Equal() of Sampler.
  265. func (s *GuaranteedThroughputProbabilisticSampler) Equal(other Sampler) bool {
  266. // NB The Equal() function is expensive and will be removed. See PerOperationSampler.Equal() for
  267. // more information.
  268. return false
  269. }
  270. // this function should only be called while holding a Write lock
  271. func (s *GuaranteedThroughputProbabilisticSampler) update(lowerBound, samplingRate float64) {
  272. s.setProbabilisticSampler(samplingRate)
  273. if s.lowerBound != lowerBound {
  274. s.lowerBoundSampler.Update(lowerBound)
  275. s.lowerBound = lowerBound
  276. }
  277. }
  278. func (s GuaranteedThroughputProbabilisticSampler) String() string {
  279. return fmt.Sprintf("GuaranteedThroughputProbabilisticSampler(lowerBound=%f, samplingRate=%f)", s.lowerBound, s.samplingRate)
  280. }
  281. // -----------------------
  282. // PerOperationSampler is a delegating sampler that applies GuaranteedThroughputProbabilisticSampler
  283. // on a per-operation basis.
  284. type PerOperationSampler struct {
  285. sync.RWMutex
  286. samplers map[string]*GuaranteedThroughputProbabilisticSampler
  287. defaultSampler *ProbabilisticSampler
  288. lowerBound float64
  289. maxOperations int
  290. // see description in PerOperationSamplerParams
  291. operationNameLateBinding bool
  292. }
  293. // NewAdaptiveSampler returns a new PerOperationSampler.
  294. // Deprecated: please use NewPerOperationSampler.
  295. func NewAdaptiveSampler(strategies *sampling.PerOperationSamplingStrategies, maxOperations int) (*PerOperationSampler, error) {
  296. return NewPerOperationSampler(PerOperationSamplerParams{
  297. MaxOperations: maxOperations,
  298. Strategies: strategies,
  299. }), nil
  300. }
  301. // PerOperationSamplerParams defines parameters when creating PerOperationSampler.
  302. type PerOperationSamplerParams struct {
  303. // Max number of operations that will be tracked. Other operations will be given default strategy.
  304. MaxOperations int
  305. // Opt-in feature for applications that require late binding of span name via explicit call to SetOperationName.
  306. // When this feature is enabled, the sampler will return retryable=true from OnCreateSpan(), thus leaving
  307. // the sampling decision as non-final (and the span as writeable). This may lead to degraded performance
  308. // in applications that always provide the correct span name on trace creation.
  309. //
  310. // For backwards compatibility this option is off by default.
  311. OperationNameLateBinding bool
  312. // Initial configuration of the sampling strategies (usually retrieved from the backend by Remote Sampler).
  313. Strategies *sampling.PerOperationSamplingStrategies
  314. }
  315. // NewPerOperationSampler returns a new PerOperationSampler.
  316. func NewPerOperationSampler(params PerOperationSamplerParams) *PerOperationSampler {
  317. if params.MaxOperations <= 0 {
  318. params.MaxOperations = defaultMaxOperations
  319. }
  320. samplers := make(map[string]*GuaranteedThroughputProbabilisticSampler)
  321. for _, strategy := range params.Strategies.PerOperationStrategies {
  322. sampler := newGuaranteedThroughputProbabilisticSampler(
  323. params.Strategies.DefaultLowerBoundTracesPerSecond,
  324. strategy.ProbabilisticSampling.SamplingRate,
  325. )
  326. samplers[strategy.Operation] = sampler
  327. }
  328. return &PerOperationSampler{
  329. samplers: samplers,
  330. defaultSampler: newProbabilisticSampler(params.Strategies.DefaultSamplingProbability),
  331. lowerBound: params.Strategies.DefaultLowerBoundTracesPerSecond,
  332. maxOperations: params.MaxOperations,
  333. operationNameLateBinding: params.OperationNameLateBinding,
  334. }
  335. }
  336. // IsSampled is not used and only exists to match Sampler V1 API.
  337. // TODO (breaking change) remove when upgrading everything to SamplerV2
  338. func (s *PerOperationSampler) IsSampled(id TraceID, operation string) (bool, []Tag) {
  339. return false, nil
  340. }
  341. func (s *PerOperationSampler) trySampling(span *Span, operationName string) (bool, []Tag) {
  342. samplerV1 := s.getSamplerForOperation(operationName)
  343. var sampled bool
  344. var tags []Tag
  345. if span.context.samplingState.isLocalRootSpan(span.context.spanID) {
  346. sampled, tags = samplerV1.IsSampled(span.context.TraceID(), operationName)
  347. }
  348. return sampled, tags
  349. }
  350. // OnCreateSpan implements OnCreateSpan of SamplerV2.
  351. func (s *PerOperationSampler) OnCreateSpan(span *Span) SamplingDecision {
  352. sampled, tags := s.trySampling(span, span.OperationName())
  353. return SamplingDecision{Sample: sampled, Retryable: s.operationNameLateBinding, Tags: tags}
  354. }
  355. // OnSetOperationName implements OnSetOperationName of SamplerV2.
  356. func (s *PerOperationSampler) OnSetOperationName(span *Span, operationName string) SamplingDecision {
  357. sampled, tags := s.trySampling(span, operationName)
  358. return SamplingDecision{Sample: sampled, Retryable: false, Tags: tags}
  359. }
  360. // OnSetTag implements OnSetTag of SamplerV2.
  361. func (s *PerOperationSampler) OnSetTag(span *Span, key string, value interface{}) SamplingDecision {
  362. return SamplingDecision{Sample: false, Retryable: true}
  363. }
  364. // OnFinishSpan implements OnFinishSpan of SamplerV2.
  365. func (s *PerOperationSampler) OnFinishSpan(span *Span) SamplingDecision {
  366. return SamplingDecision{Sample: false, Retryable: true}
  367. }
  368. func (s *PerOperationSampler) getSamplerForOperation(operation string) Sampler {
  369. s.RLock()
  370. sampler, ok := s.samplers[operation]
  371. if ok {
  372. defer s.RUnlock()
  373. return sampler
  374. }
  375. s.RUnlock()
  376. s.Lock()
  377. defer s.Unlock()
  378. // Check if sampler has already been created
  379. sampler, ok = s.samplers[operation]
  380. if ok {
  381. return sampler
  382. }
  383. // Store only up to maxOperations of unique ops.
  384. if len(s.samplers) >= s.maxOperations {
  385. return s.defaultSampler
  386. }
  387. newSampler := newGuaranteedThroughputProbabilisticSampler(s.lowerBound, s.defaultSampler.SamplingRate())
  388. s.samplers[operation] = newSampler
  389. return newSampler
  390. }
  391. // Close invokes Close on all underlying samplers.
  392. func (s *PerOperationSampler) Close() {
  393. s.Lock()
  394. defer s.Unlock()
  395. for _, sampler := range s.samplers {
  396. sampler.Close()
  397. }
  398. s.defaultSampler.Close()
  399. }
  400. func (s *PerOperationSampler) String() string {
  401. var sb strings.Builder
  402. fmt.Fprintf(&sb, "PerOperationSampler(defaultSampler=%v, ", s.defaultSampler)
  403. fmt.Fprintf(&sb, "lowerBound=%f, ", s.lowerBound)
  404. fmt.Fprintf(&sb, "maxOperations=%d, ", s.maxOperations)
  405. fmt.Fprintf(&sb, "operationNameLateBinding=%t, ", s.operationNameLateBinding)
  406. fmt.Fprintf(&sb, "numOperations=%d,\n", len(s.samplers))
  407. fmt.Fprintf(&sb, "samplers=[")
  408. for operationName, sampler := range s.samplers {
  409. fmt.Fprintf(&sb, "\n(operationName=%s, sampler=%v)", operationName, sampler)
  410. }
  411. fmt.Fprintf(&sb, "])")
  412. return sb.String()
  413. }
  414. // Equal is not used.
  415. // TODO (breaking change) remove this in the future
  416. func (s *PerOperationSampler) Equal(other Sampler) bool {
  417. // NB The Equal() function is overly expensive for PerOperationSampler since it's composed of multiple
  418. // samplers which all need to be initialized before this function can be called for a comparison.
  419. // Therefore, PerOperationSampler uses the update() function to only alter the samplers that need
  420. // changing. Hence this function always returns false so that the update function can be called.
  421. // Once the Equal() function is removed from the Sampler API, this will no longer be needed.
  422. return false
  423. }
  424. func (s *PerOperationSampler) update(strategies *sampling.PerOperationSamplingStrategies) {
  425. s.Lock()
  426. defer s.Unlock()
  427. newSamplers := map[string]*GuaranteedThroughputProbabilisticSampler{}
  428. for _, strategy := range strategies.PerOperationStrategies {
  429. operation := strategy.Operation
  430. samplingRate := strategy.ProbabilisticSampling.SamplingRate
  431. lowerBound := strategies.DefaultLowerBoundTracesPerSecond
  432. if sampler, ok := s.samplers[operation]; ok {
  433. sampler.update(lowerBound, samplingRate)
  434. newSamplers[operation] = sampler
  435. } else {
  436. sampler := newGuaranteedThroughputProbabilisticSampler(
  437. lowerBound,
  438. samplingRate,
  439. )
  440. newSamplers[operation] = sampler
  441. }
  442. }
  443. s.lowerBound = strategies.DefaultLowerBoundTracesPerSecond
  444. if s.defaultSampler.SamplingRate() != strategies.DefaultSamplingProbability {
  445. s.defaultSampler = newProbabilisticSampler(strategies.DefaultSamplingProbability)
  446. }
  447. s.samplers = newSamplers
  448. }