mirrors/dnscrypt-proxy
1
0
Fork 0
mirror of https://github.com/DNSCrypt/dnscrypt-proxy.git synced 2025-03-03 18:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Back to VividCortex/ewma

Browse source
This commit is contained in:
Frank Denis 2022-04-05 14:04:26 +02:00
parent 77dc3b1e85
commit 5fca7ea49e
14 changed files with 300 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
dnscrypt-proxy/estimators.go
View file

@ -3,23 +3,19 @@ package main
import (
"sync"
"github.com/lifenjoiner/ewma"
)
const (
SizeEstimatorEwmaDecay = 100.0
"github.com/VividCortex/ewma"
)
type QuestionSizeEstimator struct {
sync.RWMutex
minQuestionSize int
ewma *ewma.EWMA
ewma ewma.MovingAverage
}
func NewQuestionSizeEstimator() QuestionSizeEstimator {
return QuestionSizeEstimator{
minQuestionSize: InitialMinQuestionSize,
ewma: ewma.NewMovingAverage(SizeEstimatorEwmaDecay),
ewma: &ewma.SimpleEWMA{},
}
}

4
dnscrypt-proxy/serversInfo.go
View file

@ -15,10 +15,10 @@ import (
"sync"
"time"
"github.com/VividCortex/ewma"
"github.com/jedisct1/dlog"
clocksmith "github.com/jedisct1/go-clocksmith"
stamps "github.com/jedisct1/go-dnsstamps"
"github.com/lifenjoiner/ewma"
"github.com/miekg/dns"
"golang.org/x/crypto/ed25519"
)
@ -46,7 +46,7 @@ type DOHClientCreds struct {
type ServerInfo struct {
DOHClientCreds DOHClientCreds
lastActionTS time.Time
rtt *ewma.EWMA
rtt ewma.MovingAverage
Name string
HostName string
UDPAddr *net.UDPAddr

2
go.mod
View file

@ -4,6 +4,7 @@ go 1.18
require (
github.com/BurntSushi/toml v1.0.0
github.com/VividCortex/ewma v1.2.0
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
github.com/dchest/safefile v0.0.0-20151022103144-855e8d98f185
github.com/hashicorp/go-immutable-radix v1.3.1
@ -17,7 +18,6 @@ require (
github.com/jedisct1/xsecretbox v0.0.0-20210927135450-ebe41aef7bef
github.com/k-sone/critbitgo v1.4.0
github.com/kardianos/service v1.2.1
github.com/lifenjoiner/ewma v0.0.0-20210320054258-4f227d7eb8a2
github.com/miekg/dns v1.1.48
github.com/powerman/check v1.6.0
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29

4
go.sum
View file

@ -52,6 +52,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@ -421,8 +423,6 @@ github.com/letsencrypt/pkcs11key/v4 v4.0.0/go.mod h1:EFUvBDay26dErnNb70Nd0/VW3tJ
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lifenjoiner/ewma v0.0.0-20210320054258-4f227d7eb8a2 h1:eD3+F7WMC7wryFGBrLSyzoRqK+kR7nCT/9VT2E3XJzc=
github.com/lifenjoiner/ewma v0.0.0-20210320054258-4f227d7eb8a2/go.mod h1:SJvYtJnDKXqTrIvyRocCJmuNuM3bUb4krn9UbZXj+tw=
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=

3
vendor/github.com/VividCortex/ewma/.gitignore generated vendored Normal file
View file

@ -0,0 +1,3 @@
.DS_Store
.*.sw?
/coverage.txt

3
vendor/github.com/VividCortex/ewma/.whitesource generated vendored Normal file
View file

@ -0,0 +1,3 @@
{
"settingsInheritedFrom": "VividCortex/whitesource-config@master"
}

12
vendor/github.com/lifenjoiner/ewma/LICENSE → vendor/github.com/VividCortex/ewma/LICENSE generated vendored
View file

@ -1,6 +1,6 @@
MIT License
The MIT License
Copyright (c) 2021 lifenjoiner
Copyright (c) 2013 VividCortex
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@ -9,13 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

145
vendor/github.com/VividCortex/ewma/README.md generated vendored Normal file
View file

@ -0,0 +1,145 @@
# EWMA
[![GoDoc](https://godoc.org/github.com/VividCortex/ewma?status.svg)](https://godoc.org/github.com/VividCortex/ewma)
![build](https://github.com/VividCortex/ewma/workflows/build/badge.svg)
[![codecov](https://codecov.io/gh/VividCortex/ewma/branch/master/graph/badge.svg)](https://codecov.io/gh/VividCortex/ewma)
This repo provides Exponentially Weighted Moving Average algorithms, or EWMAs for short, [based on our
Quantifying Abnormal Behavior talk](https://vividcortex.com/blog/2013/07/23/a-fast-go-library-for-exponential-moving-averages/).
### Exponentially Weighted Moving Average
An exponentially weighted moving average is a way to continuously compute a type of
average for a series of numbers, as the numbers arrive. After a value in the series is
added to the average, its weight in the average decreases exponentially over time. This
biases the average towards more recent data. EWMAs are useful for several reasons, chiefly
their inexpensive computational and memory cost, as well as the fact that they represent
the recent central tendency of the series of values.
The EWMA algorithm requires a decay factor, alpha. The larger the alpha, the more the average
is biased towards recent history. The alpha must be between 0 and 1, and is typically
a fairly small number, such as 0.04. We will discuss the choice of alpha later.
The algorithm works thus, in pseudocode:
1. Multiply the next number in the series by alpha.
2. Multiply the current value of the average by 1 minus alpha.
3. Add the result of steps 1 and 2, and store it as the new current value of the average.
4. Repeat for each number in the series.
There are special-case behaviors for how to initialize the current value, and these vary
between implementations. One approach is to start with the first value in the series;
another is to average the first 10 or so values in the series using an arithmetic average,
and then begin the incremental updating of the average. Each method has pros and cons.
It may help to look at it pictorially. Suppose the series has five numbers, and we choose
alpha to be 0.50 for simplicity. Here's the series, with numbers in the neighborhood of 300.
![Data Series](https://user-images.githubusercontent.com/279875/28242350-463289a2-6977-11e7-88ca-fd778ccef1f0.png)
Now let's take the moving average of those numbers. First we set the average to the value
of the first number.
![EWMA Step 1](https://user-images.githubusercontent.com/279875/28242353-464c96bc-6977-11e7-9981-dc4e0789c7ba.png)
Next we multiply the next number by alpha, multiply the current value by 1-alpha, and add
them to generate a new value.
![EWMA Step 2](https://user-images.githubusercontent.com/279875/28242351-464abefa-6977-11e7-95d0-43900f29bef2.png)
This continues until we are done.
![EWMA Step N](https://user-images.githubusercontent.com/279875/28242352-464c58f0-6977-11e7-8cd0-e01e4efaac7f.png)
Notice how each of the values in the series decays by half each time a new value
is added, and the top of the bars in the lower portion of the image represents the
size of the moving average. It is a smoothed, or low-pass, average of the original
series.
For further reading, see [Exponentially weighted moving average](http://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average) on wikipedia.
### Choosing Alpha
Consider a fixed-size sliding-window moving average (not an exponentially weighted moving average)
that averages over the previous N samples. What is the average age of each sample? It is N/2.
Now suppose that you wish to construct a EWMA whose samples have the same average age. The formula
to compute the alpha required for this is: alpha = 2/(N+1). Proof is in the book
"Production and Operations Analysis" by Steven Nahmias.
So, for example, if you have a time-series with samples once per second, and you want to get the
moving average over the previous minute, you should use an alpha of .032786885. This, by the way,
is the constant alpha used for this repository's SimpleEWMA.
### Implementations
This repository contains two implementations of the EWMA algorithm, with different properties.
The implementations all conform to the MovingAverage interface, and the constructor returns
that type.
Current implementations assume an implicit time interval of 1.0 between every sample added.
That is, the passage of time is treated as though it's the same as the arrival of samples.
If you need time-based decay when samples are not arriving precisely at set intervals, then
this package will not support your needs at present.
#### SimpleEWMA
A SimpleEWMA is designed for low CPU and memory consumption. It **will** have different behavior than the VariableEWMA
for multiple reasons. It has no warm-up period and it uses a constant
decay. These properties let it use less memory. It will also behave
differently when it's equal to zero, which is assumed to mean
uninitialized, so if a value is likely to actually become zero over time,
then any non-zero value will cause a sharp jump instead of a small change.
#### VariableEWMA
Unlike SimpleEWMA, this supports a custom age which must be stored, and thus uses more memory.
It also has a "warmup" time when you start adding values to it. It will report a value of 0.0
until you have added the required number of samples to it. It uses some memory to store the
number of samples added to it. As a result it uses a little over twice the memory of SimpleEWMA.
## Usage
### API Documentation
View the GoDoc generated documentation [here](http://godoc.org/github.com/VividCortex/ewma).
```go
package main
import "github.com/VividCortex/ewma"
func main() {
samples := [100]float64{
4599, 5711, 4746, 4621, 5037, 4218, 4925, 4281, 5207, 5203, 5594, 5149,
}
e := ewma.NewMovingAverage() //=> Returns a SimpleEWMA if called without params
a := ewma.NewMovingAverage(5) //=> returns a VariableEWMA with a decay of 2 / (5 + 1)
for _, f := range samples {
e.Add(f)
a.Add(f)
}
e.Value() //=> 13.577404704631077
a.Value() //=> 1.5806140565521463e-12
}
```
## Contributing
We only accept pull requests for minor fixes or improvements. This includes:
* Small bug fixes
* Typos
* Documentation or comments
Please open issues to discuss new features. Pull requests for new features will be rejected,
so we recommend forking the repository and making changes in your fork for your use case.
## License
This repository is Copyright (c) 2013 VividCortex, Inc. All rights reserved.
It is licensed under the MIT license. Please see the LICENSE file for applicable license terms.

6
vendor/github.com/VividCortex/ewma/codecov.yml generated vendored Normal file
View file

@ -0,0 +1,6 @@
coverage:
status:
project:
default:
threshold: 15%
patch: off

126
vendor/github.com/VividCortex/ewma/ewma.go generated vendored Normal file
View file

@ -0,0 +1,126 @@
// Package ewma implements exponentially weighted moving averages.
package ewma
// Copyright (c) 2013 VividCortex, Inc. All rights reserved.
// Please see the LICENSE file for applicable license terms.
const (
// By default, we average over a one-minute period, which means the average
// age of the metrics in the period is 30 seconds.
AVG_METRIC_AGE float64 = 30.0
// The formula for computing the decay factor from the average age comes
// from "Production and Operations Analysis" by Steven Nahmias.
DECAY float64 = 2 / (float64(AVG_METRIC_AGE) + 1)
// For best results, the moving average should not be initialized to the
// samples it sees immediately. The book "Production and Operations
// Analysis" by Steven Nahmias suggests initializing the moving average to
// the mean of the first 10 samples. Until the VariableEwma has seen this
// many samples, it is not "ready" to be queried for the value of the
// moving average. This adds some memory cost.
WARMUP_SAMPLES uint8 = 10
)
// MovingAverage is the interface that computes a moving average over a time-
// series stream of numbers. The average may be over a window or exponentially
// decaying.
type MovingAverage interface {
Add(float64)
Value() float64
Set(float64)
}
// NewMovingAverage constructs a MovingAverage that computes an average with the
// desired characteristics in the moving window or exponential decay. If no
// age is given, it constructs a default exponentially weighted implementation
// that consumes minimal memory. The age is related to the decay factor alpha
// by the formula given for the DECAY constant. It signifies the average age
// of the samples as time goes to infinity.
func NewMovingAverage(age ...float64) MovingAverage {
if len(age) == 0 || age[0] == AVG_METRIC_AGE {
return new(SimpleEWMA)
}
return &VariableEWMA{
decay: 2 / (age[0] + 1),
}
}
// A SimpleEWMA represents the exponentially weighted moving average of a
// series of numbers. It WILL have different behavior than the VariableEWMA
// for multiple reasons. It has no warm-up period and it uses a constant
// decay. These properties let it use less memory. It will also behave
// differently when it's equal to zero, which is assumed to mean
// uninitialized, so if a value is likely to actually become zero over time,
// then any non-zero value will cause a sharp jump instead of a small change.
// However, note that this takes a long time, and the value may just
// decays to a stable value that's close to zero, but which won't be mistaken
// for uninitialized. See http://play.golang.org/p/litxBDr_RC for example.
type SimpleEWMA struct {
// The current value of the average. After adding with Add(), this is
// updated to reflect the average of all values seen thus far.
value float64
}
// Add adds a value to the series and updates the moving average.
func (e *SimpleEWMA) Add(value float64) {
if e.value == 0 { // this is a proxy for "uninitialized"
e.value = value
} else {
e.value = (value * DECAY) + (e.value * (1 - DECAY))
}
}
// Value returns the current value of the moving average.
func (e *SimpleEWMA) Value() float64 {
return e.value
}
// Set sets the EWMA's value.
func (e *SimpleEWMA) Set(value float64) {
e.value = value
}
// VariableEWMA represents the exponentially weighted moving average of a series of
// numbers. Unlike SimpleEWMA, it supports a custom age, and thus uses more memory.
type VariableEWMA struct {
// The multiplier factor by which the previous samples decay.
decay float64
// The current value of the average.
value float64
// The number of samples added to this instance.
count uint8
}
// Add adds a value to the series and updates the moving average.
func (e *VariableEWMA) Add(value float64) {
switch {
case e.count < WARMUP_SAMPLES:
e.count++
e.value += value
case e.count == WARMUP_SAMPLES:
e.count++
e.value = e.value / float64(WARMUP_SAMPLES)
e.value = (value * e.decay) + (e.value * (1 - e.decay))
default:
e.value = (value * e.decay) + (e.value * (1 - e.decay))
}
}
// Value returns the current value of the average, or 0.0 if the series hasn't
// warmed up yet.
func (e *VariableEWMA) Value() float64 {
if e.count <= WARMUP_SAMPLES {
return 0.0
}
return e.value
}
// Set sets the EWMA's value.
func (e *VariableEWMA) Set(value float64) {
e.value = value
if e.count <= WARMUP_SAMPLES {
e.count = WARMUP_SAMPLES + 1
}
}

15
vendor/github.com/lifenjoiner/ewma/.gitignore generated vendored
View file

@ -1,15 +0,0 @@
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/

17
vendor/github.com/lifenjoiner/ewma/README.md generated vendored
View file

@ -1,17 +0,0 @@
# EWMA
EWMA: Exponentially Weighted Moving Average algorithms
This is a variant of [EWMA](https://github.com/VividCortex/ewma).
### Variant
During the "warmup" stage, it uses a just-in-time `alpha` to get a more reasonable average.
Just one form.
[EWMA Comparisons](https://github.com/lifenjoiner/ewma/issues/1)
### Implementation
https://github.com/lifenjoiner/ewma

44
vendor/github.com/lifenjoiner/ewma/ewma.go generated vendored
View file

@ -1,44 +0,0 @@
// Package ewma: exponentially weighted moving averages
package ewma
// New EWMA by moving window size.
func NewMovingAverage(slide int) *EWMA {
return &EWMA{
slide: slide,
}
}
type EWMA struct {
// Too big slide is meaningless.
slide int
// Count before warmed up.
count int
// Decay by slide size.
decay float64
// The average.
value float64
}
// Add a value to the series and update the moving average.
func (a *EWMA) Add(value float64) {
switch {
case a.count <= a.slide:
a.count++
a.decay = 2 / float64(a.count + 1)
a.value = a.value * (1 - a.decay) + value * a.decay
default:
a.value = a.value * (1 - a.decay) + value * a.decay
}
}
// Return the current EWMA value.
func (a *EWMA) Value() float64 {
return a.value
}
// Set the EWMA value for continuing.
func (a *EWMA) Set(value float64) {
a.value = value
a.decay = 2 / float64(a.slide + 1)
a.count = a.slide + 1
}

6
vendor/modules.txt vendored
View file

@ -14,6 +14,9 @@ github.com/Masterminds/semver
# github.com/OpenPeeDeeP/depguard v1.0.1
## explicit; go 1.13
github.com/OpenPeeDeeP/depguard
# github.com/VividCortex/ewma v1.2.0
## explicit; go 1.12
github.com/VividCortex/ewma
# github.com/alexkohler/prealloc v1.0.0
## explicit; go 1.15
github.com/alexkohler/prealloc/pkg
@ -303,9 +306,6 @@ github.com/ldez/gomoddirectives
# github.com/ldez/tagliatelle v0.2.0
## explicit; go 1.16
github.com/ldez/tagliatelle
# github.com/lifenjoiner/ewma v0.0.0-20210320054258-4f227d7eb8a2
## explicit; go 1.16
github.com/lifenjoiner/ewma
# github.com/magiconair/properties v1.8.1
## explicit
github.com/magiconair/properties