Skip to content

Commit

Permalink
Merge pull request #545 from tsearle/allow_out_of_order_timestamps
Browse files Browse the repository at this point in the history 
add flag to allow heplify-server to send packets out of order to loki
  • Loading branch information
@adubovikov
adubovikov authored Feb 20, 2024
2 parents c755212 + 56a0919 commit 8059379
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 76 deletions.
137 changes: 69 additions & 68 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,72 +5,73 @@ const Version = "heplify-server 1.59.6"
var Setting HeplifyServer

type HeplifyServer struct {
HEPAddr string `default:"0.0.0.0:9060"`
HEPTCPAddr string `default:""`
HEPTLSAddr string `default:""`
HEPWSAddr string `default:""`
ESAddr string `default:""`
ESDiscovery bool `default:"true"`
HEPv2Enable bool `default:"true"`
ESUser string `default:""`
ESPass string `default:""`
LokiURL string `default:""`
LokiBulk int `default:"400"`
LokiTimer int `default:"4"`
LokiBuffer int `default:"100000"`
LokiHEPFilter []int `default:"1,5,100"`
LokiIPPortLabels bool `default:"false"`
ForceHEPPayload []int `default:""`
PromAddr string `default:":9096"`
PromTargetIP string `default:""`
PromTargetName string `default:""`
DBShema string `default:"homer5"`
DBDriver string `default:"mysql"`
DBAddr string `default:"localhost:3306"`
DBSSLMode string `default:"disable"`
DBUser string `default:"root"`
DBPass string `default:""`
DBDataTable string `default:"homer_data"`
DBConfTable string `default:"homer_configuration"`
DBBulk int `default:"400"`
DBTimer int `default:"4"`
DBBuffer int `default:"400000"`
DBWorker int `default:"8"`
DBRotate bool `default:"true"`
DBPartLog string `default:"2h"`
DBPartIsup string `default:"6h"`
DBPartSip string `default:"2h"`
DBPartQos string `default:"6h"`
DBDropDays int `default:"14"`
DBDropDaysCall int `default:"0"`
DBDropDaysRegister int `default:"0"`
DBDropDaysDefault int `default:"0"`
DBDropOnStart bool `default:"false"`
DBUsageProtection bool `default:"false"`
DBUsageScheme string `default:"percentage"`
DBPercentageUsage string `default:"80%"`
DBMaxSize string `default:"20GB"`
DBProcDropLimit int `default:"2"`
Dedup bool `default:"false"`
DiscardMethod []string `default:""`
CensorMethod []string `default:""`
AlegIDs []string `default:""`
ForceALegID bool `default:"false"`
CustomHeader []string `default:""`
IgnoreCaseCH bool `default:"false"`
SIPHeader []string `default:"ruri_user,ruri_domain,from_user,from_tag,to_user,callid,cseq,method,user_agent"`
LogDbg string `default:""`
LogLvl string `default:"info"`
LogStd bool `default:"false"`
LogSys bool `default:"false"`
Config string `default:"./heplify-server.toml"`
ConfigHTTPAddr string `default:""`
ConfigHTTPPW string `default:""`
Version bool `default:"false"`
ScriptEnable bool `default:"false"`
ScriptEngine string `default:"lua"`
ScriptFolder string `default:""`
ScriptHEPFilter []int `default:"1,5,100"`
TLSCertFolder string `default:"."`
TLSMinVersion string `default:"1.2"`
HEPAddr string `default:"0.0.0.0:9060"`
HEPTCPAddr string `default:""`
HEPTLSAddr string `default:""`
HEPWSAddr string `default:""`
ESAddr string `default:""`
ESDiscovery bool `default:"true"`
HEPv2Enable bool `default:"true"`
ESUser string `default:""`
ESPass string `default:""`
LokiURL string `default:""`
LokiBulk int `default:"400"`
LokiTimer int `default:"4"`
LokiBuffer int `default:"100000"`
LokiHEPFilter []int `default:"1,5,100"`
LokiIPPortLabels bool `default:"false"`
LokiAllowOutOfOrder bool `default:"false"`
ForceHEPPayload []int `default:""`
PromAddr string `default:":9096"`
PromTargetIP string `default:""`
PromTargetName string `default:""`
DBShema string `default:"homer5"`
DBDriver string `default:"mysql"`
DBAddr string `default:"localhost:3306"`
DBSSLMode string `default:"disable"`
DBUser string `default:"root"`
DBPass string `default:""`
DBDataTable string `default:"homer_data"`
DBConfTable string `default:"homer_configuration"`
DBBulk int `default:"400"`
DBTimer int `default:"4"`
DBBuffer int `default:"400000"`
DBWorker int `default:"8"`
DBRotate bool `default:"true"`
DBPartLog string `default:"2h"`
DBPartIsup string `default:"6h"`
DBPartSip string `default:"2h"`
DBPartQos string `default:"6h"`
DBDropDays int `default:"14"`
DBDropDaysCall int `default:"0"`
DBDropDaysRegister int `default:"0"`
DBDropDaysDefault int `default:"0"`
DBDropOnStart bool `default:"false"`
DBUsageProtection bool `default:"false"`
DBUsageScheme string `default:"percentage"`
DBPercentageUsage string `default:"80%"`
DBMaxSize string `default:"20GB"`
DBProcDropLimit int `default:"2"`
Dedup bool `default:"false"`
DiscardMethod []string `default:""`
CensorMethod []string `default:""`
AlegIDs []string `default:""`
ForceALegID bool `default:"false"`
CustomHeader []string `default:""`
IgnoreCaseCH bool `default:"false"`
SIPHeader []string `default:"ruri_user,ruri_domain,from_user,from_tag,to_user,callid,cseq,method,user_agent"`
LogDbg string `default:""`
LogLvl string `default:"info"`
LogStd bool `default:"false"`
LogSys bool `default:"false"`
Config string `default:"./heplify-server.toml"`
ConfigHTTPAddr string `default:""`
ConfigHTTPPW string `default:""`
Version bool `default:"false"`
ScriptEnable bool `default:"false"`
ScriptEngine string `default:"lua"`
ScriptFolder string `default:""`
ScriptHEPFilter []int `default:"1,5,100"`
TLSCertFolder string `default:"."`
TLSMinVersion string `default:"1.2"`
}
3 changes: 2 additions & 1 deletion example/homer5_config/heplify-server.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ LokiBulk = 200
LokiTimer = 4
LokiBuffer = 100000
LokiHEPFilter = [1,5,100]
LokiAllowOutOfOrder = false
PromAddr = ""
PromTargetIP = ""
PromTargetName = ""
Expand Down Expand Up @@ -58,4 +59,4 @@ ConfigHTTPAddr = ""
# ConfigHTTPAddr = "0.0.0.0:9876"
# -------------------------------------
# To hot reload PromTargetIP and PromTargetName run:
# killall -HUP heplify-server
# killall -HUP heplify-server
1 change: 1 addition & 0 deletions example/homer7_config/heplify-server.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ LokiBulk = 200
LokiTimer = 4
LokiBuffer = 100000
LokiHEPFilter = [1,5,100]
LokiAllowOutOfOrder = false
ForceHEPPayload = []
PromAddr = ""
PromTargetIP = ""
Expand Down
18 changes: 11 additions & 7 deletions remotelog/loki.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,18 @@ type entry struct {
}

type Loki struct {
URL string
BatchWait time.Duration
BatchSize int
URL string
BatchWait time.Duration
BatchSize int
AllowOutOfOrder bool
entry
}

func (l *Loki) setup() error {
l.BatchSize = config.Setting.LokiBulk * 1024
l.BatchWait = time.Duration(config.Setting.LokiTimer) * time.Second
l.URL = config.Setting.LokiURL
l.AllowOutOfOrder = config.Setting.LokiAllowOutOfOrder

u, err := url.Parse(l.URL)
if err != nil {
Expand Down Expand Up @@ -100,11 +102,13 @@ func (l *Loki) start(hCh chan *decoder.HEP) {
return
}
curPktTime = pkt.Timestamp
// guard against entry out of order errors
if lastPktTime.After(curPktTime) {
curPktTime = time.Now()
if !l.AllowOutOfOrder {
// guard against entry out of order errors
if lastPktTime.After(curPktTime) {
curPktTime = time.Now()
}
lastPktTime = curPktTime
}
lastPktTime = curPktTime

pktMeta.Reset()

Expand Down

0 comments on commit 8059379

Please sign in to comment.