From a26024d28caa34d7b3ea7405bfd9246ffb32105b Mon Sep 17 00:00:00 2001 From: e-gun Date: Sun, 1 Sep 2024 19:21:11 -0400 Subject: [PATCH] back to school tidyups --- go.mod | 20 ++++++------- internal/base/gen/setsandslices.go | 19 ++++++------ internal/base/mm/messaging.go | 8 +++++ internal/db/getmorph.go | 7 ++--- internal/search/resultformatting.go | 4 +-- internal/vec/vectorgraphovverrides.go | 2 +- internal/vlt/wsmessage.go | 4 +-- web/echoserver.go | 42 ++++++++++----------------- 8 files changed, 51 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index be97be1..1dcd395 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,11 @@ module github.com/e-gun/HipparchiaGoServer -go 1.22 +go 1.23 require ( github.com/e-gun/lastnlines v0.1.0 github.com/e-gun/nlp v0.0.2 + github.com/e-gun/policeresponses v0.0.2 github.com/e-gun/tsnemp v0.1.7 github.com/e-gun/wego v0.1.0 github.com/go-echarts/go-echarts/v2 v2.4.1 // v2.4 breaks the api @@ -13,20 +14,19 @@ require ( github.com/jackc/pgx/v5 v5.6.0 github.com/labstack/echo/v4 v4.12.0 github.com/pkg/profile v1.7.0 - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - golang.org/x/text v0.16.0 - gonum.org/v1/gonum v0.15.0 + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 + golang.org/x/text v0.17.0 + gonum.org/v1/gonum v0.15.1 ) require ( github.com/armon/go-radix v1.0.0 // indirect - github.com/e-gun/policeresponses v0.0.1 // indirect github.com/e-gun/safestack v0.1.1 // indirect github.com/e-gun/sparse v0.0.0-20230418220937-07063da15582 // indirect - github.com/felixge/fgprof v0.9.4 // indirect + github.com/felixge/fgprof v0.9.5 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect - github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect + github.com/google/pprof v0.0.0-20240829160300-da1f7e9f2b25 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -44,10 +44,10 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect - golang.org/x/crypto v0.25.0 // indirect - golang.org/x/net v0.27.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/net v0.28.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.23.0 // indirect + golang.org/x/sys v0.24.0 // indirect golang.org/x/time v0.6.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect ) diff --git a/internal/base/gen/setsandslices.go b/internal/base/gen/setsandslices.go index c3c625e..0018c2e 100644 --- a/internal/base/gen/setsandslices.go +++ b/internal/base/gen/setsandslices.go @@ -39,9 +39,16 @@ func Unique[T comparable](s []T) []T { set := ToSet(s) - var result []T + //var result []T + //for k := range set { + // result = append(result, k) + //} + + result := make([]T, len(set)) + count := 0 for k := range set { - result = append(result, k) + result[count] = k + count += 1 } return result @@ -96,11 +103,3 @@ func StringMapKeysIntoSlice[T any](mp map[string]T) []string { } return sl } - -// ChunkSlice - turn a slice into a slice of slices of size N; thanks to https://stackoverflow.com/questions/35179656/slice-chunking-in-go -func ChunkSlice[T any](items []T, size int) (chunks [][]T) { - for size < len(items) { - items, chunks = items[size:], append(chunks, items[0:size:size]) - } - return append(chunks, items) -} diff --git a/internal/base/mm/messaging.go b/internal/base/mm/messaging.go index 7c1fa32..3596991 100644 --- a/internal/base/mm/messaging.go +++ b/internal/base/mm/messaging.go @@ -20,6 +20,7 @@ import ( const ( LOGFILE = "hgs-msg.log" // circular import problem; need to edit "vv.constants.go" too if changing this + UNCOLORED = -9 MSGMAND = -1 MSGCRIT = 0 MSGWARN = 1 @@ -114,6 +115,11 @@ func (m *MessageMaker) TMI(s string) { m.Emit(s, MSGTMI) } +// AlwaysEmit - - send a message to the terminal or to the logfile; ignore threshold check +func (m *MessageMaker) AlwaysEmit(s string) { + m.Emit(s, UNCOLORED) +} + // Emit - send a message to the terminal or to the logfile, perhaps adding color and style to it func (m *MessageMaker) Emit(message string, threshold int) { // sample output: "[HGS] findbyform() found no results for 'Romani'" @@ -157,6 +163,8 @@ func (m *MessageMaker) ColorMessage(message string, threshold int) string { var color string switch threshold { + case UNCOLORED: + // leave color as "" case MSGMAND: color = GREEN case MSGCRIT: diff --git a/internal/db/getmorph.go b/internal/db/getmorph.go index cdcfcae..87a0553 100644 --- a/internal/db/getmorph.go +++ b/internal/db/getmorph.go @@ -8,11 +8,11 @@ package db import ( "context" "fmt" - "github.com/e-gun/HipparchiaGoServer/internal/base/gen" "github.com/e-gun/HipparchiaGoServer/internal/base/str" "github.com/e-gun/HipparchiaGoServer/internal/vv" "github.com/google/uuid" "github.com/jackc/pgx/v5" + "slices" "strings" ) @@ -57,7 +57,7 @@ func ArrayToGetRequiredMorphObjects(wordlist []string) map[string]str.DbMorpholo QT = `SELECT observed_form, xrefs, prefixrefs, possible_dictionary_forms, related_headwords FROM %s_morphology WHERE EXISTS (SELECT 1 FROM ttw_%s temptable WHERE temptable.w = %s_morphology.observed_form)` MSG1 = "ArrayToGetRequiredMorphObjects() will search among %d words" - CHUNKSIZE = 999999 + CHUNKSIZE = 500000 ) dbconn := getdbconnection() @@ -99,8 +99,7 @@ func ArrayToGetRequiredMorphObjects(wordlist []string) map[string]str.DbMorpholo // this could be parallelized... - chunkedlist := gen.ChunkSlice(wordlist, CHUNKSIZE) - for _, cl := range chunkedlist { + for cl := range slices.Chunk(wordlist, CHUNKSIZE) { // a waste of time to check the language on every word; just flail/fail once for _, uselang := range vv.TheLanguages { u := strings.Replace(uuid.New().String(), "-", "", -1) diff --git a/internal/search/resultformatting.go b/internal/search/resultformatting.go index 659ecb1..c024ab7 100644 --- a/internal/search/resultformatting.go +++ b/internal/search/resultformatting.go @@ -218,7 +218,7 @@ func FormatWithContextResults(thesearch *str.SearchStruct) str.SearchOutputJSON SSBuildQueries(&ctxsearch) SearchAndInsertResults(&ctxsearch) - // now you have all the lines you will ever need + // now you have all the lines you will ever need; map those lines for subsequent lookup linemap := make(map[string]str.DbWorkline) rr = ctxsearch.Results.Yield() @@ -266,7 +266,7 @@ func FormatWithContextResults(thesearch *str.SearchStruct) str.SearchOutputJSON kk++ } - // fix the unmattched spans + // fix the unmatched spans for _, p := range allpassages { // at the top p.CookedCTX[0].Contents = unbalancedspancleaner(p.CookedCTX[0].Contents) diff --git a/internal/vec/vectorgraphovverrides.go b/internal/vec/vectorgraphovverrides.go index 461022d..12fca3c 100644 --- a/internal/vec/vectorgraphovverrides.go +++ b/internal/vec/vectorgraphovverrides.go @@ -17,7 +17,7 @@ import ( // // this is kludgy and likely to be a source of ongoing pain: v2.4.0 broke the api of v2.3.3 -// any/all of the following could break again; the code are scattered across the go-echarts repo too +// any/all of the following could break again; the code is scattered across the go-echarts repo too // 9cde6a4b was the last time HipparchiaGoServer used v2.3.3 // the material below used to be provided by the bottom of vectorgraphing.go diff --git a/internal/vlt/wsmessage.go b/internal/vlt/wsmessage.go index 8367ae7..fe4a6d8 100644 --- a/internal/vlt/wsmessage.go +++ b/internal/vlt/wsmessage.go @@ -158,7 +158,7 @@ func (c *WSClient) WSMessageLoop() { func (pool *wspool) WSPoolStartListening() { const ( MSG1 = "Starting polling loop for %s" - MSG2 = "wspool client failed on WriteMessage()" + MSG2 = "WSPoolStartListening(): wspool client %s failed on WriteMessage()" ) writemsg := func(jso *wsjsout) { @@ -168,7 +168,7 @@ func (pool *wspool) WSPoolStartListening() { Msg.EC(y) e := cl.Conn.WriteMessage(websocket.TextMessage, js) if e != nil { - Msg.WARN(MSG2) + Msg.WARN(fmt.Sprintf(MSG2, jso.ID)) delete(pool.ClientMap, cl) } } diff --git a/web/echoserver.go b/web/echoserver.go index ef5685b..96e2696 100644 --- a/web/echoserver.go +++ b/web/echoserver.go @@ -19,7 +19,6 @@ import ( "net/http" "os" "strings" - "time" ) var ( @@ -30,7 +29,7 @@ var ( func StartEchoServer() { const ( LLOGFMT = "r: ${status}\tt: ${latency_human}\tu: ${uri}\n" - RLOGFMT = "${remote_ip}\t${custom}\t${status}\t${bytes_out}\t${uri}\n" + RLOGFMT = "${time_rfc3339}\t${remote_ip}\t${custom}\t${status}\t${bytes_out}\t${uri}\n" ) e := echo.New() @@ -243,15 +242,8 @@ func configureecho(e *echo.Echo) { // assume that anyone who is using authentication is serving via the internet and so set timeouts e.Server.ReadTimeout = vv.TIMEOUTRD e.Server.WriteTimeout = vv.TIMEOUTWR - // also assume that internet exposure yields scanning attempts that will spam 404s & 500s; block IPs that do this - e.Use(pr.PoliceRequestAndResponse) - go pr.ResponseStatsKeeper() - go pr.IPBlacklistKeeper() - pr.Emit.ColorOn() - if lnch.Config.TickerActive || lnch.Config.LogToFile { - pr.Emit.E = emittofile - } + policing(e) } e.Use(middleware.RateLimiter(middleware.NewRateLimiterMemoryStore(vv.MAXECHOREQPERSECONDPERIP))) @@ -263,6 +255,20 @@ func configureecho(e *echo.Echo) { } } +func policing(e *echo.Echo) { + e.Use(pr.PoliceRequestAndResponse) + go pr.ResponseStatsKeeper() + go pr.IPBlacklistKeeper() + if !lnch.Config.BlackAndWhite { + pr.Emit.ColorOn() + } + if lnch.Config.TickerActive || lnch.Config.LogToFile { + pr.Emit.E = Msg.EmitToFile + } else { + pr.Emit.E = Msg.AlwaysEmit + } +} + func candossl() bool { ok1 := false if _, err := os.Stat(lnch.Config.SSLCertDir + vv.SSLCPEM); err == nil { @@ -292,19 +298,3 @@ func starttlsserver(e *echo.Echo) { log.Fatal(err) } } - -// emittofile - send the message to a file -func emittofile(message string) { - tn := time.Now().Format(time.RFC850) - ms := fmt.Sprintf("[%s] %s\n", tn, message) - uh, _ := os.UserHomeDir() - f, err := os.OpenFile(uh+"/"+vv.LOGFILEML, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - panic(err) - } - defer f.Close() - if _, err = f.WriteString(ms); err != nil { - panic(err) - } - return -}