back to school tidyups

go.mod

@@ -1,10 +1,11 @@
 module github.com/e-gun/HipparchiaGoServer
 
-go 1.22
+go 1.23
 
 require (
 	github.com/e-gun/lastnlines v0.1.0
 	github.com/e-gun/nlp v0.0.2
+	github.com/e-gun/policeresponses v0.0.2
 	github.com/e-gun/tsnemp v0.1.7
 	github.com/e-gun/wego v0.1.0
 	github.com/go-echarts/go-echarts/v2 v2.4.1 // v2.4 breaks the api
@@ -13,20 +14,19 @@ require (
 	github.com/jackc/pgx/v5 v5.6.0
 	github.com/labstack/echo/v4 v4.12.0
 	github.com/pkg/profile v1.7.0
-	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
-	golang.org/x/text v0.16.0
-	gonum.org/v1/gonum v0.15.0
+	golang.org/x/exp v0.0.0-20240823005443-9b4947da3948
+	golang.org/x/text v0.17.0
+	gonum.org/v1/gonum v0.15.1
 )
 
 require (
 	github.com/armon/go-radix v1.0.0 // indirect
-	github.com/e-gun/policeresponses v0.0.1 // indirect
 	github.com/e-gun/safestack v0.1.1 // indirect
 	github.com/e-gun/sparse v0.0.0-20230418220937-07063da15582 // indirect
-	github.com/felixge/fgprof v0.9.4 // indirect
+	github.com/felixge/fgprof v0.9.5 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
-	github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
+	github.com/google/pprof v0.0.0-20240829160300-da1f7e9f2b25 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
@@ -44,10 +44,10 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
-	golang.org/x/crypto v0.25.0 // indirect
-	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/crypto v0.26.0 // indirect
+	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.23.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
 	golang.org/x/time v0.6.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 )

internal/base/gen/setsandslices.go

@@ -39,9 +39,16 @@ func Unique[T comparable](s []T) []T {
 
 	set := ToSet(s)
 
-	var result []T
+	//var result []T
+	//for k := range set {
+	//	result = append(result, k)
+	//}
+
+	result := make([]T, len(set))
+	count := 0
 	for k := range set {
-		result = append(result, k)
+		result[count] = k
+		count += 1
 	}
 
 	return result
@@ -96,11 +103,3 @@ func StringMapKeysIntoSlice[T any](mp map[string]T) []string {
 	}
 	return sl
 }
-
-// ChunkSlice - turn a slice into a slice of slices of size N; thanks to https://stackoverflow.com/questions/35179656/slice-chunking-in-go
-func ChunkSlice[T any](items []T, size int) (chunks [][]T) {
-	for size < len(items) {
-		items, chunks = items[size:], append(chunks, items[0:size:size])
-	}
-	return append(chunks, items)
-}

internal/base/mm/messaging.go

@@ -20,6 +20,7 @@ import (
 
 const (
 	LOGFILE              = "hgs-msg.log" // circular import problem; need to edit "vv.constants.go" too if changing this
+	UNCOLORED            = -9
 	MSGMAND              = -1
 	MSGCRIT              = 0
 	MSGWARN              = 1
@@ -114,6 +115,11 @@ func (m *MessageMaker) TMI(s string) {
 	m.Emit(s, MSGTMI)
 }
 
+// AlwaysEmit - - send a message to the terminal or to the logfile; ignore threshold check
+func (m *MessageMaker) AlwaysEmit(s string) {
+	m.Emit(s, UNCOLORED)
+}
+
 // Emit - send a message to the terminal or to the logfile, perhaps adding color and style to it
 func (m *MessageMaker) Emit(message string, threshold int) {
 	// sample output: "[HGS] findbyform() found no results for 'Romani'"
@@ -157,6 +163,8 @@ func (m *MessageMaker) ColorMessage(message string, threshold int) string {
 	var color string
 
 	switch threshold {
+	case UNCOLORED:
+		// leave color as ""
 	case MSGMAND:
 		color = GREEN
 	case MSGCRIT:

internal/db/getmorph.go

@@ -8,11 +8,11 @@ package db
 import (
 	"context"
 	"fmt"
-	"github.com/e-gun/HipparchiaGoServer/internal/base/gen"
 	"github.com/e-gun/HipparchiaGoServer/internal/base/str"
 	"github.com/e-gun/HipparchiaGoServer/internal/vv"
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5"
+	"slices"
 	"strings"
 )
 
@@ -57,7 +57,7 @@ func ArrayToGetRequiredMorphObjects(wordlist []string) map[string]str.DbMorpholo
 		QT = `SELECT observed_form, xrefs, prefixrefs, possible_dictionary_forms, related_headwords FROM %s_morphology WHERE EXISTS 
 				(SELECT 1 FROM ttw_%s temptable WHERE temptable.w = %s_morphology.observed_form)`
 		MSG1      = "ArrayToGetRequiredMorphObjects() will search among %d words"
-		CHUNKSIZE = 999999
+		CHUNKSIZE = 500000
 	)
 
 	dbconn := getdbconnection()
@@ -99,8 +99,7 @@ func ArrayToGetRequiredMorphObjects(wordlist []string) map[string]str.DbMorpholo
 
 	// this could be parallelized...
 
-	chunkedlist := gen.ChunkSlice(wordlist, CHUNKSIZE)
-	for _, cl := range chunkedlist {
+	for cl := range slices.Chunk(wordlist, CHUNKSIZE) {
 		// a waste of time to check the language on every word; just flail/fail once
 		for _, uselang := range vv.TheLanguages {
 			u := strings.Replace(uuid.New().String(), "-", "", -1)

internal/search/resultformatting.go

@@ -218,7 +218,7 @@ func FormatWithContextResults(thesearch *str.SearchStruct) str.SearchOutputJSON
 	SSBuildQueries(&ctxsearch)
 	SearchAndInsertResults(&ctxsearch)
 
-	// now you have all the lines you will ever need
+	// now you have all the lines you will ever need; map those lines for subsequent lookup
 	linemap := make(map[string]str.DbWorkline)
 
 	rr = ctxsearch.Results.Yield()
@@ -266,7 +266,7 @@ func FormatWithContextResults(thesearch *str.SearchStruct) str.SearchOutputJSON
 		kk++
 	}
 
-	// fix the unmattched spans
+	// fix the unmatched spans
 	for _, p := range allpassages {
 		// at the top
 		p.CookedCTX[0].Contents = unbalancedspancleaner(p.CookedCTX[0].Contents)

internal/vec/vectorgraphovverrides.go

@@ -17,7 +17,7 @@ import (
 //
 
 // this is kludgy and likely to be a source of ongoing pain: v2.4.0 broke the api of v2.3.3
-// any/all of the following could break again; the code are scattered across the go-echarts repo too
+// any/all of the following could break again; the code is scattered across the go-echarts repo too
 
 // 9cde6a4b was the last time HipparchiaGoServer used v2.3.3
 // the material below used to be provided by the bottom of vectorgraphing.go

internal/vlt/wsmessage.go

@@ -158,7 +158,7 @@ func (c *WSClient) WSMessageLoop() {
 func (pool *wspool) WSPoolStartListening() {
 	const (
 		MSG1 = "Starting polling loop for %s"
-		MSG2 = "wspool client failed on WriteMessage()"
+		MSG2 = "WSPoolStartListening(): wspool client %s failed on WriteMessage()"
 	)
 
 	writemsg := func(jso *wsjsout) {
@@ -168,7 +168,7 @@ func (pool *wspool) WSPoolStartListening() {
 				Msg.EC(y)
 				e := cl.Conn.WriteMessage(websocket.TextMessage, js)
 				if e != nil {
-					Msg.WARN(MSG2)
+					Msg.WARN(fmt.Sprintf(MSG2, jso.ID))
 					delete(pool.ClientMap, cl)
 				}
 			}

web/echoserver.go

@@ -19,7 +19,6 @@ import (
 	"net/http"
 	"os"
 	"strings"
-	"time"
 )
 
 var (
@@ -30,7 +29,7 @@ var (
 func StartEchoServer() {
 	const (
 		LLOGFMT = "r: ${status}\tt: ${latency_human}\tu: ${uri}\n"
-		RLOGFMT = "${remote_ip}\t${custom}\t${status}\t${bytes_out}\t${uri}\n"
+		RLOGFMT = "${time_rfc3339}\t${remote_ip}\t${custom}\t${status}\t${bytes_out}\t${uri}\n"
 	)
 
 	e := echo.New()
@@ -243,15 +242,8 @@ func configureecho(e *echo.Echo) {
 		// assume that anyone who is using authentication is serving via the internet and so set timeouts
 		e.Server.ReadTimeout = vv.TIMEOUTRD
 		e.Server.WriteTimeout = vv.TIMEOUTWR
-
 		// also assume that internet exposure yields scanning attempts that will spam 404s & 500s; block IPs that do this
-		e.Use(pr.PoliceRequestAndResponse)
-		go pr.ResponseStatsKeeper()
-		go pr.IPBlacklistKeeper()
-		pr.Emit.ColorOn()
-		if lnch.Config.TickerActive || lnch.Config.LogToFile {
-			pr.Emit.E = emittofile
-		}
+		policing(e)
 	}
 
 	e.Use(middleware.RateLimiter(middleware.NewRateLimiterMemoryStore(vv.MAXECHOREQPERSECONDPERIP)))
@@ -263,6 +255,20 @@ func configureecho(e *echo.Echo) {
 	}
 }
 
+func policing(e *echo.Echo) {
+	e.Use(pr.PoliceRequestAndResponse)
+	go pr.ResponseStatsKeeper()
+	go pr.IPBlacklistKeeper()
+	if !lnch.Config.BlackAndWhite {
+		pr.Emit.ColorOn()
+	}
+	if lnch.Config.TickerActive || lnch.Config.LogToFile {
+		pr.Emit.E = Msg.EmitToFile
+	} else {
+		pr.Emit.E = Msg.AlwaysEmit
+	}
+}
+
 func candossl() bool {
 	ok1 := false
 	if _, err := os.Stat(lnch.Config.SSLCertDir + vv.SSLCPEM); err == nil {
@@ -292,19 +298,3 @@ func starttlsserver(e *echo.Echo) {
 		log.Fatal(err)
 	}
 }
-
-// emittofile - send the message to a file
-func emittofile(message string) {
-	tn := time.Now().Format(time.RFC850)
-	ms := fmt.Sprintf("[%s] %s\n", tn, message)
-	uh, _ := os.UserHomeDir()
-	f, err := os.OpenFile(uh+"/"+vv.LOGFILEML, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
-	if err != nil {
-		panic(err)
-	}
-	defer f.Close()
-	if _, err = f.WriteString(ms); err != nil {
-		panic(err)
-	}
-	return
-}