From ed201de318cac47831df126dee3f9061ea56f0f8 Mon Sep 17 00:00:00 2001 From: Alex Wellnitz Date: Tue, 25 Mar 2025 21:09:01 +0100 Subject: [PATCH] feat: Update Fiber version to v2 and update internal controller and search logic to use new index module --- go.mod | 14 ++++-------- go.sum | 34 +++++----------------------- internal/controller/index.go | 20 ++++++++++++++--- internal/controller/search.go | 7 +++--- internal/http/router.go | 15 ++++++------- internal/http/server.go | 4 ++-- main.go | 4 ++-- pkg/{search => index}/index.go | 41 +++++++++++++++++++++++++++++++--- pkg/search/search.go | 8 ++++--- 9 files changed, 85 insertions(+), 62 deletions(-) rename pkg/{search => index}/index.go (62%) diff --git a/go.mod b/go.mod index 99e5614..ece8e80 100644 --- a/go.mod +++ b/go.mod @@ -2,21 +2,15 @@ module git.dev-null.rocks/alexohneander/gosearch go 1.23.5 +require github.com/gofiber/fiber/v2 v2.52.6 + require ( - github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/gofiber/schema v1.2.0 // indirect - github.com/gofiber/utils/v2 v2.0.0-beta.7 // indirect - github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect - github.com/tinylib/msgp v1.2.5 // indirect - github.com/x448/float16 v0.8.4 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/net v0.31.0 // indirect - golang.org/x/text v0.21.0 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/rivo/uniseg v0.2.0 // indirect ) require ( github.com/andybalholm/brotli v1.1.1 // indirect - github.com/gofiber/fiber/v3 v3.0.0-beta.4 github.com/google/uuid v1.6.0 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/go.sum b/go.sum index 93a18fd..e07755e 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,7 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= -github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/gofiber/fiber/v3 v3.0.0-beta.4 h1:KzDSavvhG7m81NIsmnu5l3ZDbVS4feCidl4xlIfu6V0= -github.com/gofiber/fiber/v3 v3.0.0-beta.4/go.mod h1:/WFUoHRkZEsGHyy2+fYcdqi109IVOFbVwxv1n1RU+kk= -github.com/gofiber/schema v1.2.0 h1:j+ZRrNnUa/0ZuWrn/6kAtAufEr4jCJ+JuTURAMxNSZg= -github.com/gofiber/schema v1.2.0/go.mod h1:YYwj01w3hVfaNjhtJzaqetymL56VW642YS3qZPhuE6c= -github.com/gofiber/utils/v2 v2.0.0-beta.7 h1:NnHFrRHvhrufPABdWajcKZejz9HnCWmT/asoxRsiEbQ= -github.com/gofiber/utils/v2 v2.0.0-beta.7/go.mod h1:J/M03s+HMdZdvhAeyh76xT72IfVqBzuz/OJkrMa7cwU= +github.com/gofiber/fiber/v2 v2.52.6 h1:Rfp+ILPiYSvvVuIPvxrBns+HJp8qGLDnLJawAu27XVI= +github.com/gofiber/fiber/v2 v2.52.6/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= @@ -19,33 +11,19 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= -github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tinylib/msgp v1.2.5 h1:WeQg1whrXRFiZusidTQqzETkRpGjFjcIhW6uqWH09po= -github.com/tinylib/msgp v1.2.5/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.58.0 h1:GGB2dWxSbEprU9j0iMJHgdKYJVDyjrOwF9RE59PbRuE= github.com/valyala/fasthttp v1.58.0/go.mod h1:SYXvHHaFp7QZHGKSHmoMipInhrI5StHrhDTYVEjK/Kw= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/controller/index.go b/internal/controller/index.go index e1e7f3d..4b5ef4e 100644 --- a/internal/controller/index.go +++ b/internal/controller/index.go @@ -1,9 +1,23 @@ package controller import ( - "github.com/gofiber/fiber/v3" + "git.dev-null.rocks/alexohneander/gosearch/pkg/index" + "github.com/gofiber/fiber/v2" ) -func Index(c fiber.Ctx) error { - return c.SendString("Hello, World!") +type Document struct { + Url string `json:"url" xml:"url" form:"url"` + Content string `json:"content" xml:"content" form:"content"` +} + +func AddDocumentToIndex(c *fiber.Ctx) error { + doc := new(Document) + + if err := c.BodyParser(doc); err != nil { + return err + } + + index.AddDocToIndex(doc.Url, doc.Content) + + return c.SendString("Document added") } diff --git a/internal/controller/search.go b/internal/controller/search.go index a738db7..257b464 100644 --- a/internal/controller/search.go +++ b/internal/controller/search.go @@ -5,16 +5,17 @@ import ( "os" "strings" + "git.dev-null.rocks/alexohneander/gosearch/pkg/index" "git.dev-null.rocks/alexohneander/gosearch/pkg/search" - "github.com/gofiber/fiber/v3" + "github.com/gofiber/fiber/v2" ) -func SearchQuery(c fiber.Ctx) error { +func SearchQuery(c *fiber.Ctx) error { query := c.Params("query") query = strings.TrimSpace(query) terms, queryType := parseQuery(query) - results := search.Search(terms, queryType, search.Index, search.DocFreq, len(search.Files)) + results := search.Search(terms, queryType, index.Index, index.DocFreq, len(index.Documents)) var response string diff --git a/internal/http/router.go b/internal/http/router.go index be3b67b..1d72847 100644 --- a/internal/http/router.go +++ b/internal/http/router.go @@ -2,24 +2,23 @@ package http import ( "git.dev-null.rocks/alexohneander/gosearch/internal/controller" - "github.com/gofiber/fiber/v3" - "github.com/gofiber/fiber/v3/middleware/healthcheck" + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/monitor" ) func configureRoutes(app *fiber.App) *fiber.App { // Index - app.Get("/", controller.Index) - app.Get("/test", controller.Index) + app.Post("/api/index", controller.AddDocumentToIndex) // Search app.Get("/api/search/:query", controller.SearchQuery) // Monitor - // app.Get("/metrics", monitor.New(monitor.Config{Title: "MyService Metrics Page"})) + app.Get("/metrics", monitor.New(monitor.Config{Title: "gosearch Metrics"})) // Health Checks - app.Get(healthcheck.DefaultLivenessEndpoint, healthcheck.NewHealthChecker()) - app.Get(healthcheck.DefaultReadinessEndpoint, healthcheck.NewHealthChecker()) - app.Get(healthcheck.DefaultStartupEndpoint, healthcheck.NewHealthChecker()) + // app.Get(healthcheck.DefaultLivenessEndpoint, healthcheck.NewHealthChecker()) + // app.Get(healthcheck.DefaultReadinessEndpoint, healthcheck.NewHealthChecker()) + // app.Get(healthcheck.DefaultStartupEndpoint, healthcheck.NewHealthChecker()) return app } diff --git a/internal/http/server.go b/internal/http/server.go index 338ff53..50057d0 100644 --- a/internal/http/server.go +++ b/internal/http/server.go @@ -1,8 +1,8 @@ package http import ( - "github.com/gofiber/fiber/v3" - "github.com/gofiber/fiber/v3/middleware/logger" + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/logger" ) func StartService() { diff --git a/main.go b/main.go index 04ca71f..0f39ab7 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,11 @@ package main import ( "git.dev-null.rocks/alexohneander/gosearch/internal/http" - "git.dev-null.rocks/alexohneander/gosearch/pkg/search" + "git.dev-null.rocks/alexohneander/gosearch/pkg/index" ) func main() { - search.TestIndex() + index.TestIndex() // Start HTTP Server http.StartService() diff --git a/pkg/search/index.go b/pkg/index/index.go similarity index 62% rename from pkg/search/index.go rename to pkg/index/index.go index 7c6dbb0..50ebfc4 100644 --- a/pkg/search/index.go +++ b/pkg/index/index.go @@ -1,4 +1,4 @@ -package search +package index import ( "bufio" @@ -13,7 +13,7 @@ type DocumentFrequency map[string]int var Index InvertedIndex var DocFreq DocumentFrequency -var Files []string +var Documents []string // BuildIndex reads files and builds an inverted index. func BuildIndex(files []string) (InvertedIndex, DocumentFrequency, error) { @@ -57,7 +57,42 @@ func TestIndex() { log.Fatalf("Error building index: %v", err) } - Files = files + Documents = files Index = index DocFreq = docFreq } + +func CreateIndex() { + index := make(InvertedIndex) + docFreq := make(DocumentFrequency) + var docs []string + + Documents = docs + Index = index + DocFreq = docFreq +} + +func AddDocToIndex(url string, content string) { + Documents = append(Documents, url) + + reader := strings.NewReader(content) + + seenTerms := make(map[string]bool) // Track terms in this document + scanner := bufio.NewScanner(reader) + scanner.Split(bufio.ScanWords) + + for scanner.Scan() { + word := strings.ToLower(strings.Trim(scanner.Text(), ",.!?")) + + if Index[word] == nil { + Index[word] = make(map[string]int) + } + Index[word][url]++ + + if !seenTerms[word] { + DocFreq[word]++ + seenTerms[word] = true + } + } + +} diff --git a/pkg/search/search.go b/pkg/search/search.go index 05b543f..9f457b0 100644 --- a/pkg/search/search.go +++ b/pkg/search/search.go @@ -3,6 +3,8 @@ package search import ( "math" "sort" + + "git.dev-null.rocks/alexohneander/gosearch/pkg/index" ) // SearchResult stores the document and its relevance score. @@ -12,7 +14,7 @@ type SearchResult struct { } // Search processes different types of queries using TF-IDF scoring. -func Search(terms []string, queryType string, index InvertedIndex, docFreq DocumentFrequency, numDocs int) []SearchResult { +func Search(terms []string, queryType string, index index.InvertedIndex, docFreq index.DocumentFrequency, numDocs int) []SearchResult { scores := make(map[string]float64) if queryType == "AND" { @@ -40,7 +42,7 @@ func Search(terms []string, queryType string, index InvertedIndex, docFreq Docum } // Helper function to score a single document based on terms -func scoreDoc(terms []string, doc string, index InvertedIndex, docFreq DocumentFrequency, numDocs int) float64 { +func scoreDoc(terms []string, doc string, index index.InvertedIndex, docFreq index.DocumentFrequency, numDocs int) float64 { score := 0.0 for _, term := range terms { tf := float64(index[term][doc]) @@ -52,7 +54,7 @@ func scoreDoc(terms []string, doc string, index InvertedIndex, docFreq DocumentF } // Helper function to intersect documents for AND logic -func intersectDocs(terms []string, index InvertedIndex) []string { +func intersectDocs(terms []string, index index.InvertedIndex) []string { if len(terms) == 0 { return nil }