feat: Update Fiber version to v2 and update internal controller and search logic to use new index module
This commit is contained in:
parent
8746f789c4
commit
ed201de318
14
go.mod
14
go.mod
@ -2,21 +2,15 @@ module git.dev-null.rocks/alexohneander/gosearch
|
||||
|
||||
go 1.23.5
|
||||
|
||||
require github.com/gofiber/fiber/v2 v2.52.6
|
||||
|
||||
require (
|
||||
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
|
||||
github.com/gofiber/schema v1.2.0 // indirect
|
||||
github.com/gofiber/utils/v2 v2.0.0-beta.7 // indirect
|
||||
github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect
|
||||
github.com/tinylib/msgp v1.2.5 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
golang.org/x/crypto v0.31.0 // indirect
|
||||
golang.org/x/net v0.31.0 // indirect
|
||||
golang.org/x/text v0.21.0 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/rivo/uniseg v0.2.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.1.1 // indirect
|
||||
github.com/gofiber/fiber/v3 v3.0.0-beta.4
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.17.11 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
|
||||
34
go.sum
34
go.sum
@ -1,15 +1,7 @@
|
||||
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
|
||||
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
|
||||
github.com/gofiber/fiber/v3 v3.0.0-beta.4 h1:KzDSavvhG7m81NIsmnu5l3ZDbVS4feCidl4xlIfu6V0=
|
||||
github.com/gofiber/fiber/v3 v3.0.0-beta.4/go.mod h1:/WFUoHRkZEsGHyy2+fYcdqi109IVOFbVwxv1n1RU+kk=
|
||||
github.com/gofiber/schema v1.2.0 h1:j+ZRrNnUa/0ZuWrn/6kAtAufEr4jCJ+JuTURAMxNSZg=
|
||||
github.com/gofiber/schema v1.2.0/go.mod h1:YYwj01w3hVfaNjhtJzaqetymL56VW642YS3qZPhuE6c=
|
||||
github.com/gofiber/utils/v2 v2.0.0-beta.7 h1:NnHFrRHvhrufPABdWajcKZejz9HnCWmT/asoxRsiEbQ=
|
||||
github.com/gofiber/utils/v2 v2.0.0-beta.7/go.mod h1:J/M03s+HMdZdvhAeyh76xT72IfVqBzuz/OJkrMa7cwU=
|
||||
github.com/gofiber/fiber/v2 v2.52.6 h1:Rfp+ILPiYSvvVuIPvxrBns+HJp8qGLDnLJawAu27XVI=
|
||||
github.com/gofiber/fiber/v2 v2.52.6/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
@ -19,33 +11,19 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY=
|
||||
github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tinylib/msgp v1.2.5 h1:WeQg1whrXRFiZusidTQqzETkRpGjFjcIhW6uqWH09po=
|
||||
github.com/tinylib/msgp v1.2.5/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.58.0 h1:GGB2dWxSbEprU9j0iMJHgdKYJVDyjrOwF9RE59PbRuE=
|
||||
github.com/valyala/fasthttp v1.58.0/go.mod h1:SYXvHHaFp7QZHGKSHmoMipInhrI5StHrhDTYVEjK/Kw=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
|
||||
golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@ -1,9 +1,23 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"github.com/gofiber/fiber/v3"
|
||||
"git.dev-null.rocks/alexohneander/gosearch/pkg/index"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
)
|
||||
|
||||
func Index(c fiber.Ctx) error {
|
||||
return c.SendString("Hello, World!")
|
||||
type Document struct {
|
||||
Url string `json:"url" xml:"url" form:"url"`
|
||||
Content string `json:"content" xml:"content" form:"content"`
|
||||
}
|
||||
|
||||
func AddDocumentToIndex(c *fiber.Ctx) error {
|
||||
doc := new(Document)
|
||||
|
||||
if err := c.BodyParser(doc); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
index.AddDocToIndex(doc.Url, doc.Content)
|
||||
|
||||
return c.SendString("Document added")
|
||||
}
|
||||
|
||||
@ -5,16 +5,17 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.dev-null.rocks/alexohneander/gosearch/pkg/index"
|
||||
"git.dev-null.rocks/alexohneander/gosearch/pkg/search"
|
||||
"github.com/gofiber/fiber/v3"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
)
|
||||
|
||||
func SearchQuery(c fiber.Ctx) error {
|
||||
func SearchQuery(c *fiber.Ctx) error {
|
||||
query := c.Params("query")
|
||||
query = strings.TrimSpace(query)
|
||||
|
||||
terms, queryType := parseQuery(query)
|
||||
results := search.Search(terms, queryType, search.Index, search.DocFreq, len(search.Files))
|
||||
results := search.Search(terms, queryType, index.Index, index.DocFreq, len(index.Documents))
|
||||
|
||||
var response string
|
||||
|
||||
|
||||
@ -2,24 +2,23 @@ package http
|
||||
|
||||
import (
|
||||
"git.dev-null.rocks/alexohneander/gosearch/internal/controller"
|
||||
"github.com/gofiber/fiber/v3"
|
||||
"github.com/gofiber/fiber/v3/middleware/healthcheck"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/monitor"
|
||||
)
|
||||
|
||||
func configureRoutes(app *fiber.App) *fiber.App {
|
||||
// Index
|
||||
app.Get("/", controller.Index)
|
||||
app.Get("/test", controller.Index)
|
||||
app.Post("/api/index", controller.AddDocumentToIndex)
|
||||
|
||||
// Search
|
||||
app.Get("/api/search/:query", controller.SearchQuery)
|
||||
|
||||
// Monitor
|
||||
// app.Get("/metrics", monitor.New(monitor.Config{Title: "MyService Metrics Page"}))
|
||||
app.Get("/metrics", monitor.New(monitor.Config{Title: "gosearch Metrics"}))
|
||||
|
||||
// Health Checks
|
||||
app.Get(healthcheck.DefaultLivenessEndpoint, healthcheck.NewHealthChecker())
|
||||
app.Get(healthcheck.DefaultReadinessEndpoint, healthcheck.NewHealthChecker())
|
||||
app.Get(healthcheck.DefaultStartupEndpoint, healthcheck.NewHealthChecker())
|
||||
// app.Get(healthcheck.DefaultLivenessEndpoint, healthcheck.NewHealthChecker())
|
||||
// app.Get(healthcheck.DefaultReadinessEndpoint, healthcheck.NewHealthChecker())
|
||||
// app.Get(healthcheck.DefaultStartupEndpoint, healthcheck.NewHealthChecker())
|
||||
return app
|
||||
}
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"github.com/gofiber/fiber/v3"
|
||||
"github.com/gofiber/fiber/v3/middleware/logger"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
)
|
||||
|
||||
func StartService() {
|
||||
|
||||
4
main.go
4
main.go
@ -2,11 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"git.dev-null.rocks/alexohneander/gosearch/internal/http"
|
||||
"git.dev-null.rocks/alexohneander/gosearch/pkg/search"
|
||||
"git.dev-null.rocks/alexohneander/gosearch/pkg/index"
|
||||
)
|
||||
|
||||
func main() {
|
||||
search.TestIndex()
|
||||
index.TestIndex()
|
||||
|
||||
// Start HTTP Server
|
||||
http.StartService()
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package search
|
||||
package index
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
@ -13,7 +13,7 @@ type DocumentFrequency map[string]int
|
||||
|
||||
var Index InvertedIndex
|
||||
var DocFreq DocumentFrequency
|
||||
var Files []string
|
||||
var Documents []string
|
||||
|
||||
// BuildIndex reads files and builds an inverted index.
|
||||
func BuildIndex(files []string) (InvertedIndex, DocumentFrequency, error) {
|
||||
@ -57,7 +57,42 @@ func TestIndex() {
|
||||
log.Fatalf("Error building index: %v", err)
|
||||
}
|
||||
|
||||
Files = files
|
||||
Documents = files
|
||||
Index = index
|
||||
DocFreq = docFreq
|
||||
}
|
||||
|
||||
func CreateIndex() {
|
||||
index := make(InvertedIndex)
|
||||
docFreq := make(DocumentFrequency)
|
||||
var docs []string
|
||||
|
||||
Documents = docs
|
||||
Index = index
|
||||
DocFreq = docFreq
|
||||
}
|
||||
|
||||
func AddDocToIndex(url string, content string) {
|
||||
Documents = append(Documents, url)
|
||||
|
||||
reader := strings.NewReader(content)
|
||||
|
||||
seenTerms := make(map[string]bool) // Track terms in this document
|
||||
scanner := bufio.NewScanner(reader)
|
||||
scanner.Split(bufio.ScanWords)
|
||||
|
||||
for scanner.Scan() {
|
||||
word := strings.ToLower(strings.Trim(scanner.Text(), ",.!?"))
|
||||
|
||||
if Index[word] == nil {
|
||||
Index[word] = make(map[string]int)
|
||||
}
|
||||
Index[word][url]++
|
||||
|
||||
if !seenTerms[word] {
|
||||
DocFreq[word]++
|
||||
seenTerms[word] = true
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -3,6 +3,8 @@ package search
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
|
||||
"git.dev-null.rocks/alexohneander/gosearch/pkg/index"
|
||||
)
|
||||
|
||||
// SearchResult stores the document and its relevance score.
|
||||
@ -12,7 +14,7 @@ type SearchResult struct {
|
||||
}
|
||||
|
||||
// Search processes different types of queries using TF-IDF scoring.
|
||||
func Search(terms []string, queryType string, index InvertedIndex, docFreq DocumentFrequency, numDocs int) []SearchResult {
|
||||
func Search(terms []string, queryType string, index index.InvertedIndex, docFreq index.DocumentFrequency, numDocs int) []SearchResult {
|
||||
scores := make(map[string]float64)
|
||||
|
||||
if queryType == "AND" {
|
||||
@ -40,7 +42,7 @@ func Search(terms []string, queryType string, index InvertedIndex, docFreq Docum
|
||||
}
|
||||
|
||||
// Helper function to score a single document based on terms
|
||||
func scoreDoc(terms []string, doc string, index InvertedIndex, docFreq DocumentFrequency, numDocs int) float64 {
|
||||
func scoreDoc(terms []string, doc string, index index.InvertedIndex, docFreq index.DocumentFrequency, numDocs int) float64 {
|
||||
score := 0.0
|
||||
for _, term := range terms {
|
||||
tf := float64(index[term][doc])
|
||||
@ -52,7 +54,7 @@ func scoreDoc(terms []string, doc string, index InvertedIndex, docFreq DocumentF
|
||||
}
|
||||
|
||||
// Helper function to intersect documents for AND logic
|
||||
func intersectDocs(terms []string, index InvertedIndex) []string {
|
||||
func intersectDocs(terms []string, index index.InvertedIndex) []string {
|
||||
if len(terms) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user