From 62dc6809d575991677c9d3835a3b821749985e4f Mon Sep 17 00:00:00 2001 From: johanneskarrer Date: Mon, 15 Jul 2024 14:41:55 +0200 Subject: [PATCH 001/117] adds global search box for searching course slugs and names --- api/search.go | 40 ++++++++++++++++++++++++++++ dao/courses.go | 27 +++++++++++++++++++ tools/meiliExporter.go | 43 +++++++++++++++++++++++++++++++ web/template/home.gohtml | 31 ++++++++++++++++------ web/template/search-global.gohtml | 36 ++++++++++++++++++++++++++ web/ts/entry/home.ts | 1 + web/ts/search-courses.ts | 34 ++++++++++++++++++++++++ 7 files changed, 204 insertions(+), 8 deletions(-) create mode 100644 web/template/search-global.gohtml create mode 100644 web/ts/search-courses.ts diff --git a/api/search.go b/api/search.go index ca48e76e8..0ff6573ad 100644 --- a/api/search.go +++ b/api/search.go @@ -1,7 +1,10 @@ package api import ( + "context" + "github.com/TUM-Dev/gocast/model" "net/http" + "strconv" "github.com/TUM-Dev/gocast/dao" "github.com/TUM-Dev/gocast/tools" @@ -15,6 +18,8 @@ func configGinSearchRouter(router *gin.Engine, daoWrapper dao.DaoWrapper) { withStream := searchGroup.Group("/stream/:streamID") withStream.Use(tools.InitStream(daoWrapper)) withStream.GET("/subtitles", routes.searchSubtitles) + + searchGroup.GET("/courses", routes.searchCourses) } type searchRoutes struct { @@ -26,3 +31,38 @@ func (r searchRoutes) searchSubtitles(c *gin.Context) { q := c.Query("q") c.JSON(http.StatusOK, tools.SearchSubtitles(q, s.ID)) } + +func (r searchRoutes) searchCourses(c *gin.Context) { + user := c.MustGet("TUMLiveContext").(tools.TUMLiveContext).User + q := c.Query("q") + t := c.Query("term") + y, err := strconv.ParseInt(c.Query("year"), 10, 64) + if err != nil { + return + } + + var courses []model.Course + if user != nil { + switch user.Role { + case model.AdminType: + courses = r.GetAllCoursesForSemester(int(y), t, c) + default: // user.CoursesForSemesters includes both Administered Courses and enrolled Courses + courses, _ = r.CoursesDao.GetPublicAndLoggedInCourses(int(y), t) + courses = append(courses, user.CoursesForSemester(int(y), t, context.Background())...) + } + } else { + courses, _ = r.GetPublicCourses(int(y), t) + } + + distinctCourseIDs := make(map[uint]bool) + var courseIDs []uint + for _, course := range courses { + value := distinctCourseIDs[course.ID] + if !value { + courseIDs = append(courseIDs, course.ID) + distinctCourseIDs[course.ID] = true + } + } + + c.JSON(http.StatusOK, tools.SearchCourses(q, int(y), t, &courseIDs)) +} diff --git a/dao/courses.go b/dao/courses.go index 7a169f0c9..55de1b108 100644 --- a/dao/courses.go +++ b/dao/courses.go @@ -35,6 +35,7 @@ type CoursesDao interface { GetAvailableSemesters(c context.Context) []Semester GetCourseByShortLink(link string) (model.Course, error) GetCourseAdmins(courseID uint) ([]model.User, error) + ExecAllCourses(f func([]Course)) UpdateCourse(ctx context.Context, course model.Course) error UpdateCourseMetadata(ctx context.Context, course model.Course) @@ -281,6 +282,32 @@ func (d coursesDao) GetCourseAdmins(courseID uint) ([]model.User, error) { return admins, err } +type Course struct { + Name, Slug, TeachingTerm string + ID uint + Year int +} + +// ExecAllCourses executes f on all courses. +func (d coursesDao) ExecAllCourses(f func([]Course)) { + var res []Course + batchNum := 0 + batchSize := 100 + var numCourses int64 + DB.Model(&model.Course{}).Count(&numCourses) + for batchSize*batchNum < int(numCourses) { + err := DB.Raw(`SELECT id, name, slug, year, teaching_term, visibility + FROM courses + WHERE deleted_at IS NULL + LIMIT ? OFFSET ?`, batchSize, batchNum*batchSize).Scan(&res).Error + if err != nil { + fmt.Println(err) + } + f(res) + batchNum++ + } +} + func (d coursesDao) UpdateCourse(ctx context.Context, course model.Course) error { defer Cache.Clear() return DB.Session(&gorm.Session{FullSaveAssociations: true}).Updates(&course).Error diff --git a/tools/meiliExporter.go b/tools/meiliExporter.go index b3906fd93..237982548 100644 --- a/tools/meiliExporter.go +++ b/tools/meiliExporter.go @@ -29,6 +29,15 @@ type MeiliSubtitles struct { TextNext string `json:"textNext"` // the next subtitle line } +type MeiliCourse struct { + ID uint `json:"ID"` + Name string `json:"name"` + Slug string `json:"slug"` + Year int `json:"year"` + TeachingTerm string `json:"teachingTerm"` + Visibility string `json:"visibility"` +} + type MeiliExporter struct { c *meilisearch.Client d dao.DaoWrapper @@ -106,6 +115,31 @@ func (m *MeiliExporter) Export() { logger.Error("issue adding documents to meili", "err", err) } }) + + coursesIndex := m.c.Index("COURSES") + _, err = coursesIndex.DeleteAllDocuments() + if err != nil { + logger.Warn("could not delete all old courses", "err", err) + } + + m.d.CoursesDao.ExecAllCourses(func(courses []dao.Course) { + meilicourses := make([]MeiliCourse, len(courses)) + courseIDs := make([]uint, len(courses)) + for i, course := range courses { + courseIDs[i] = course.ID + meilicourses[i] = MeiliCourse{ + ID: course.ID, + Name: course.Name, + Slug: course.Slug, + Year: course.Year, + TeachingTerm: course.TeachingTerm, + } + } + _, err := coursesIndex.AddDocuments(&meilicourses, "ID") + if err != nil { + logger.Error("issue adding courses to meili", "err", err) + } + }) } func (m *MeiliExporter) SetIndexSettings() { @@ -130,4 +164,13 @@ func (m *MeiliExporter) SetIndexSettings() { if err != nil { logger.Warn("could not set settings for meili index SUBTITLES", "err", err) } + + _, err = m.c.Index("COURSES").UpdateSettings(&meilisearch.Settings{ + FilterableAttributes: []string{"ID", "visibility", "year", "teachingTerm"}, + SearchableAttributes: []string{"slug", "name"}, + SortableAttributes: []string{"year", "teachingTerm"}, + }) + if err != nil { + logger.Warn("could not set settings for meili index COURSES", "err", err) + } } diff --git a/web/template/home.gohtml b/web/template/home.gohtml index 9d38dade3..0313ab5c1 100644 --- a/web/template/home.gohtml +++ b/web/template/home.gohtml @@ -53,14 +53,29 @@ - + + + +{{template "search-global"}} + + + + + + + + + + + + + + + + + + +
{{template "notifications"}} diff --git a/web/template/search-global.gohtml b/web/template/search-global.gohtml new file mode 100644 index 000000000..b1493789d --- /dev/null +++ b/web/template/search-global.gohtml @@ -0,0 +1,36 @@ +{{define "search-global"}} + +{{end}} \ No newline at end of file diff --git a/web/ts/entry/home.ts b/web/ts/entry/home.ts index 9aacde30f..09bf7dd31 100644 --- a/web/ts/entry/home.ts +++ b/web/ts/entry/home.ts @@ -4,3 +4,4 @@ export * from "../components/livestreams"; export * from "../components/course"; export * from "../components/servernotifications"; export * from "../components/main"; +export * from "../search-courses"; diff --git a/web/ts/search-courses.ts b/web/ts/search-courses.ts new file mode 100644 index 000000000..4addb8d88 --- /dev/null +++ b/web/ts/search-courses.ts @@ -0,0 +1,34 @@ +export function coursesSearch() { + return { + hits: [], + open: false, + searchInput: "", + search: function (year: number, teachingTerm: string) { + if (this.searchInput.length > 2) { + fetch(`/api/search/courses?q=${this.searchInput}&year=${year}&term=${teachingTerm}`).then((res) => { + if (res.ok) { + res.json().then((data) => { + this.hits = data.hits; + this.open = true; + }); + } + }); + } else { + this.hits = []; + this.open = false; + } + }, + /*openRes: function () { + if (this.lastEventTimestamp + 1000 < Date.now()) { + this.lastEventTimestamp = Date.now(); + this.open = true; + } + }, + closeRes: function () { + if (this.lastEventTimestamp + 1000 < Date.now()) { + this.lastEventTimestamp = Date.now(); + this.open = false; + } + },*/ + }; +} \ No newline at end of file From 19217fd6405c4aa029b76f79032b4e75e8fc910b Mon Sep 17 00:00:00 2001 From: johanneskarrer Date: Mon, 15 Jul 2024 15:44:02 +0200 Subject: [PATCH 002/117] adds missing file for search box --- tools/meiliSearch.go | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tools/meiliSearch.go b/tools/meiliSearch.go index ae86717b3..dc8919c7e 100644 --- a/tools/meiliSearch.go +++ b/tools/meiliSearch.go @@ -2,8 +2,9 @@ package tools import ( "fmt" - "github.com/meilisearch/meilisearch-go" + "strconv" + "strings" ) func SearchSubtitles(q string, streamID uint) *meilisearch.SearchResponse { @@ -21,3 +22,36 @@ func SearchSubtitles(q string, streamID uint) *meilisearch.SearchResponse { } return response } + +func SearchCourses(q string, year int, t string, searchableCourseIDs *[]uint) *meilisearch.SearchResponse { + c, err := Cfg.GetMeiliClient() + if err != nil { + return nil + } + if t != "W" && t != "S" { + return nil + } + + courseIdFilter := courseIdFilter(searchableCourseIDs) + response, err := c.Index("COURSES").Search(q, &meilisearch.SearchRequest{ + Filter: fmt.Sprintf("year = %d AND teachingTerm = %s AND ID IN %s", year, t, courseIdFilter), + Limit: 10, + }) + + if err != nil { + logger.Error("could not search courses in meili", "err", err) + return nil + } + return response +} + +// returns a string conforming to Meili Search Filters Format containing each courseId passed onto the function +func courseIdFilter(searchableCourseIDs *[]uint) string { + var courseIDsAsStringArray []string + courseIDsAsStringArray = make([]string, len(*searchableCourseIDs)) + for i, courseID := range *searchableCourseIDs { + courseIDsAsStringArray[i] = strconv.FormatUint(uint64(courseID), 10) + } + courseIdFilter := "[" + strings.Join(courseIDsAsStringArray, ", ") + "]" + return courseIdFilter +} From 61519d452f3eed1e5171140ba559c0d32a6ab216 Mon Sep 17 00:00:00 2001 From: johanneskarrer Date: Tue, 13 Aug 2024 13:52:46 +0200 Subject: [PATCH 003/117] basic structure for searching in courses or semester(s) --- api/search.go | 169 +++++++++++++++++++++++++++++++++++++++-- model/user.go | 34 +++++++++ tools/meiliExporter.go | 9 ++- tools/meiliSearch.go | 40 +++++----- 4 files changed, 226 insertions(+), 26 deletions(-) diff --git a/api/search.go b/api/search.go index 0ff6573ad..afe6b0fef 100644 --- a/api/search.go +++ b/api/search.go @@ -2,13 +2,15 @@ package api import ( "context" - "github.com/TUM-Dev/gocast/model" - "net/http" - "strconv" - + "fmt" "github.com/TUM-Dev/gocast/dao" + "github.com/TUM-Dev/gocast/model" "github.com/TUM-Dev/gocast/tools" "github.com/gin-gonic/gin" + "net/http" + "regexp" + "strconv" + "strings" ) func configGinSearchRouter(router *gin.Engine, daoWrapper dao.DaoWrapper) { @@ -19,6 +21,10 @@ func configGinSearchRouter(router *gin.Engine, daoWrapper dao.DaoWrapper) { withStream.Use(tools.InitStream(daoWrapper)) withStream.GET("/subtitles", routes.searchSubtitles) + /*withCourse := searchGroup.Group("/course/:courseID") + withCourse.Use(tools.InitCourse(daoWrapper)) + //withCourse.GET("/streams", routes.searchStreams)*/ + searchGroup.GET("/courses", routes.searchCourses) } @@ -32,15 +38,165 @@ func (r searchRoutes) searchSubtitles(c *gin.Context) { c.JSON(http.StatusOK, tools.SearchSubtitles(q, s.ID)) } +/* +für alle: +q=...&limit=... + +Format für Semester:2024W +semester=... + +firstSemester=...&lastSemester=... +semester=...,..., + +courseID=... + + + +*/ + +func (r searchRoutes) search(c *gin.Context) { + user := c.MustGet("TUMLiveContext").(tools.TUMLiveContext).User + query := c.Query("q") + limit, err := strconv.ParseUint(c.Query("limit"), 10, 64) + if err != nil { + c.AbortWithStatus(http.StatusBadRequest) + return + } + + if courseIDParam := c.Query("courseID"); courseIDParam != "" { + if courseID, err := strconv.Atoi(courseIDParam); err != nil { + c.AbortWithStatus(http.StatusBadRequest) + return + } else { + // course search + c.JSON(http.StatusOK, &courseID) //dummy + } + } + + if semestersParam := c.Query("semester"); semestersParam != "" { + if semesters, err := parseSemesters(semestersParam); err != nil { + c.AbortWithStatus(http.StatusBadRequest) + return + } else { + if len(*semesters) == 1 { + // one semester search + } else { + // multiple semesters search + } + } + } + c.JSON(http.StatusOK, fmt.Sprintf("%s%s%d", user.Name, query, limit)) //dummy +} + +func parseSemesters(semestersParam string) (*[]dao.Semester, error) { + semesterStrings := strings.Split(semestersParam, ",") + + regex, err := regexp.Compile("[0-9]{4}[WS]") + if err != nil { + return nil, err + } + + semesters := make([]dao.Semester, len(semesterStrings)) + for _, semester := range semesterStrings { + if year, err := strconv.Atoi(semester[:4]); regex.MatchString(semestersParam) { + semesters = append(semesters, dao.Semester{ + TeachingTerm: semester[4:], + Year: year, + }) + } else { + return nil, err + } + } + return &semesters, nil +} + +func courseFilter(c *gin.Context, user *model.User, firstSemester dao.Semester, lastSemester dao.Semester) string { + semesterFilter := semesterFilter(firstSemester, lastSemester) + if user != nil && user.Role == model.AdminType { + permissionFilter := permissionFilter(c, user, firstSemester, lastSemester) + return fmt.Sprintf("(%s AND %s)", permissionFilter, semesterFilter) + } else { + return semesterFilter + } +} + +func semesterFilter(firstSemester dao.Semester, lastSemester dao.Semester) string { + if firstSemester.Year == lastSemester.Year && firstSemester.TeachingTerm == lastSemester.TeachingTerm { + return fmt.Sprintf("(year = %d AND teachingTerm = %s)", firstSemester.Year, firstSemester.TeachingTerm) + } else { + var constraint1, constraint2 string + if firstSemester.TeachingTerm == "W" { + constraint1 = fmt.Sprintf("(year = %d AND teachingTerm = %s)", firstSemester.Year, firstSemester.TeachingTerm) + } else { + constraint1 = fmt.Sprintf("year = %d", firstSemester.Year) + } + if lastSemester.TeachingTerm == "S" { + constraint2 = fmt.Sprintf("(year = %d AND teachingTerm = %s)", lastSemester.Year, lastSemester.TeachingTerm) + } else { + constraint2 = fmt.Sprintf("year = %d", lastSemester.Year) + } + if firstSemester.Year+1 < lastSemester.Year { + return fmt.Sprintf("(%s OR (year > %d AND year < %d) OR %s)", constraint1, firstSemester.Year, lastSemester.Year, constraint2) + } else { + return fmt.Sprintf("(%s OR %s)", constraint1, constraint2) + } + } +} + +func permissionFilter(c *gin.Context, user *model.User, firstSemester dao.Semester, lastSemester dao.Semester) string { + if user == nil { + return "(visibility = \"public\")" + } else if user.Role != model.AdminType { + return fmt.Sprintf("(visibility = \"loggedin\" OR visibility = \"public\" OR ID in %s)", courseIdFilter(c, user, firstSemester, lastSemester)) + } else { + return "" + } +} + +// returns a string conforming to MeiliSearch filter format containing each courseId passed onto the function +func courseIdFilter(c *gin.Context, user *model.User, firstSemester dao.Semester, lastSemester dao.Semester) string { + courses := make([]model.Course, 0) + if user != nil { + if firstSemester.Year == lastSemester.Year && firstSemester.TeachingTerm == lastSemester.TeachingTerm { + courses = user.CoursesForSemester(firstSemester.Year, firstSemester.TeachingTerm, c) + } else { + courses = user.CoursesForSemesters(firstSemester.Year, firstSemester.TeachingTerm, lastSemester.Year, lastSemester.TeachingTerm, c) + } + } + courseIDs := make([]uint, 0) + for _, course := range courses { + courseIDs = append(courseIDs, course.ID) + } + + var courseIDsAsStringArray []string + courseIDsAsStringArray = make([]string, len(courseIDs)) + for i, courseID := range courseIDs { + courseIDsAsStringArray[i] = strconv.FormatUint(uint64(courseID), 10) + } + courseIdFilter := "[" + strings.Join(courseIDsAsStringArray, ", ") + "]" + return courseIdFilter +} + func (r searchRoutes) searchCourses(c *gin.Context) { user := c.MustGet("TUMLiveContext").(tools.TUMLiveContext).User q := c.Query("q") t := c.Query("term") y, err := strconv.ParseInt(c.Query("year"), 10, 64) - if err != nil { + if err != nil || (t != "W" && t != "S") { return } + courseIDs := r.getSearchableCoursesOfUserForOneSemester(c, user, y, t) + var courseIDsAsStringArray []string + courseIDsAsStringArray = make([]string, len(*courseIDs)) + for i, courseID := range *courseIDs { + courseIDsAsStringArray[i] = strconv.FormatUint(uint64(courseID), 10) + } + filter := "[" + strings.Join(courseIDsAsStringArray, ", ") + "]" + c.JSON(http.StatusOK, tools.SearchCourses(q, int(y), t, filter)) +} + +func (r searchRoutes) getSearchableCoursesOfUserForOneSemester(c *gin.Context, user *model.User, y int64, t string) *[]uint { var courses []model.Course if user != nil { switch user.Role { @@ -63,6 +219,5 @@ func (r searchRoutes) searchCourses(c *gin.Context) { distinctCourseIDs[course.ID] = true } } - - c.JSON(http.StatusOK, tools.SearchCourses(q, int(y), t, &courseIDs)) + return &courseIDs } diff --git a/model/user.go b/model/user.go index 04be99a44..768d1e129 100755 --- a/model/user.go +++ b/model/user.go @@ -293,6 +293,40 @@ func (u *User) CoursesForSemester(year int, term string, context context.Context return cRes } +func (u *User) CoursesForSemesters(firstYear int, firstTerm string, lastYear int, lastTerm string, context context.Context) []Course { + type Semester struct { + TeachingTerm string + Year int + } + inRangeOfSemesters := func(s Semester, firstSemester Semester, lastSemester Semester) bool { + return !(s.Year < firstSemester.Year || s.Year > lastSemester.Year || + (s.Year == firstSemester.Year && s.TeachingTerm == "S" && firstSemester.TeachingTerm == "W") || + (s.Year == lastSemester.Year && s.TeachingTerm == "W" && lastSemester.TeachingTerm == "S")) + } + + firstSemester := Semester{firstTerm, firstYear} + lastSemester := Semester{lastTerm, lastYear} + cMap := make(map[uint]Course) + var semester Semester + for _, c := range u.Courses { + semester = Semester{TeachingTerm: c.TeachingTerm, Year: c.Year} + if inRangeOfSemesters(semester, firstSemester, lastSemester) { + cMap[c.ID] = c + } + } + for _, c := range u.AdministeredCourses { + semester = Semester{TeachingTerm: c.TeachingTerm, Year: c.Year} + if inRangeOfSemesters(semester, firstSemester, lastSemester) { + cMap[c.ID] = c + } + } + var cRes []Course + for _, c := range cMap { + cRes = append(cRes, c) + } + return cRes +} + var ( ErrInvalidHash = errors.New("the encoded hash is not in the correct format") ErrIncompatibleVersion = errors.New("incompatible version of argon2") diff --git a/tools/meiliExporter.go b/tools/meiliExporter.go index 237982548..76ab79468 100644 --- a/tools/meiliExporter.go +++ b/tools/meiliExporter.go @@ -151,7 +151,14 @@ func (m *MeiliExporter) SetIndexSettings() { "W": {"Wintersemester", "Winter", "WS", "WiSe"}, "S": {"Sommersemester", "Sommer", "SS", "SoSe", "Summer"}, } - _, err := index.UpdateSynonyms(&synonyms) + _, err := m.c.Index("STREAMS").UpdateSettings(&meilisearch.Settings{ + FilterableAttributes: []string{"courseID", "year", "semester"}, + SearchableAttributes: []string{"name", "description"}, + }) + if err != nil { + logger.Warn("could not set settings for meili index STREAMS", "err", err) + } + _, err = index.UpdateSynonyms(&synonyms) if err != nil { logger.Error("could not set synonyms for meili index STREAMS", "err", err) } diff --git a/tools/meiliSearch.go b/tools/meiliSearch.go index dc8919c7e..58f1bbf6a 100644 --- a/tools/meiliSearch.go +++ b/tools/meiliSearch.go @@ -3,8 +3,6 @@ package tools import ( "fmt" "github.com/meilisearch/meilisearch-go" - "strconv" - "strings" ) func SearchSubtitles(q string, streamID uint) *meilisearch.SearchResponse { @@ -23,16 +21,33 @@ func SearchSubtitles(q string, streamID uint) *meilisearch.SearchResponse { return response } -func SearchCourses(q string, year int, t string, searchableCourseIDs *[]uint) *meilisearch.SearchResponse { +/* +func Search(q string, searchType int, filter string) *meilisearch.SearchResponse { + + bit_operator := 1 + var _ []meilisearch.SearchRequest + + for i := 0; i < 4; i++ { + switch searchType & bit_operator { + case 0: + continue + case 1: + // add Subtitles Request + case 2: + // add Other request + } + bit_operator <<= 1 + } + + //multisearch request +}*/ + +func SearchCourses(q string, year int, t string, courseIdFilter string) *meilisearch.SearchResponse { c, err := Cfg.GetMeiliClient() if err != nil { return nil } - if t != "W" && t != "S" { - return nil - } - courseIdFilter := courseIdFilter(searchableCourseIDs) response, err := c.Index("COURSES").Search(q, &meilisearch.SearchRequest{ Filter: fmt.Sprintf("year = %d AND teachingTerm = %s AND ID IN %s", year, t, courseIdFilter), Limit: 10, @@ -44,14 +59,3 @@ func SearchCourses(q string, year int, t string, searchableCourseIDs *[]uint) *m } return response } - -// returns a string conforming to Meili Search Filters Format containing each courseId passed onto the function -func courseIdFilter(searchableCourseIDs *[]uint) string { - var courseIDsAsStringArray []string - courseIDsAsStringArray = make([]string, len(*searchableCourseIDs)) - for i, courseID := range *searchableCourseIDs { - courseIDsAsStringArray[i] = strconv.FormatUint(uint64(courseID), 10) - } - courseIdFilter := "[" + strings.Join(courseIDsAsStringArray, ", ") + "]" - return courseIdFilter -} From 1dd1188ece27821fc8c6a842b4b44cfd0c4d1538 Mon Sep 17 00:00:00 2001 From: johanneskarrer Date: Thu, 15 Aug 2024 21:39:22 +0200 Subject: [PATCH 004/117] adds missing search request functions --- api/search.go | 87 ++++++++++++++++++++++++++++++++++-------- dao/streams.go | 8 ++-- tools/meiliExporter.go | 4 ++ tools/meiliSearch.go | 65 +++++++++++++++++++++++++------ 4 files changed, 135 insertions(+), 29 deletions(-) diff --git a/api/search.go b/api/search.go index afe6b0fef..c32fedb2d 100644 --- a/api/search.go +++ b/api/search.go @@ -54,6 +54,8 @@ courseID=... */ +// TODO param check? +// TODO after search eligibility check func (r searchRoutes) search(c *gin.Context) { user := c.MustGet("TUMLiveContext").(tools.TUMLiveContext).User query := c.Query("q") @@ -70,6 +72,7 @@ func (r searchRoutes) search(c *gin.Context) { } else { // course search c.JSON(http.StatusOK, &courseID) //dummy + } } @@ -98,7 +101,7 @@ func parseSemesters(semestersParam string) (*[]dao.Semester, error) { semesters := make([]dao.Semester, len(semesterStrings)) for _, semester := range semesterStrings { - if year, err := strconv.Atoi(semester[:4]); regex.MatchString(semestersParam) { + if year, err := strconv.Atoi(semester[:4]); regex.MatchString(semestersParam) && err == nil { semesters = append(semesters, dao.Semester{ TeachingTerm: semester[4:], Year: year, @@ -110,10 +113,50 @@ func parseSemesters(semestersParam string) (*[]dao.Semester, error) { return &semesters, nil } +func subtitleFilter(user *model.User, courses []model.Course) string { + if len(courses) == 0 { + return "" + } + + var streamIDs []uint + for _, course := range courses { + if user.IsEligibleToWatchCourse(course) { + for _, stream := range course.Streams { + if !stream.Private || user.IsAdminOfCourse(course) { + streamIDs = append(streamIDs, stream.ID) + } + } + } + } + return uintArrayToString(&streamIDs) +} + +func streamFilter(c *gin.Context, user *model.User, semester dao.Semester) string { + semesterFilter := fmt.Sprintf("(year = %d AND teachingTerm = %s)", semester.Year, semester.TeachingTerm) + if user == nil || user.Role != model.AdminType { + permissionFilter := streamPermissionFilter(c, user, semester) + return fmt.Sprintf("(%s AND %s)", permissionFilter, semesterFilter) + } else { + return semesterFilter + } +} + +// TODO private streams searchable for course admins +// TODO mit coursePermissionFilter zusammenlegen +func streamPermissionFilter(c *gin.Context, user *model.User, semester dao.Semester) string { + if user == nil { + return "(visibility = public AND private = 0)" + } else if user.Role != model.AdminType { + return fmt.Sprintf("((visibility = loggedin OR visibility = public OR (visibility = enrolled AND courseID in %s)) AND private = 0)", courseIdFilter(c, user, semester, semester)) + } else { + return "" + } +} + func courseFilter(c *gin.Context, user *model.User, firstSemester dao.Semester, lastSemester dao.Semester) string { semesterFilter := semesterFilter(firstSemester, lastSemester) - if user != nil && user.Role == model.AdminType { - permissionFilter := permissionFilter(c, user, firstSemester, lastSemester) + if user == nil || user.Role != model.AdminType { + permissionFilter := coursePermissionFilter(c, user, firstSemester, lastSemester) return fmt.Sprintf("(%s AND %s)", permissionFilter, semesterFilter) } else { return semesterFilter @@ -143,11 +186,12 @@ func semesterFilter(firstSemester dao.Semester, lastSemester dao.Semester) strin } } -func permissionFilter(c *gin.Context, user *model.User, firstSemester dao.Semester, lastSemester dao.Semester) string { +// TODO OR ID in [administeredcourses] +func coursePermissionFilter(c *gin.Context, user *model.User, firstSemester dao.Semester, lastSemester dao.Semester) string { if user == nil { - return "(visibility = \"public\")" + return "(visibility = public)" } else if user.Role != model.AdminType { - return fmt.Sprintf("(visibility = \"loggedin\" OR visibility = \"public\" OR ID in %s)", courseIdFilter(c, user, firstSemester, lastSemester)) + return fmt.Sprintf("(visibility = loggedin OR visibility = public OR (visibility = enrolled AND ID IN %s))", courseIdFilter(c, user, firstSemester, lastSemester)) } else { return "" } @@ -168,15 +212,10 @@ func courseIdFilter(c *gin.Context, user *model.User, firstSemester dao.Semester courseIDs = append(courseIDs, course.ID) } - var courseIDsAsStringArray []string - courseIDsAsStringArray = make([]string, len(courseIDs)) - for i, courseID := range courseIDs { - courseIDsAsStringArray[i] = strconv.FormatUint(uint64(courseID), 10) - } - courseIdFilter := "[" + strings.Join(courseIDsAsStringArray, ", ") + "]" - return courseIdFilter + return uintArrayToString(&courseIDs) } +// TODO refactor to match function search func (r searchRoutes) searchCourses(c *gin.Context) { user := c.MustGet("TUMLiveContext").(tools.TUMLiveContext).User q := c.Query("q") @@ -192,8 +231,13 @@ func (r searchRoutes) searchCourses(c *gin.Context) { for i, courseID := range *courseIDs { courseIDsAsStringArray[i] = strconv.FormatUint(uint64(courseID), 10) } - filter := "[" + strings.Join(courseIDsAsStringArray, ", ") + "]" - c.JSON(http.StatusOK, tools.SearchCourses(q, int(y), t, filter)) + courses := "[" + strings.Join(courseIDsAsStringArray, ", ") + "]" + sem := dao.Semester{ + TeachingTerm: t, + Year: int(y), + } + filter := fmt.Sprintf("%s AND ID IN %s", semesterFilter(sem, sem), courses) + c.JSON(http.StatusOK, tools.SearchCourses(q, filter)) } func (r searchRoutes) getSearchableCoursesOfUserForOneSemester(c *gin.Context, user *model.User, y int64, t string) *[]uint { @@ -221,3 +265,16 @@ func (r searchRoutes) getSearchableCoursesOfUserForOneSemester(c *gin.Context, u } return &courseIDs } + +func uintArrayToString(ids *[]uint) string { + if len(*ids) == 0 { + return "" + } + var idsAsStringArray []string + idsAsStringArray = make([]string, len(*ids)) + for i, id := range *ids { + idsAsStringArray[i] = strconv.FormatUint(uint64(id), 10) + } + filter := "[" + strings.Join(idsAsStringArray, ", ") + "]" + return filter +} diff --git a/dao/streams.go b/dao/streams.go index fb838b531..0431c8fdb 100755 --- a/dao/streams.go +++ b/dao/streams.go @@ -212,9 +212,9 @@ func (d streamsDao) GetAllStreams() ([]model.Stream, error) { } type StreamWithCourseAndSubtitles struct { - Name, Description, TeachingTerm, CourseName, Subtitles string - ID, CourseID uint - Year int + Name, Description, TeachingTerm, CourseName, Subtitles, Visibility string + ID, CourseID, Private uint + Year int } // ExecAllStreamsWithCoursesAndSubtitles executes f on all streams with their courses and subtitles preloaded. @@ -229,10 +229,12 @@ func (d streamsDao) ExecAllStreamsWithCoursesAndSubtitles(f func([]StreamWithCou SELECT streams.id, streams.name, streams.description, + streams.private as private, c.id as course_id, c.name as course_name, c.teaching_term, c.year, + c.visibility as visibility, s.content as subtitles, IFNULL(s.stream_id, streams.id) as sid FROM streams diff --git a/tools/meiliExporter.go b/tools/meiliExporter.go index 76ab79468..86aac1540 100644 --- a/tools/meiliExporter.go +++ b/tools/meiliExporter.go @@ -18,6 +18,8 @@ type MeiliStream struct { Year int `json:"year"` TeachingTerm string `json:"semester"` CourseID uint `json:"courseID"` + Private uint `json:"private"` + Visibility string `json:"visibility"` //corresponds to the visibility of the course } type MeiliSubtitles struct { @@ -78,6 +80,8 @@ func (m *MeiliExporter) Export() { CourseName: stream.CourseName, Year: stream.Year, TeachingTerm: stream.TeachingTerm, + Visibility: stream.Visibility, + Private: stream.Private, } if stream.Subtitles != "" { meiliSubtitles := make([]MeiliSubtitles, 0) diff --git a/tools/meiliSearch.go b/tools/meiliSearch.go index 58f1bbf6a..4c575fe3b 100644 --- a/tools/meiliSearch.go +++ b/tools/meiliSearch.go @@ -21,35 +21,77 @@ func SearchSubtitles(q string, streamID uint) *meilisearch.SearchResponse { return response } -/* -func Search(q string, searchType int, filter string) *meilisearch.SearchResponse { +func getCourseWideSubtitleSearchRequest(q string, limit int, streamFilter string) meilisearch.SearchRequest { + req := meilisearch.SearchRequest{ + IndexUID: "SUBTITLES", + Query: q, + Limit: int64(limit) + 2, + Filter: streamFilter, + } + return req +} + +func getStreamsSearchRequest(q string, limit int, streamFilter string) meilisearch.SearchRequest { + req := meilisearch.SearchRequest{ + IndexUID: "STREAMS", + Query: q, + Limit: int64(limit) + 2, + Filter: streamFilter, + } + return req +} + +func getCoursesSearchRequest(q string, limit int, courseFilter string) meilisearch.SearchRequest { + req := meilisearch.SearchRequest{ + IndexUID: "COURSES", + Query: q, + Limit: int64(limit) + 2, + Filter: courseFilter, + } + return req +} + +func Search(q string, limit int, searchType int, courseFilter string, streamFilter string) *meilisearch.MultiSearchResponse { + c, err := Cfg.GetMeiliClient() + if err != nil { + return nil + } - bit_operator := 1 - var _ []meilisearch.SearchRequest + bitOperator := 1 + var reqs []meilisearch.SearchRequest for i := 0; i < 4; i++ { - switch searchType & bit_operator { + switch searchType & bitOperator { case 0: continue case 1: // add Subtitles Request + reqs = append(reqs, getCourseWideSubtitleSearchRequest(q, limit, streamFilter)) case 2: - // add Other request + reqs = append(reqs, getStreamsSearchRequest(q, limit, streamFilter)) + case 4: + reqs = append(reqs, getCoursesSearchRequest(q, limit, courseFilter)) } - bit_operator <<= 1 + bitOperator <<= 1 } - //multisearch request -}*/ + //multisearch Request + response, err := c.MultiSearch(&meilisearch.MultiSearchRequest{Queries: reqs}) + if err != nil { + logger.Error("could not search in meili", "err", err) + return nil + } + return response +} -func SearchCourses(q string, year int, t string, courseIdFilter string) *meilisearch.SearchResponse { +func SearchCourses(q string, filter string) *meilisearch.SearchResponse { c, err := Cfg.GetMeiliClient() if err != nil { return nil } response, err := c.Index("COURSES").Search(q, &meilisearch.SearchRequest{ - Filter: fmt.Sprintf("year = %d AND teachingTerm = %s AND ID IN %s", year, t, courseIdFilter), + Filter: filter, Limit: 10, }) @@ -57,5 +99,6 @@ func SearchCourses(q string, year int, t string, courseIdFilter string) *meilise logger.Error("could not search courses in meili", "err", err) return nil } + print(response.ProcessingTimeMs) return response } From 65115dfa746bf8a5c065639462797cd0a99be954 Mon Sep 17 00:00:00 2001 From: johanneskarrer Date: Fri, 16 Aug 2024 11:40:31 +0200 Subject: [PATCH 005/117] fixed user == nil case --- model/user.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/model/user.go b/model/user.go index 768d1e129..e6b1a91da 100755 --- a/model/user.go +++ b/model/user.go @@ -263,7 +263,10 @@ func (u *User) IsAdminOfCourse(course Course) bool { } func (u *User) IsEligibleToWatchCourse(course Course) bool { - if course.Visibility == "loggedin" || course.Visibility == "public" { + if u == nil { + return course.Visibility == "public" + } + if course.Visibility == "loggedin" { return true } for _, invCourse := range u.Courses { From 537d67c666e7fc0132088c0b8c45657953279a8c Mon Sep 17 00:00:00 2001 From: johanneskarrer Date: Fri, 16 Aug 2024 16:44:15 +0200 Subject: [PATCH 006/117] course admins can search for their courses; meili only retrieves relevant fields --- api/search.go | 99 +++++++++++++++++++++++++++++--------------- model/user.go | 62 ++++++++++++++++++++------- tools/meiliSearch.go | 35 +++++++++------- 3 files changed, 133 insertions(+), 63 deletions(-) diff --git a/api/search.go b/api/search.go index c32fedb2d..6377d246b 100644 --- a/api/search.go +++ b/api/search.go @@ -61,8 +61,7 @@ func (r searchRoutes) search(c *gin.Context) { query := c.Query("q") limit, err := strconv.ParseUint(c.Query("limit"), 10, 64) if err != nil { - c.AbortWithStatus(http.StatusBadRequest) - return + limit = 10 } if courseIDParam := c.Query("courseID"); courseIDParam != "" { @@ -76,22 +75,42 @@ func (r searchRoutes) search(c *gin.Context) { } } - if semestersParam := c.Query("semester"); semestersParam != "" { + firstSemesterParam := c.Query("firstSemester") + lastSemesterParam := c.Query("lastSemester") + if firstSemesterParam != "" && lastSemesterParam != "" { + semesters1, err1 := parseSemesters(firstSemesterParam) + semesters2, err2 := parseSemesters(lastSemesterParam) + if err1 != nil || err2 != nil { + c.AbortWithStatus(http.StatusBadRequest) + return + } + firstSemester := semesters1[0] + lastSemester := semesters2[0] + res := tools.Search(query, int64(limit), 4, courseFilter(c, user, firstSemester, lastSemester), "") + //TODO response check + + return + } + + semestersParam := c.Query("semester") + if semestersParam != "" { if semesters, err := parseSemesters(semestersParam); err != nil { c.AbortWithStatus(http.StatusBadRequest) return } else { - if len(*semesters) == 1 { + if len(semesters) == 1 { // one semester search + res := tools.Search(query, int64(limit), 6, courseFilter(c, user, semesters[0], semesters[0]), streamFilter(c, user, semesters[0])) } else { // multiple semesters search + res := tools.Search(query, int64(limit), 4, courseFilter(c, user, semesters[0], semesters[1]), "") } } } c.JSON(http.StatusOK, fmt.Sprintf("%s%s%d", user.Name, query, limit)) //dummy } -func parseSemesters(semestersParam string) (*[]dao.Semester, error) { +func parseSemesters(semestersParam string) ([]dao.Semester, error) { semesterStrings := strings.Split(semestersParam, ",") regex, err := regexp.Compile("[0-9]{4}[WS]") @@ -110,7 +129,7 @@ func parseSemesters(semestersParam string) (*[]dao.Semester, error) { return nil, err } } - return &semesters, nil + return semesters, nil } func subtitleFilter(user *model.User, courses []model.Course) string { @@ -128,7 +147,7 @@ func subtitleFilter(user *model.User, courses []model.Course) string { } } } - return uintArrayToString(&streamIDs) + return uintSliceToString(streamIDs) } func streamFilter(c *gin.Context, user *model.User, semester dao.Semester) string { @@ -147,14 +166,25 @@ func streamPermissionFilter(c *gin.Context, user *model.User, semester dao.Semes if user == nil { return "(visibility = public AND private = 0)" } else if user.Role != model.AdminType { - return fmt.Sprintf("((visibility = loggedin OR visibility = public OR (visibility = enrolled AND courseID in %s)) AND private = 0)", courseIdFilter(c, user, semester, semester)) + if len(user.AdministeredCourses) == 0 { + return fmt.Sprintf("((visibility = loggedin OR visibility = public OR (visibility = enrolled AND courseID in %s)) AND private = 0)", courseIdFilter(c, user, semester, semester)) + } else { + administeredCourses := user.AdministeredCoursesForSemester(semester.Year, semester.TeachingTerm, c) + var administeredCourseIDs []uint + for _, course := range administeredCourses { + administeredCourseIDs = append(administeredCourseIDs, course.ID) + } + administeredCoursesFilter := uintSliceToString(administeredCourseIDs) + return fmt.Sprintf("((visibility = loggedin OR visibility = public OR (visibility = enrolled AND courseID in %s)) AND private = 0 OR courseID IN %s)", courseIdFilter(c, user, semester, semester), administeredCoursesFilter) + + } } else { return "" } } func courseFilter(c *gin.Context, user *model.User, firstSemester dao.Semester, lastSemester dao.Semester) string { - semesterFilter := semesterFilter(firstSemester, lastSemester) + semesterFilter := meiliSemesterFilterInRange(firstSemester, lastSemester) if user == nil || user.Role != model.AdminType { permissionFilter := coursePermissionFilter(c, user, firstSemester, lastSemester) return fmt.Sprintf("(%s AND %s)", permissionFilter, semesterFilter) @@ -163,7 +193,7 @@ func courseFilter(c *gin.Context, user *model.User, firstSemester dao.Semester, } } -func semesterFilter(firstSemester dao.Semester, lastSemester dao.Semester) string { +func meiliSemesterFilterInRange(firstSemester dao.Semester, lastSemester dao.Semester) string { if firstSemester.Year == lastSemester.Year && firstSemester.TeachingTerm == lastSemester.TeachingTerm { return fmt.Sprintf("(year = %d AND teachingTerm = %s)", firstSemester.Year, firstSemester.TeachingTerm) } else { @@ -191,7 +221,17 @@ func coursePermissionFilter(c *gin.Context, user *model.User, firstSemester dao. if user == nil { return "(visibility = public)" } else if user.Role != model.AdminType { - return fmt.Sprintf("(visibility = loggedin OR visibility = public OR (visibility = enrolled AND ID IN %s))", courseIdFilter(c, user, firstSemester, lastSemester)) + if len(user.AdministeredCourses) == 0 { + return fmt.Sprintf("(visibility = loggedin OR visibility = public OR (visibility = enrolled AND ID IN %s))", courseIdFilter(c, user, firstSemester, lastSemester)) + } else { + administeredCourses := user.AdministeredCoursesForSemesters(firstSemester.Year, firstSemester.TeachingTerm, lastSemester.Year, lastSemester.TeachingTerm, c) + var administeredCourseIDs []uint + for _, course := range administeredCourses { + administeredCourseIDs = append(administeredCourseIDs, course.ID) + } + administeredCoursesFilter := uintSliceToString(administeredCourseIDs) + return fmt.Sprintf("(visibility = loggedin OR visibility = public OR (visibility = enrolled AND ID IN %s) OR ID in %s)", courseIdFilter(c, user, firstSemester, lastSemester), administeredCoursesFilter) + } } else { return "" } @@ -200,19 +240,18 @@ func coursePermissionFilter(c *gin.Context, user *model.User, firstSemester dao. // returns a string conforming to MeiliSearch filter format containing each courseId passed onto the function func courseIdFilter(c *gin.Context, user *model.User, firstSemester dao.Semester, lastSemester dao.Semester) string { courses := make([]model.Course, 0) + courseIDs := make([]uint, 0) if user != nil { if firstSemester.Year == lastSemester.Year && firstSemester.TeachingTerm == lastSemester.TeachingTerm { - courses = user.CoursesForSemester(firstSemester.Year, firstSemester.TeachingTerm, c) + courses = user.CoursesForSemesterWithoutAdministeredCourses(firstSemester.Year, firstSemester.TeachingTerm, c) } else { - courses = user.CoursesForSemesters(firstSemester.Year, firstSemester.TeachingTerm, lastSemester.Year, lastSemester.TeachingTerm, c) + courses = user.CoursesForSemestersWithoutAdministeredCourses(firstSemester.Year, firstSemester.TeachingTerm, lastSemester.Year, lastSemester.TeachingTerm, c) + } + for _, c := range courses { + courseIDs = append(courseIDs, c.ID) } } - courseIDs := make([]uint, 0) - for _, course := range courses { - courseIDs = append(courseIDs, course.ID) - } - - return uintArrayToString(&courseIDs) + return uintSliceToString(courseIDs) } // TODO refactor to match function search @@ -226,21 +265,15 @@ func (r searchRoutes) searchCourses(c *gin.Context) { } courseIDs := r.getSearchableCoursesOfUserForOneSemester(c, user, y, t) - var courseIDsAsStringArray []string - courseIDsAsStringArray = make([]string, len(*courseIDs)) - for i, courseID := range *courseIDs { - courseIDsAsStringArray[i] = strconv.FormatUint(uint64(courseID), 10) - } - courses := "[" + strings.Join(courseIDsAsStringArray, ", ") + "]" sem := dao.Semester{ TeachingTerm: t, Year: int(y), } - filter := fmt.Sprintf("%s AND ID IN %s", semesterFilter(sem, sem), courses) + filter := fmt.Sprintf("%s AND ID IN %s", meiliSemesterFilterInRange(sem, sem), uintSliceToString(courseIDs)) c.JSON(http.StatusOK, tools.SearchCourses(q, filter)) } -func (r searchRoutes) getSearchableCoursesOfUserForOneSemester(c *gin.Context, user *model.User, y int64, t string) *[]uint { +func (r searchRoutes) getSearchableCoursesOfUserForOneSemester(c *gin.Context, user *model.User, y int64, t string) []uint { var courses []model.Course if user != nil { switch user.Role { @@ -263,16 +296,16 @@ func (r searchRoutes) getSearchableCoursesOfUserForOneSemester(c *gin.Context, u distinctCourseIDs[course.ID] = true } } - return &courseIDs + return courseIDs } -func uintArrayToString(ids *[]uint) string { - if len(*ids) == 0 { - return "" +func uintSliceToString(ids []uint) string { + if ids == nil || len(ids) == 0 { + return "[]" } var idsAsStringArray []string - idsAsStringArray = make([]string, len(*ids)) - for i, id := range *ids { + idsAsStringArray = make([]string, len(ids)) + for i, id := range ids { idsAsStringArray[i] = strconv.FormatUint(uint64(id), 10) } filter := "[" + strings.Join(idsAsStringArray, ", ") + "]" diff --git a/model/user.go b/model/user.go index e6b1a91da..5bb272375 100755 --- a/model/user.go +++ b/model/user.go @@ -296,7 +296,17 @@ func (u *User) CoursesForSemester(year int, term string, context context.Context return cRes } -func (u *User) CoursesForSemesters(firstYear int, firstTerm string, lastYear int, lastTerm string, context context.Context) []Course { +func (u *User) AdministeredCoursesForSemester(year int, term string, context context.Context) []Course { + administeredCourses := make([]Course, 0) + for _, c := range u.AdministeredCourses { + if c.Year == year && c.TeachingTerm == term { + administeredCourses = append(administeredCourses, c) + } + } + return administeredCourses +} + +func (u *User) AdministeredCoursesForSemesters(firstYear int, firstTerm string, lastYear int, lastTerm string, context context.Context) []Course { type Semester struct { TeachingTerm string Year int @@ -306,28 +316,52 @@ func (u *User) CoursesForSemesters(firstYear int, firstTerm string, lastYear int (s.Year == firstSemester.Year && s.TeachingTerm == "S" && firstSemester.TeachingTerm == "W") || (s.Year == lastSemester.Year && s.TeachingTerm == "W" && lastSemester.TeachingTerm == "S")) } - + var semester Semester firstSemester := Semester{firstTerm, firstYear} lastSemester := Semester{lastTerm, lastYear} - cMap := make(map[uint]Course) - var semester Semester - for _, c := range u.Courses { + administeredCourses := make([]Course, 0) + for _, c := range u.AdministeredCourses { semester = Semester{TeachingTerm: c.TeachingTerm, Year: c.Year} if inRangeOfSemesters(semester, firstSemester, lastSemester) { - cMap[c.ID] = c + administeredCourses = append(administeredCourses, c) } } - for _, c := range u.AdministeredCourses { - semester = Semester{TeachingTerm: c.TeachingTerm, Year: c.Year} - if inRangeOfSemesters(semester, firstSemester, lastSemester) { - cMap[c.ID] = c + return administeredCourses + +} + +func (u *User) CoursesForSemesterWithoutAdministeredCourses(year int, term string, context context.Context) []Course { + courses := make([]Course, 0) + for _, c := range u.Courses { + if c.Year == year && c.TeachingTerm == term && !u.IsAdminOfCourse(c) { + courses = append(courses, c) } } - var cRes []Course - for _, c := range cMap { - cRes = append(cRes, c) + return courses +} + +func (u *User) CoursesForSemestersWithoutAdministeredCourses(firstYear int, firstTerm string, lastYear int, lastTerm string, context context.Context) []Course { + type Semester struct { + TeachingTerm string + Year int } - return cRes + inRangeOfSemesters := func(s Semester, firstSemester Semester, lastSemester Semester) bool { + return !(s.Year < firstSemester.Year || s.Year > lastSemester.Year || + (s.Year == firstSemester.Year && s.TeachingTerm == "S" && firstSemester.TeachingTerm == "W") || + (s.Year == lastSemester.Year && s.TeachingTerm == "W" && lastSemester.TeachingTerm == "S")) + } + + var semester Semester + firstSemester := Semester{firstTerm, firstYear} + lastSemester := Semester{lastTerm, lastYear} + courses := make([]Course, 0) + for _, c := range u.Courses { + semester = Semester{TeachingTerm: c.TeachingTerm, Year: c.Year} + if inRangeOfSemesters(semester, firstSemester, lastSemester) && !u.IsAdminOfCourse(c) { + courses = append(courses, c) + } + } + return courses } var ( diff --git a/tools/meiliSearch.go b/tools/meiliSearch.go index 4c575fe3b..6b58fd6eb 100644 --- a/tools/meiliSearch.go +++ b/tools/meiliSearch.go @@ -21,37 +21,40 @@ func SearchSubtitles(q string, streamID uint) *meilisearch.SearchResponse { return response } -func getCourseWideSubtitleSearchRequest(q string, limit int, streamFilter string) meilisearch.SearchRequest { +func getCourseWideSubtitleSearchRequest(q string, limit int64, streamFilter string) meilisearch.SearchRequest { req := meilisearch.SearchRequest{ - IndexUID: "SUBTITLES", - Query: q, - Limit: int64(limit) + 2, - Filter: streamFilter, + IndexUID: "SUBTITLES", + Query: q, + Limit: limit + 2, + Filter: streamFilter, + AttributesToRetrieve: []string{"streamID", "timestamp", "textPrev", "text", "textNext"}, } return req } -func getStreamsSearchRequest(q string, limit int, streamFilter string) meilisearch.SearchRequest { +func getStreamsSearchRequest(q string, limit int64, streamFilter string) meilisearch.SearchRequest { req := meilisearch.SearchRequest{ - IndexUID: "STREAMS", - Query: q, - Limit: int64(limit) + 2, - Filter: streamFilter, + IndexUID: "STREAMS", + Query: q, + Limit: limit + 2, + Filter: streamFilter, + AttributesToRetrieve: []string{"ID", "name", "description", "courseName", "year", "teachingTerm"}, } return req } -func getCoursesSearchRequest(q string, limit int, courseFilter string) meilisearch.SearchRequest { +func getCoursesSearchRequest(q string, limit int64, courseFilter string) meilisearch.SearchRequest { req := meilisearch.SearchRequest{ - IndexUID: "COURSES", - Query: q, - Limit: int64(limit) + 2, - Filter: courseFilter, + IndexUID: "COURSES", + Query: q, + Limit: limit + 2, + Filter: courseFilter, + AttributesToRetrieve: []string{"name", "slug", "year", "teachingTerm"}, } return req } -func Search(q string, limit int, searchType int, courseFilter string, streamFilter string) *meilisearch.MultiSearchResponse { +func Search(q string, limit int64, searchType int, courseFilter string, streamFilter string) *meilisearch.MultiSearchResponse { c, err := Cfg.GetMeiliClient() if err != nil { return nil From 2e4edb51fabf5b1f6d4ef50f8e10a53e0cb1818d Mon Sep 17 00:00:00 2001 From: johanneskarrer Date: Fri, 16 Aug 2024 16:50:19 +0200 Subject: [PATCH 007/117] converted function slicetostring to type course --- api/search.go | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/api/search.go b/api/search.go index 6377d246b..aa6d2ce2d 100644 --- a/api/search.go +++ b/api/search.go @@ -170,11 +170,7 @@ func streamPermissionFilter(c *gin.Context, user *model.User, semester dao.Semes return fmt.Sprintf("((visibility = loggedin OR visibility = public OR (visibility = enrolled AND courseID in %s)) AND private = 0)", courseIdFilter(c, user, semester, semester)) } else { administeredCourses := user.AdministeredCoursesForSemester(semester.Year, semester.TeachingTerm, c) - var administeredCourseIDs []uint - for _, course := range administeredCourses { - administeredCourseIDs = append(administeredCourseIDs, course.ID) - } - administeredCoursesFilter := uintSliceToString(administeredCourseIDs) + administeredCoursesFilter := courseSliceToString(administeredCourses) return fmt.Sprintf("((visibility = loggedin OR visibility = public OR (visibility = enrolled AND courseID in %s)) AND private = 0 OR courseID IN %s)", courseIdFilter(c, user, semester, semester), administeredCoursesFilter) } @@ -225,11 +221,7 @@ func coursePermissionFilter(c *gin.Context, user *model.User, firstSemester dao. return fmt.Sprintf("(visibility = loggedin OR visibility = public OR (visibility = enrolled AND ID IN %s))", courseIdFilter(c, user, firstSemester, lastSemester)) } else { administeredCourses := user.AdministeredCoursesForSemesters(firstSemester.Year, firstSemester.TeachingTerm, lastSemester.Year, lastSemester.TeachingTerm, c) - var administeredCourseIDs []uint - for _, course := range administeredCourses { - administeredCourseIDs = append(administeredCourseIDs, course.ID) - } - administeredCoursesFilter := uintSliceToString(administeredCourseIDs) + administeredCoursesFilter := courseSliceToString(administeredCourses) return fmt.Sprintf("(visibility = loggedin OR visibility = public OR (visibility = enrolled AND ID IN %s) OR ID in %s)", courseIdFilter(c, user, firstSemester, lastSemester), administeredCoursesFilter) } } else { @@ -240,18 +232,14 @@ func coursePermissionFilter(c *gin.Context, user *model.User, firstSemester dao. // returns a string conforming to MeiliSearch filter format containing each courseId passed onto the function func courseIdFilter(c *gin.Context, user *model.User, firstSemester dao.Semester, lastSemester dao.Semester) string { courses := make([]model.Course, 0) - courseIDs := make([]uint, 0) if user != nil { if firstSemester.Year == lastSemester.Year && firstSemester.TeachingTerm == lastSemester.TeachingTerm { courses = user.CoursesForSemesterWithoutAdministeredCourses(firstSemester.Year, firstSemester.TeachingTerm, c) } else { courses = user.CoursesForSemestersWithoutAdministeredCourses(firstSemester.Year, firstSemester.TeachingTerm, lastSemester.Year, lastSemester.TeachingTerm, c) } - for _, c := range courses { - courseIDs = append(courseIDs, c.ID) - } } - return uintSliceToString(courseIDs) + return courseSliceToString(courses) } // TODO refactor to match function search @@ -299,15 +287,28 @@ func (r searchRoutes) getSearchableCoursesOfUserForOneSemester(c *gin.Context, u return courseIDs } +func courseSliceToString(courses []model.Course) string { + if courses == nil || len(courses) == 0 { + return "[]" + } + var idsStringSlice []string + idsStringSlice = make([]string, len(courses)) + for i, c := range courses { + idsStringSlice[i] = strconv.FormatUint(uint64(c.ID), 10) + } + filter := "[" + strings.Join(idsStringSlice, ", ") + "]" + return filter +} + func uintSliceToString(ids []uint) string { if ids == nil || len(ids) == 0 { return "[]" } - var idsAsStringArray []string - idsAsStringArray = make([]string, len(ids)) + var idsStringSlice []string + idsStringSlice = make([]string, len(ids)) for i, id := range ids { - idsAsStringArray[i] = strconv.FormatUint(uint64(id), 10) + idsStringSlice[i] = strconv.FormatUint(uint64(id), 10) } - filter := "[" + strings.Join(idsAsStringArray, ", ") + "]" + filter := "[" + strings.Join(idsStringSlice, ", ") + "]" return filter } From 7217610a2b9be2fb7ba8fb205e67646ddd68c97c Mon Sep 17 00:00:00 2001 From: johanneskarrer Date: Mon, 19 Aug 2024 14:13:03 +0200 Subject: [PATCH 008/117] limited fields retrieved by meilisearch and names to be uniform with existing code --- api/search.go | 35 ++++++++++++------------------- tools/meiliExporter.go | 6 +++--- tools/meiliSearch.go | 10 ++++----- web/template/search-global.gohtml | 4 ++-- 4 files changed, 23 insertions(+), 32 deletions(-) diff --git a/api/search.go b/api/search.go index aa6d2ce2d..c49e96218 100644 --- a/api/search.go +++ b/api/search.go @@ -7,6 +7,7 @@ import ( "github.com/TUM-Dev/gocast/model" "github.com/TUM-Dev/gocast/tools" "github.com/gin-gonic/gin" + "github.com/meilisearch/meilisearch-go" "net/http" "regexp" "strconv" @@ -63,6 +64,7 @@ func (r searchRoutes) search(c *gin.Context) { if err != nil { limit = 10 } + var res *meilisearch.MultiSearchResponse if courseIDParam := c.Query("courseID"); courseIDParam != "" { if courseID, err := strconv.Atoi(courseIDParam); err != nil { @@ -86,28 +88,17 @@ func (r searchRoutes) search(c *gin.Context) { } firstSemester := semesters1[0] lastSemester := semesters2[0] - res := tools.Search(query, int64(limit), 4, courseFilter(c, user, firstSemester, lastSemester), "") - //TODO response check - return - } - - semestersParam := c.Query("semester") - if semestersParam != "" { - if semesters, err := parseSemesters(semestersParam); err != nil { - c.AbortWithStatus(http.StatusBadRequest) - return + if firstSemester.Year == lastSemester.Year && firstSemester.TeachingTerm == lastSemester.TeachingTerm { + // single semester search + res = tools.Search(query, int64(limit), 6, courseFilter(c, user, firstSemester, firstSemester), streamFilter(c, user, firstSemester)) } else { - if len(semesters) == 1 { - // one semester search - res := tools.Search(query, int64(limit), 6, courseFilter(c, user, semesters[0], semesters[0]), streamFilter(c, user, semesters[0])) - } else { - // multiple semesters search - res := tools.Search(query, int64(limit), 4, courseFilter(c, user, semesters[0], semesters[1]), "") - } + // multiple semester search + res = tools.Search(query, int64(limit), 4, courseFilter(c, user, firstSemester, lastSemester), "") } + //TODO response check + c.JSON(http.StatusOK, res) } - c.JSON(http.StatusOK, fmt.Sprintf("%s%s%d", user.Name, query, limit)) //dummy } func parseSemesters(semestersParam string) ([]dao.Semester, error) { @@ -151,7 +142,7 @@ func subtitleFilter(user *model.User, courses []model.Course) string { } func streamFilter(c *gin.Context, user *model.User, semester dao.Semester) string { - semesterFilter := fmt.Sprintf("(year = %d AND teachingTerm = %s)", semester.Year, semester.TeachingTerm) + semesterFilter := fmt.Sprintf("(year = %d AND semester = %s)", semester.Year, semester.TeachingTerm) if user == nil || user.Role != model.AdminType { permissionFilter := streamPermissionFilter(c, user, semester) return fmt.Sprintf("(%s AND %s)", permissionFilter, semesterFilter) @@ -191,16 +182,16 @@ func courseFilter(c *gin.Context, user *model.User, firstSemester dao.Semester, func meiliSemesterFilterInRange(firstSemester dao.Semester, lastSemester dao.Semester) string { if firstSemester.Year == lastSemester.Year && firstSemester.TeachingTerm == lastSemester.TeachingTerm { - return fmt.Sprintf("(year = %d AND teachingTerm = %s)", firstSemester.Year, firstSemester.TeachingTerm) + return fmt.Sprintf("(year = %d AND semester = %s)", firstSemester.Year, firstSemester.TeachingTerm) } else { var constraint1, constraint2 string if firstSemester.TeachingTerm == "W" { - constraint1 = fmt.Sprintf("(year = %d AND teachingTerm = %s)", firstSemester.Year, firstSemester.TeachingTerm) + constraint1 = fmt.Sprintf("(year = %d AND semester = %s)", firstSemester.Year, firstSemester.TeachingTerm) } else { constraint1 = fmt.Sprintf("year = %d", firstSemester.Year) } if lastSemester.TeachingTerm == "S" { - constraint2 = fmt.Sprintf("(year = %d AND teachingTerm = %s)", lastSemester.Year, lastSemester.TeachingTerm) + constraint2 = fmt.Sprintf("(year = %d AND semester = %s)", lastSemester.Year, lastSemester.TeachingTerm) } else { constraint2 = fmt.Sprintf("year = %d", lastSemester.Year) } diff --git a/tools/meiliExporter.go b/tools/meiliExporter.go index 86aac1540..9279653b6 100644 --- a/tools/meiliExporter.go +++ b/tools/meiliExporter.go @@ -36,7 +36,7 @@ type MeiliCourse struct { Name string `json:"name"` Slug string `json:"slug"` Year int `json:"year"` - TeachingTerm string `json:"teachingTerm"` + TeachingTerm string `json:"semester"` Visibility string `json:"visibility"` } @@ -177,9 +177,9 @@ func (m *MeiliExporter) SetIndexSettings() { } _, err = m.c.Index("COURSES").UpdateSettings(&meilisearch.Settings{ - FilterableAttributes: []string{"ID", "visibility", "year", "teachingTerm"}, + FilterableAttributes: []string{"ID", "visibility", "year", "semester"}, SearchableAttributes: []string{"slug", "name"}, - SortableAttributes: []string{"year", "teachingTerm"}, + SortableAttributes: []string{"year", "semester"}, }) if err != nil { logger.Warn("could not set settings for meili index COURSES", "err", err) diff --git a/tools/meiliSearch.go b/tools/meiliSearch.go index 6b58fd6eb..e945bc764 100644 --- a/tools/meiliSearch.go +++ b/tools/meiliSearch.go @@ -38,7 +38,7 @@ func getStreamsSearchRequest(q string, limit int64, streamFilter string) meilise Query: q, Limit: limit + 2, Filter: streamFilter, - AttributesToRetrieve: []string{"ID", "name", "description", "courseName", "year", "teachingTerm"}, + AttributesToRetrieve: []string{"ID", "name", "description", "courseName", "year", "semester"}, } return req } @@ -49,7 +49,7 @@ func getCoursesSearchRequest(q string, limit int64, courseFilter string) meilise Query: q, Limit: limit + 2, Filter: courseFilter, - AttributesToRetrieve: []string{"name", "slug", "year", "teachingTerm"}, + AttributesToRetrieve: []string{"name", "slug", "year", "semester"}, } return req } @@ -94,14 +94,14 @@ func SearchCourses(q string, filter string) *meilisearch.SearchResponse { } response, err := c.Index("COURSES").Search(q, &meilisearch.SearchRequest{ - Filter: filter, - Limit: 10, + Filter: filter, + Limit: 10, + AttributesToRetrieve: []string{"name", "slug", "year", "semester"}, }) if err != nil { logger.Error("could not search courses in meili", "err", err) return nil } - print(response.ProcessingTimeMs) return response } diff --git a/web/template/search-global.gohtml b/web/template/search-global.gohtml index b1493789d..dd1d7302b 100644 --- a/web/template/search-global.gohtml +++ b/web/template/search-global.gohtml @@ -15,8 +15,8 @@