diff --git a/apps/backend/src/modules/class/formatter.ts b/apps/backend/src/modules/class/formatter.ts index 7c6b053a9..e538b276b 100644 --- a/apps/backend/src/modules/class/formatter.ts +++ b/apps/backend/src/modules/class/formatter.ts @@ -7,16 +7,16 @@ import { } from "../../generated-types/graphql"; import { ClassModule } from "./generated-types/module-types"; -export type IntermediateClass = Omit< - ClassModule.Class, - "course" | "term" | "primarySection" | "sections" -> & { +interface Relationships { course: null; term: null; primarySection: null; sections: null; gradeDistribution: null; -}; +} + +export type IntermediateClass = Omit & + Relationships; export const formatDate = (date?: string | number | Date | null) => { if (!date) return date; diff --git a/apps/backend/src/modules/course/formatter.ts b/apps/backend/src/modules/course/formatter.ts index f3e070f75..a5e0873c4 100644 --- a/apps/backend/src/modules/course/formatter.ts +++ b/apps/backend/src/modules/course/formatter.ts @@ -9,15 +9,18 @@ import { import { formatDate } from "../class/formatter"; import { CourseModule } from "./generated-types/module-types"; -export type IntermediateCourse = Omit< - CourseModule.Course, - "classes" | "crossListing" | "requiredCourses" | "gradeDistribution" -> & { +interface Relationships { classes: null; crossListing: string[]; requiredCourses: string[]; gradeDistribution: null; -}; +} + +export type IntermediateCourse = Omit< + CourseModule.Course, + keyof Relationships +> & + Relationships; export function formatCourse(course: CourseType) { return { diff --git a/apps/backend/src/modules/grade-distribution/controller.ts b/apps/backend/src/modules/grade-distribution/controller.ts index 3acc77f8c..498def61f 100644 --- a/apps/backend/src/modules/grade-distribution/controller.ts +++ b/apps/backend/src/modules/grade-distribution/controller.ts @@ -138,21 +138,23 @@ export const points: { [key: string]: number } = { D: 1, "D-": 0.7, "D+": 1.3, + F: 0, }; export const getAverageGrade = (distribution: Grade[]) => { const total = distribution.reduce((acc, { letter, count }) => { - if (points[letter]) return acc + count; + if (Object.keys(points).includes(letter)) return acc + count; - // Ignore letters not included in grade point average + // Ignore letters not included in GPA return acc; }, 0); - // For distributions without a grade point average, return null + // For distributions without a GPA, return null if (total === 0) return null; const weightedTotal = distribution.reduce((acc, { letter, count }) => { - if (points[letter]) return points[letter] * count + acc; + if (Object.keys(points).includes(letter)) + return points[letter] * count + acc; return acc; }, 0); diff --git a/apps/backend/src/modules/schedule/formatter.ts b/apps/backend/src/modules/schedule/formatter.ts index 4fc2b9ef4..0eae73954 100644 --- a/apps/backend/src/modules/schedule/formatter.ts +++ b/apps/backend/src/modules/schedule/formatter.ts @@ -2,13 +2,16 @@ import { ScheduleType } from "@repo/common"; import { ScheduleModule } from "./generated-types/module-types"; +interface Relationships { + classes: ScheduleModule.SelectedClassInput[]; + term: null; +} + export type IntermediateSchedule = Omit< ScheduleModule.Schedule, - "term" | "classes" -> & { - term: null; - classes: ScheduleModule.SelectedClassInput[]; -}; + keyof Relationships +> & + Relationships; export const formatSchedule = async (schedule: ScheduleType) => { return { diff --git a/apps/backend/src/modules/user/formatter.ts b/apps/backend/src/modules/user/formatter.ts index 8be92bc6c..0ccb6ca37 100644 --- a/apps/backend/src/modules/user/formatter.ts +++ b/apps/backend/src/modules/user/formatter.ts @@ -2,13 +2,13 @@ import { UserType } from "@repo/common"; import { UserModule } from "./generated-types/module-types"; -export type IntermediateUser = Omit< - UserModule.User, - "bookmarkedClasses" | "bookmarkedCourses" -> & { +interface Relationships { bookmarkedCourses: UserModule.BookmarkedCourseInput[]; bookmarkedClasses: UserModule.BookmarkedClassInput[]; -}; +} + +export type IntermediateUser = Omit & + Relationships; export const formatUser = (user: UserType) => { return { diff --git a/apps/backend/src/scripts/update-catalog.ts b/apps/backend/src/scripts/update-catalog.ts index e9a136417..60c96e232 100644 --- a/apps/backend/src/scripts/update-catalog.ts +++ b/apps/backend/src/scripts/update-catalog.ts @@ -29,20 +29,21 @@ const queryPage = async ( key: string, url: string, field: string, + page: number, params?: Record ) => { let retries = 3; - console.log("Querying SIS API page..."); - console.log(`URL: ${url}`); - if (params) console.log(`Params: ${params.toString()}`); + // console.log("Querying SIS API page..."); + // console.log(`URL: ${url}`); + // if (params) console.log(`Params: ${params.toString()}`); while (retries > 0) { try { const _params = new URLSearchParams({ - "page-number": "1", - "page-size": "100", ...params, + "page-number": page.toString(), + "page-size": "100", }); const response = await fetch(`${url}?${_params}`, { @@ -62,17 +63,17 @@ const queryPage = async ( ? data.apiResponse.response[field] : data.response[field]; } catch (error) { - console.log(`Unexpected error querying SIS API. Error: "${error}"`); + // console.log(`Unexpected error querying SIS API. Error: "${error}"`); if (retries === 0) { - console.log(`Too many errors querying SIS API. Terminating update...`); + // console.log(`Too many errors querying SIS API. Terminating update...`); break; } retries--; - console.log(`Retrying...`); + // console.log(`Retrying...`); continue; } @@ -88,74 +89,31 @@ const queryPages = async ( field: string, params?: Record ) => { - let page = 1; - let retries = 3; - const values: V[] = []; - console.log("Querying SIS API pages..."); - console.log(`URL: ${url}`); - if (params) console.log(`Params: ${params}`); - - while (retries > 0) { - try { - const _params = new URLSearchParams({ - "page-number": page.toString(), - "page-size": "100", - ...params, - }); - - const response = await fetch(`${url}?${_params}`, { - headers: { - app_id: id, - app_key: key, - }, - }); - - if (response.status !== 200) throw new Error(response.statusText); - - const data = (await response.json()) as - | DeprecatedSISResponse - | SISResponse; - - const _values = data.apiResponse - ? data.apiResponse.response[field] - : data.response[field]; + // Query courses in batches of 50 + const queryBatchSize = 50; + let page = 1; - values.push(..._values); + while (values.length % 100 === 0) { + console.log(`Querying ${queryBatchSize} pages from page ${page}...`); - if (_values.length < 100) { - console.log( - `No more data found on page ${page}. Terminating update...` - ); + const promises = []; - break; - } - } catch (error) { - console.log(`Unexpected error querying SIS API. Error: "${error}"`); - - if (retries === 0) { - console.log(`Too many errors querying SIS API. Terminating update...`); + for (let i = 0; i < queryBatchSize; i++) { + promises.push(queryPage(id, key, url, field, page + i, params)); + } - break; - } + const results = await Promise.all(promises); + const flattenedResults = results.flat(); - retries--; + if (flattenedResults.length === 0) break; - console.log(`Retrying...`); + values.push(...flattenedResults); - continue; - } - - page++; + page += queryBatchSize; } - console.log( - `Finished querying SIS API. Received ${values.length} objects in ${ - page - } pages.` - ); - return values; }; @@ -166,26 +124,26 @@ const updateCourses = async () => { config.sis.COURSE_APP_ID, config.sis.COURSE_APP_KEY, SIS_COURSE_URL, - "courses", - { - "status-code": "ACTIVE", - } + "courses" ); - const operations = courses.map((course) => ({ - replaceOne: { - filter: { classDisplayName: course.classDisplayName }, - replacement: course, - upsert: true, - }, - })); + console.log(`Received ${courses.length} courses from SIS API.`); - const { upsertedCount, modifiedCount } = - await CourseModel.bulkWrite(operations); + // Remove all courses + await CourseModel.deleteMany({}); - console.log( - `Completed updating database with new course data. Created ${upsertedCount} and updated ${modifiedCount} course objects.` - ); + // Insert courses in batches of 5000 + const insertBatchSize = 5000; + + for (let i = 0; i < courses.length; i += insertBatchSize) { + const batch = courses.slice(i, i + insertBatchSize); + + console.log(`Inserting batch ${i / insertBatchSize + 1}...`); + + await CourseModel.insertMany(batch, { ordered: false }); + } + + console.log(`Completed updating database with new course data.`); }; const updateClasses = async (currentTerms: TermType[]) => { @@ -207,22 +165,27 @@ const updateClasses = async (currentTerms: TermType[]) => { classes.push(...termClasses); } - console.log("Updating database with new class data..."); + console.log(`Received ${classes.length} classes from SIS API.`); - const operations = classes.map((_class) => ({ - replaceOne: { - filter: { displayName: _class.displayName }, - replacement: _class, - upsert: true, + // Remove all classes + await ClassModel.deleteMany({ + "session.term.name": { + $in: currentTerms.map((term) => term.name), }, - })); + }); - const { upsertedCount, modifiedCount } = - await ClassModel.bulkWrite(operations); + // Split classes into batches of 5000 + const batchSize = 5000; - console.log( - `Completed updating database with new class data. Created ${upsertedCount} and updated ${modifiedCount} class objects.` - ); + for (let i = 0; i < classes.length; i += batchSize) { + const batch = classes.slice(i, i + batchSize); + + console.log(`Inserting batch ${i / batchSize + 1}...`); + + await ClassModel.insertMany(batch, { ordered: false }); + } + + console.log(`Completed updating database with new class data.`); }; const updateSections = async (currentTerms: TermType[]) => { @@ -231,7 +194,7 @@ const updateSections = async (currentTerms: TermType[]) => { for (const term of currentTerms) { console.log(`Updating sections for ${term.name}...`); - const termClasses = await queryPages( + const termSections = await queryPages( config.sis.CLASS_APP_ID, config.sis.CLASS_APP_KEY, SIS_SECTION_URL, @@ -239,25 +202,30 @@ const updateSections = async (currentTerms: TermType[]) => { { "term-id": term.id } ); - sections.push(...termClasses); + sections.push(...termSections); } - console.log("Updating database with new section data..."); + console.log(`Received ${sections.length} sections from SIS API.`); - const operations = sections.map((section) => ({ - replaceOne: { - filter: { displayName: section.displayName }, - replacement: section, - upsert: true, + // Remove all sections + await SectionModel.deleteMany({ + "class.session.term.name": { + $in: currentTerms.map((term) => term.name), }, - })); + }); - const { upsertedCount, modifiedCount } = - await SectionModel.bulkWrite(operations); + // Split sections into batches of 5000 + const batchSize = 5000; - console.log( - `Completed updating database with new section data. Created ${upsertedCount} and updated ${modifiedCount} section objects.` - ); + for (let i = 0; i < sections.length; i += batchSize) { + const batch = sections.slice(i, i + batchSize); + + console.log(`Inserting batch ${i / batchSize + 1}...`); + + await SectionModel.insertMany(batch, { ordered: false }); + } + + console.log(`Completed updating database with new section data.`); }; const updateTerms = async () => { @@ -269,6 +237,7 @@ const updateTerms = async () => { config.sis.TERM_APP_KEY, SIS_TERM_URL, "terms", + 1, { "temporal-position": "Next", } @@ -285,6 +254,7 @@ const updateTerms = async () => { config.sis.TERM_APP_KEY, SIS_TERM_URL, "terms", + 1, { "temporal-position": "Previous", "as-of-date": currentTerm.beginDate as unknown as string, @@ -294,22 +264,23 @@ const updateTerms = async () => { if (currentTerm) terms.push(currentTerm); } - console.log("Updating database with new term data..."); + console.log(`Received ${terms.length} terms from SIS API.`); - const operations = terms.map((term) => ({ - replaceOne: { - filter: { name: term.name }, - replacement: term, - upsert: true, - }, - })); + // Remove all terms + await TermModel.deleteMany({}); - const { upsertedCount, modifiedCount } = - await TermModel.bulkWrite(operations); + // Split terms into batches of 5000 + const batchSize = 5000; - console.log( - `Completed updating database with new term data. Created ${upsertedCount} and updated ${modifiedCount} term objects.` - ); + for (let i = 0; i < terms.length; i += batchSize) { + const batch = terms.slice(i, i + batchSize); + + console.log(`Inserting batch ${i / batchSize + 1}...`); + + await TermModel.insertMany(batch, { ordered: false }); + } + + console.log(`Completed updating database with new term data.`); }; const initialize = async () => { @@ -323,7 +294,7 @@ const initialize = async () => { await updateCourses(); const currentTerms = await TermModel.find({ - temporalPosition: "Current", + temporalPosition: { $or: ["Current", "Next"] }, }).lean(); console.log("\n=== UPDATE CLASSES ==="); diff --git a/apps/frontend/src/components/Class/Grades/index.tsx b/apps/frontend/src/components/Class/Grades/index.tsx index 0eaa583c9..4a382814d 100644 --- a/apps/frontend/src/components/Class/Grades/index.tsx +++ b/apps/frontend/src/components/Class/Grades/index.tsx @@ -1,78 +1,104 @@ +import { useMemo } from "react"; + import { Bar, BarChart, CartesianGrid, - LabelList, Legend, ResponsiveContainer, + Tooltip, XAxis, } from "recharts"; +import useClass from "@/hooks/useClass"; +import { Grade } from "@/lib/api"; + import styles from "./Grades.module.scss"; -const data = [ - { - grade: "A", - percentage: 20, - average: 25, - }, - { - grade: "B", - percentage: 15, - average: 20, - }, - { - grade: "C", - percentage: 10, - average: 15, - }, - { - grade: "D", - percentage: 5, - average: 10, - }, - { - grade: "F", - percentage: 2.5, - average: 5, - }, - { - grade: "Pass", - percentage: 35, - average: 20, - }, - { - grade: "Not pass", - percentage: 17.5, - average: 5, - }, +export const points: { [key: string]: number } = { + A: 4, + "A-": 3.7, + "A+": 4, + B: 3, + "B-": 2.7, + "B+": 3.3, + C: 2, + "C-": 1.7, + "C+": 2.3, + D: 1, + "D-": 0.7, + "D+": 1.3, + F: 0, +}; + +const letters = [ + "A+", + "A", + "A-", + "B+", + "B", + "B-", + "C+", + "C", + "C-", + "D", + "F", + "P", + "NP", ]; export default function Grades() { + const { + class: { + gradeDistribution, + course: { gradeDistribution: courseGradeDistribution }, + }, + } = useClass(); + + const data = useMemo(() => { + const getTotal = (distribution: Grade[]) => + distribution.reduce((acc, grade) => acc + grade.count, 0); + + const classTotal = getTotal(gradeDistribution.distribution); + const courseTotal = getTotal(courseGradeDistribution.distribution); + + return letters.map((letter) => { + const getCount = (distribution: Grade[]) => + distribution.find((grade) => grade.letter === letter)?.count || 0; + + return { + letter, + class: getCount(gradeDistribution.distribution) / classTotal, + course: getCount(courseGradeDistribution.distribution) / courseTotal, + }; + }); + }, [gradeDistribution, courseGradeDistribution]); + return (
- - + - - - - - - + + {gradeDistribution.average && ( + + )} + +
diff --git a/apps/frontend/src/lib/api/classes.ts b/apps/frontend/src/lib/api/classes.ts index c6b4b7e96..f001267ce 100644 --- a/apps/frontend/src/lib/api/classes.ts +++ b/apps/frontend/src/lib/api/classes.ts @@ -1,6 +1,6 @@ import { gql } from "@apollo/client"; -import { ICourse } from "."; +import { GradeDistribution, ICourse } from "."; import { ITerm, Semester } from "./terms"; export enum InstructionMethod { @@ -196,6 +196,7 @@ export interface IClass { primarySection: ISection; sections: ISection[]; term: ITerm; + gradeDistribution: GradeDistribution; // Attributes session: string; @@ -237,11 +238,26 @@ export const READ_CLASS = gql` unitsMin gradingBasis finalExam + gradeDistribution { + average + distribution { + letter + count + } + } course { title description + classes { + year + semester + } gradeDistribution { average + distribution { + letter + count + } } academicCareer requirements @@ -310,3 +326,43 @@ export const READ_CLASS = gql` } } `; + +export interface GetClassesResponse { + catalog: ICourse[]; +} + +export const GET_CLASSES = gql` + query GetClasses($year: Int!, $semester: Semester!) { + catalog(year: $year, semester: $semester) { + subject + number + title + gradeDistribution { + average + } + academicCareer + classes { + subject + courseNumber + number + title + unitsMax + unitsMin + finalExam + gradingBasis + primarySection { + component + online + open + enrollCount + enrollMax + waitlistCount + waitlistMax + meetings { + days + } + } + } + } + } +`; diff --git a/apps/frontend/src/lib/api/courses.ts b/apps/frontend/src/lib/api/courses.ts index d9c81067c..5256cd437 100644 --- a/apps/frontend/src/lib/api/courses.ts +++ b/apps/frontend/src/lib/api/courses.ts @@ -97,43 +97,3 @@ export const GET_COURSES = gql` } } `; - -export interface GetClassesResponse { - catalog: ICourse[]; -} - -export const GET_CLASSES = gql` - query GetClasses($year: Int!, $semester: Semester!) { - catalog(year: $year, semester: $semester) { - subject - number - title - gradeDistribution { - average - } - academicCareer - classes { - subject - courseNumber - number - title - unitsMax - unitsMin - finalExam - gradingBasis - primarySection { - component - online - open - enrollCount - enrollMax - waitlistCount - waitlistMax - meetings { - days - } - } - } - } - } -`;