fix: Add MustPutInputFile function

.gitignore

@@ -21,7 +21,6 @@ main
 /test.*
 /config.yml
 
-
 /testing_requests
 /EduOJBackend
 /backend

app/controller/controller_test.go

@@ -9,7 +9,7 @@
 	"hash/fnv"
 	"html/template"
 	"io"
 	"io/ioutil"
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
@@ -144,6 +144,8 @@
 				httpResp := makeResp(req)
 				resp := response.Response{}
 				mustJsonDecode(httpResp, &resp)
+				t.Logf("Expected Status Code: %d", test.statusCode)
+				t.Logf("Actual Status Code: %d", httpResp.StatusCode)
 				assert.Equal(t, test.statusCode, httpResp.StatusCode)
 				assert.Equal(t, test.resp, resp)
 			})

app/controller/problem.go

@@ -397,12 +397,11 @@ func CreateTestCase(c echo.Context) error {
 		InputFileName:  inputFile.Filename,
 		OutputFileName: outputFile.Filename,
 	}
-
 	if err := base.DB.Model(&problem).Association("TestCases").Append(&testCase); err != nil {
 		panic(errors.Wrap(err, "could not create test case"))
 	}
-
-	utils.MustPutObject(inputFile, c.Request().Context(), "problems", fmt.Sprintf("%d/input/%d.in", problem.ID, testCase.ID))
+	// upload to minio
+	utils.MustPutInputFile(*req.Sanitize, inputFile, c.Request().Context(), "problems", fmt.Sprintf("%d/input/%d.in", problem.ID, testCase.ID))
 	utils.MustPutObject(outputFile, c.Request().Context(), "problems", fmt.Sprintf("%d/output/%d.out", problem.ID, testCase.ID))
 
 	return c.JSON(http.StatusCreated, response.CreateTestCaseResponse{
@@ -478,7 +477,6 @@ func UpdateTestCase(c echo.Context) error {
 	if err, ok := utils.BindAndValidate(&req, c); !ok {
 		return err
 	}
-
 	inputFile, err := c.FormFile("input_file")
 	if err != nil && err != http.ErrMissingFile && err.Error() != "request Content-Type isn't multipart/form-data" {
 		panic(errors.Wrap(err, "could not read input file"))
@@ -489,7 +487,7 @@ func UpdateTestCase(c echo.Context) error {
 	}
 
 	if inputFile != nil {
-		utils.MustPutObject(inputFile, c.Request().Context(), "problems", fmt.Sprintf("%d/input/%d.in", problem.ID, testCase.ID))
+		utils.MustPutInputFile(*req.Sanitize, inputFile, c.Request().Context(), "problems", fmt.Sprintf("%d/input/%d.in", problem.ID, testCase.ID))
 		testCase.InputFileName = inputFile.Filename
 	}
 	if outputFile != nil {

app/controller/problem_test.go

@@ -1036,6 +1036,7 @@ func TestCreateProblem(t *testing.T) {
 				assert.Equal(t, test.req.TimeLimit, databaseProblem.TimeLimit)
 				assert.Equal(t, strings.Split(test.req.LanguageAllowed, ","), []string(databaseProblem.LanguageAllowed))
 				assert.Equal(t, test.req.CompareScriptName, databaseProblem.CompareScriptName)
+				//assert.Equal(t, *test.req.Sanitize, databaseProblem.Sanitize)
 				assert.Equal(t, *test.req.Public, databaseProblem.Public)
 				assert.Equal(t, *test.req.Privacy, databaseProblem.Privacy)
 				// response == database
@@ -1055,6 +1056,8 @@ func TestCreateProblem(t *testing.T) {
 					storageContent := getObjectContent(t, "problems", fmt.Sprintf("%d/attachment", databaseProblem.ID))
 					expectedContent, err := ioutil.ReadAll(test.attachment.reader)
 					assert.NoError(t, err)
+					t.Logf("Expected Content: %+v", expectedContent)
+					t.Logf("Storage Content: %+v", storageContent)
 					assert.Equal(t, expectedContent, storageContent)
 					assert.Equal(t, test.attachment.fileName, databaseProblem.AttachmentFileName)
 				} else {
@@ -1734,8 +1737,9 @@ func TestCreateTestCase(t *testing.T) {
 				newFileContent("input_file", "test_create_test_case_non_existing_problem.in", inputTextBase64),
 				newFileContent("output_file", "test_create_test_case_non_existing_problem.out", outputTextBase64),
 			}, map[string]string{
-				"score":  "100",
-				"sample": "true",
+				"score":    "100",
+				"sample":   "true",
+				"sanitize": "false",
 			}),
 			reqOptions: []reqOption{
 				applyAdminUser,
@@ -1750,8 +1754,9 @@ func TestCreateTestCase(t *testing.T) {
 			req: addFieldContentSlice([]reqContent{
 				newFileContent("output_file", "test_create_test_case_lack_input_file.out", outputTextBase64),
 			}, map[string]string{
-				"score":  "100",
-				"sample": "true",
+				"score":    "100",
+				"sample":   "true",
+				"sanitize": "false",
 			}),
 			reqOptions: []reqOption{
 				headerOption{
@@ -1768,8 +1773,9 @@ func TestCreateTestCase(t *testing.T) {
 			req: addFieldContentSlice([]reqContent{
 				newFileContent("input_file", "test_create_test_case_lack_output_file.in", inputTextBase64),
 			}, map[string]string{
-				"score":  "100",
-				"sample": "true",
+				"score":    "100",
+				"sample":   "true",
+				"sanitize": "false",
 			}),
 			reqOptions: []reqOption{
 				headerOption{
@@ -1784,8 +1790,9 @@ func TestCreateTestCase(t *testing.T) {
 			method: "POST",
 			path:   base.Echo.Reverse("problem.createTestCase", problem.ID),
 			req: addFieldContentSlice([]reqContent{}, map[string]string{
-				"score":  "100",
-				"sample": "true",
+				"score":    "100",
+				"sample":   "true",
+				"sanitize": "false",
 			}),
 			reqOptions: []reqOption{
 				headerOption{
@@ -1803,8 +1810,9 @@ func TestCreateTestCase(t *testing.T) {
 				newFileContent("input_file", "test_create_test_case_permission_denied.in", inputTextBase64),
 				newFileContent("output_file", "test_create_test_case_permission_denied.out", outputTextBase64),
 			}, map[string]string{
-				"score":  "100",
-				"sample": "true",
+				"score":    "100",
+				"sample":   "true",
+				"sanitize": "false",
 			}),
 			reqOptions: []reqOption{
 				applyNormalUser,
@@ -1822,8 +1830,9 @@ func TestCreateTestCase(t *testing.T) {
 			newFileContent("input_file", "test_create_test_case_success.in", inputTextBase64),
 			newFileContent("output_file", "test_create_test_case_success.out", outputTextBase64),
 		}, map[string]string{
-			"score":  "100",
-			"sample": "true",
+			"score":    "100",
+			"sample":   "true",
+			"sanitize": "false",
 		}), headerOption{
 			"Set-User-For-Test": {fmt.Sprintf("%d", user.ID)},
 		})
@@ -2066,14 +2075,16 @@ func TestUpdateTestCase(t *testing.T) {
 	problem := createProblemForTest(t, "update_test_case", 0, nil, user)
 	boolTrue := true
 
+	boolFalse := false
 	failTests := []failTest{
 		{
 			name:   "NonExistingProblem",
 			method: "PUT",
 			path:   base.Echo.Reverse("problem.updateTestCase", -1, 1),
 			req: request.UpdateTestCaseRequest{
-				Score:  100,
-				Sample: &boolTrue,
+				Score:    100,
+				Sample:   &boolTrue,
+				Sanitize: &boolFalse,
 			},
 			reqOptions: []reqOption{
 				applyAdminUser,
@@ -2086,8 +2097,9 @@ func TestUpdateTestCase(t *testing.T) {
 			method: "PUT",
 			path:   base.Echo.Reverse("problem.updateTestCase", problem.ID, -1),
 			req: request.UpdateTestCaseRequest{
-				Score:  100,
-				Sample: &boolTrue,
+				Score:    100,
+				Sample:   &boolTrue,
+				Sanitize: &boolFalse,
 			},
 			reqOptions: []reqOption{
 				headerOption{
@@ -2106,8 +2118,9 @@ func TestUpdateTestCase(t *testing.T) {
 			method: "PUT",
 			path:   base.Echo.Reverse("problem.updateTestCase", problem.ID, 1),
 			req: request.UpdateTestCaseRequest{
-				Score:  100,
-				Sample: &boolTrue,
+				Score:    100,
+				Sample:   &boolTrue,
+				Sanitize: &boolFalse,
 			},
 			reqOptions: []reqOption{
 				applyNormalUser,
@@ -2128,7 +2141,7 @@ func TestUpdateTestCase(t *testing.T) {
 		{
 			name: "SuccessWithoutUpdatingFile",
 			originalData: testCaseData{
-				Score:      0,
+				Score:      100,
 				Sample:     false,
 				InputFile:  newFileContent("input_file", "test_update_test_case_1.in", inputTextBase64),
 				OutputFile: newFileContent("output_file", "test_update_test_case_1.out", outputTextBase64),
@@ -2229,10 +2242,12 @@ func TestUpdateTestCase(t *testing.T) {
 				if test.updatedData.OutputFile != nil {
 					reqContentSlice = append(reqContentSlice, test.updatedData.OutputFile)
 				}
+				sanitizeValue := false
 				req := makeReq(t, "PUT", base.Echo.Reverse("problem.updateTestCase", problem.ID, testCase.ID), addFieldContentSlice(
 					reqContentSlice, map[string]string{
-						"score":  fmt.Sprintf("%d", test.updatedData.Score),
-						"sample": fmt.Sprintf("%t", test.updatedData.Sample),
+						"score":    fmt.Sprintf("%d", test.updatedData.Score),
+						"sample":   fmt.Sprintf("%t", test.updatedData.Sample),
+						"sanitize": fmt.Sprintf("%t", sanitizeValue),
 					}), headerOption{
 					"Set-User-For-Test": {fmt.Sprintf("%d", user.ID)},
 				})

app/request/problem.go

@@ -7,6 +7,8 @@ type CreateProblemRequest struct {
 	Public  *bool `json:"public" form:"public" query:"public" validate:"required"`
 	Privacy *bool `json:"privacy" form:"privacy" query:"privacy" validate:"required"`
 
+	//Sanitize *bool `json:"sanitize" form:"sanitize" query:"sanitize" validate:"required"`
+
 	MemoryLimit       uint64 `json:"memory_limit" form:"memory_limit" query:"memory_limit" validate:"required"`                     // Byte
 	TimeLimit         uint   `json:"time_limit" form:"time_limit" query:"time_limit" validate:"required"`                           // ms
 	LanguageAllowed   string `json:"language_allowed" form:"language_allowed" query:"language_allowed" validate:"required,max=255"` // E.g.    cpp,c,java,python
@@ -36,8 +38,9 @@ type DeleteProblemRequest struct {
 }
 
 type CreateTestCaseRequest struct {
-	Score  uint  `json:"score" form:"score" query:"score"` // 0 for 平均分配
-	Sample *bool `json:"sample" form:"sample" query:"sample" validate:"required"`
+	Score    uint  `json:"score" form:"score" query:"score"`
+	Sample   *bool `json:"sample" form:"sample" query:"sample" validate:"required"`
+	Sanitize *bool `json:"sanitize" form:"sanitize" query:"sanitize" validate:"required"`
 	// input_file(required)
 	// output_file(required)
 }
@@ -49,8 +52,9 @@ type GetTestCaseOutputFileRequest struct {
 }
 
 type UpdateTestCaseRequest struct {
-	Score  uint  `json:"score" form:"score" query:"score"` // 0 for 平均分配
-	Sample *bool `json:"sample" form:"sample" query:"sample" validate:"required"`
+	Score    uint  `json:"score" form:"score" query:"score"`
+	Sample   *bool `json:"sample" form:"sample" query:"sample" validate:"required"`
+	Sanitize *bool `json:"sanitize" form:"sanitize" query:"sanitize" validate:"required"`
 	// input_file(optional)
 	// output_file(optional)
 }

base/utils/helpers.go

@@ -1,9 +1,13 @@
 package utils
 
 import (
+	"bufio"
 	"context"
+	"fmt"
 	"mime/multipart"
 	"net/http"
+	"os"
+	"strings"
 
 	"github.com/EduOJ/backend/app/response"
 	"github.com/EduOJ/backend/base"
@@ -58,6 +62,61 @@ func MustPutObject(object *multipart.FileHeader, ctx context.Context, bucket str
 	}
 }
 
+func MustPutInputFile(sanitize bool, object *multipart.FileHeader, ctx context.Context, bucket string, path string) {
+	src, err := object.Open()
+	if err != nil {
+		panic(err)
+	}
+	defer src.Close()
+
+	var fileSize int64
+
+	if sanitize {
+		scanner := bufio.NewScanner(src)
+		tempFile, err := os.CreateTemp("", "tempFile*.txt")
+		if err != nil {
+			panic(err)
+		}
+		writer := bufio.NewWriter(tempFile)
+		for scanner.Scan() {
+			line := strings.ReplaceAll(scanner.Text(), "\r\n", "\n") // replace '\r\n' to '\n'
+
+			if !strings.HasSuffix(line, "\n") {
+				line += "\n"
+			}
+			_, err := fmt.Fprint(writer, line)
+			if err != nil {
+				panic(err)
+			}
+		}
+		if err := scanner.Err(); err != nil {
+			panic(err)
+		}
+		if err := writer.Flush(); err != nil {
+			panic(err)
+		}
+
+		fileInfo, err := tempFile.Stat()
+		if err != nil {
+			panic(err)
+		}
+		fileSize = fileInfo.Size()
+
+		src, err = os.Open(tempFile.Name())
+		if err != nil {
+			panic(err)
+		}
+		defer src.Close()
+	} else {
+		fileSize = object.Size
+	}
+
+	_, err = base.Storage.PutObject(ctx, bucket, path, src, fileSize, minio.PutObjectOptions{})
+	if err != nil {
+		panic(errors.Wrap(err, "could write file to s3 storage."))
+	}
+}
+
 func MustGetObject(c echo.Context, bucket string, path string) *minio.Object {
 	object, err := base.Storage.GetObject(c.Request().Context(), bucket, path, minio.GetObjectOptions{})
 	if err != nil {

base/validator/translations/zh/zh.go

@@ -52,6 +52,7 @@ var FieldTranslations = map[string]string{
 	"Tried":              "选取尝试过题目",
 	"Passed":             "选取通过题目",
 	"Token":              "验证码",
+	"Sanitize":           "是否清洗数据",
 }
 
 // RegisterDefaultTranslations registers a set of default translations