From de92cffdb30263563c323e5a06d8a0ff4dee6408 Mon Sep 17 00:00:00 2001 From: Filippo Casarin Date: Tue, 2 Apr 2024 19:33:36 +0200 Subject: [PATCH] validators are owned by subtasks --- src/tools/find_bad_case/dag.rs | 2 +- .../src/ioi/dag/input_validator.rs | 3 +- .../src/ioi/format/italian_yaml/cases_gen.rs | 219 +++++++----------- .../src/ioi/format/italian_yaml/gen_gen.rs | 3 +- .../ioi/format/italian_yaml/static_inputs.rs | 2 +- task-maker-format/src/ioi/mod.rs | 8 +- task-maker-format/tests/ioi_task_execute.rs | 3 - task-maker-format/tests/utils.rs | 3 - 8 files changed, 94 insertions(+), 149 deletions(-) diff --git a/src/tools/find_bad_case/dag.rs b/src/tools/find_bad_case/dag.rs index a8f812eb3..dcc6d354b 100644 --- a/src/tools/find_bad_case/dag.rs +++ b/src/tools/find_bad_case/dag.rs @@ -75,7 +75,6 @@ pub fn patch_task_for_batch( let testcase = TestcaseInfo::new( testcase_id, input_generator, - task.input_validator_generator.generate(Some(0)), testcase_template.output_generator.clone(), ); @@ -97,6 +96,7 @@ pub fn patch_task_for_batch( max_score: 100.0, testcases, is_default: false, + input_validator: task.input_validator_generator.generate(Some(0)), ..Default::default() }; task.subtasks.insert(0, subtask); diff --git a/task-maker-format/src/ioi/dag/input_validator.rs b/task-maker-format/src/ioi/dag/input_validator.rs index 77fd8d0fe..ae9fae387 100644 --- a/task-maker-format/src/ioi/dag/input_validator.rs +++ b/task-maker-format/src/ioi/dag/input_validator.rs @@ -18,9 +18,10 @@ pub const TM_VALIDATION_FILE_NAME: &str = "tm_validation_file"; /// An input file validator is responsible for checking that the input file follows the format and /// constraints defined by the task. -#[derive(Debug, Clone, Serialize, Deserialize, TypeScriptify)] +#[derive(Debug, Clone, Serialize, Deserialize, TypeScriptify, Default)] pub enum InputValidator { /// Skip the validation and assume the input file is valid. + #[default] AssumeValid, /// Use a custom command to check if the input file is valid. The command should exit with /// non-zero return code if and only if the input is invalid. diff --git a/task-maker-format/src/ioi/format/italian_yaml/cases_gen.rs b/task-maker-format/src/ioi/format/italian_yaml/cases_gen.rs index a29073634..39202f568 100644 --- a/task-maker-format/src/ioi/format/italian_yaml/cases_gen.rs +++ b/task-maker-format/src/ioi/format/italian_yaml/cases_gen.rs @@ -332,8 +332,6 @@ where self.result.push(TaskInputEntry::Testcase(TestcaseInfo::new( self.testcase_id, generator, - self.get_validator(&variables) - .context("Cannot get testcase validator")?, (self.get_output_gen)(self.testcase_id), ))); self.testcase_id += 1; @@ -428,15 +426,28 @@ where /// Parse a `:VAL` command. fn parse_val(&mut self, line: Pair) -> Result<(), Error> { let line: Vec<_> = line.into_inner().collect(); - CasesGen::::process_gen_val( - line, - &self.task_dir, - self.subtask_id, - &mut self.default_validator, - &mut self.current_validator, - &mut self.validators, - "validator", - )?; + if line.len() == 1 { + let val = self + .get_validator(&self.get_auto_variables()) + .context("Failed to get validator")?; + + // set validator for the last subtask + let Some(TaskInputEntry::Subtask(subtask)) = self.result.last_mut() else { + bail!("The validator must be set after a subtask"); + }; + subtask.input_validator = val; + } else { + CasesGen::::process_gen_val( + line, + &self.task_dir, + self.subtask_id, + &mut self.default_validator, + &mut self.current_validator, + &mut self.validators, + "validator", + )?; + } + Ok(()) } @@ -530,10 +541,11 @@ where .as_deref() .map(|s| s.chars().filter(|&c| c != ' ' && c != '\t').collect()); self.subtask_name = name.clone(); + self.subtask_id += 1; self.result.push(TaskInputEntry::Subtask( #[allow(deprecated)] SubtaskInfo { - id: self.subtask_id, + id: self.subtask_id - 1, name, description, max_score: score, @@ -545,10 +557,10 @@ where ) .ok(), is_default: false, + input_validator: self.get_validator(&self.get_auto_variables())?, ..Default::default() }, )); - self.subtask_id += 1; Ok(()) } @@ -572,8 +584,6 @@ where self.result.push(TaskInputEntry::Testcase(TestcaseInfo::new( self.testcase_id, InputGenerator::StaticFile(path), - self.get_validator(&self.get_auto_variables()) - .context("Cannot get testcase validator")?, (self.get_output_gen)(self.testcase_id), ))); self.testcase_id += 1; @@ -592,16 +602,11 @@ where for arg in &validator.args { // variables may (and should!) start with `$`, remove it before accessing // the `variables` map. - let arg = if let Some(rest) = arg.strip_prefix('$') { - rest - } else { - arg.as_str() - }; - if let Some(value) = variables.get(arg) { - args.push(value.clone()); - } else { + let arg = arg.strip_prefix('$').unwrap_or(arg); + let Some(value) = variables.get(arg) else { bail!("Unknown variable in validator arguments: ${}", arg); - } + }; + args.push(value.clone()); } args }; @@ -641,7 +646,6 @@ where let mut vars = HashMap::new(); vars.insert("INPUT".to_string(), TM_VALIDATION_FILE_NAME.to_string()); vars.insert("ST_NUM".to_string(), (self.subtask_id - 1).to_string()); - vars.insert("TC_NUM".to_string(), self.testcase_id.to_string()); if let Some(name) = &self.subtask_name { vars.insert("ST_NAME".to_string(), name.clone()); } @@ -1285,6 +1289,63 @@ mod tests { } } + #[test] + fn test_add_subtask_with_default_val() { + let gen = TestHelper::new() + .add_file("gen/generator.py") + .add_file("gen/val.py") + .cases_gen( + ":GEN gen gen/generator.py\n:VAL default gen/val.py\n:SUBTASK 42\n:RUN gen 4 5 6", + ) + .unwrap(); + assert_eq!(gen.subtask_id, 1); + assert_eq!(gen.testcase_id, 1); + assert_eq!(gen.result.len(), 2); + let subtask = &gen.result[0]; + let TaskInputEntry::Subtask(subtask) = subtask else { + panic!("Expecting a subtask, got: {:?}", subtask); + }; + assert_eq!(subtask.id, 0); + if let InputValidator::Custom(_, args) = &subtask.input_validator { + assert_eq!(args.len(), 2); + assert_eq!(args[1], "0"); + } else { + panic!( + "Expecting an AssumeValid but got: {:?}", + subtask.input_validator + ); + } + } + + #[test] + fn test_subtask_validator_args_custom() { + let gen = TestHelper::new() + .add_file("gen/generator.py") + .add_file("gen/val.py") + .cases_gen(":GEN default gen/generator.py N M seed\n:VAL default gen/val.py $INPUT $ST_NUM\n:SUBTASK 42\n1 2 3") + .unwrap(); + assert_eq!(gen.subtask_id, 1); + assert_eq!(gen.testcase_id, 1); + assert_eq!(gen.result.len(), 2); + let subtask = &gen.result[0]; + let TaskInputEntry::Subtask(subtask) = subtask else { + panic!("Expecting a subtask, got: {:?}", subtask); + }; + assert_eq!(subtask.id, 0); + if let InputValidator::Custom(source, args) = &subtask.input_validator { + assert_eq!(source.name(), "val.py"); + assert_eq!( + args, + &vec![TM_VALIDATION_FILE_NAME, "0"] + ); + } else { + panic!( + "Expecting a custom validator, got: {:?}", + subtask.input_validator + ); + } + } + /********************** * : COPY *********************/ @@ -1308,13 +1369,6 @@ mod tests { testcase.input_generator ); } - if let InputValidator::AssumeValid = testcase.input_validator { - } else { - panic!( - "Expecting an AssumeValid but got: {:?}", - testcase.input_validator - ); - } } else { panic!("Expecting a testcase, got: {:?}", testcase); } @@ -1358,50 +1412,6 @@ mod tests { testcase.input_generator ); } - if let InputValidator::AssumeValid = testcase.input_validator { - } else { - panic!( - "Expecting an AssumeValid but got: {:?}", - testcase.input_validator - ); - } - } else { - panic!("Expecting a testcase, got: {:?}", testcase); - } - } - - #[test] - fn test_add_run_with_val() { - let gen = TestHelper::new() - .add_file("gen/generator.py") - .add_file("gen/val.py") - .cases_gen( - ":GEN gen gen/generator.py\n:VAL default gen/val.py\n:SUBTASK 42\n:RUN gen 4 5 6", - ) - .unwrap(); - assert_eq!(gen.subtask_id, 1); - assert_eq!(gen.testcase_id, 1); - assert_eq!(gen.result.len(), 2); - let testcase = &gen.result[1]; - if let TaskInputEntry::Testcase(testcase) = testcase { - assert_eq!(testcase.id, 0); - if let InputGenerator::Custom(_, args) = &testcase.input_generator { - assert_eq!(args, &vec!["4", "5", "6"]); - } else { - panic!( - "Expecting a custom generator, got: {:?}", - testcase.input_generator - ); - } - if let InputValidator::Custom(_, args) = &testcase.input_validator { - assert_eq!(args.len(), 2); - assert_eq!(args[1], "0"); - } else { - panic!( - "Expecting an AssumeValid but got: {:?}", - testcase.input_validator - ); - } } else { panic!("Expecting a testcase, got: {:?}", testcase); } @@ -1566,63 +1576,6 @@ mod tests { } } - #[test] - fn test_testcase_validator_args_default() { - let gen = TestHelper::new() - .add_file("gen/generator.py") - .add_file("gen/val.py") - .cases_gen(":GEN default gen/generator.py\n:VAL default gen/val.py\n:SUBTASK 42\n1 2 3") - .unwrap(); - assert_eq!(gen.subtask_id, 1); - assert_eq!(gen.testcase_id, 1); - assert_eq!(gen.result.len(), 2); - let testcase = &gen.result[1]; - if let TaskInputEntry::Testcase(testcase) = testcase { - assert_eq!(testcase.id, 0); - if let InputValidator::Custom(source, args) = &testcase.input_validator { - assert_eq!(source.name(), "val.py"); - assert_eq!(args, &vec![TM_VALIDATION_FILE_NAME, "0"]); - } else { - panic!( - "Expecting a custom validator, got: {:?}", - testcase.input_validator - ); - } - } else { - panic!("Expecting a testcase, got: {:?}", testcase); - } - } - - #[test] - fn test_testcase_validator_args_custom() { - let gen = TestHelper::new() - .add_file("gen/generator.py") - .add_file("gen/val.py") - .cases_gen(":GEN default gen/generator.py N M seed\n:VAL default gen/val.py $N $M $seed $INPUT $TC_NUM $ST_NUM\n:SUBTASK 42\n1 2 3") - .unwrap(); - assert_eq!(gen.subtask_id, 1); - assert_eq!(gen.testcase_id, 1); - assert_eq!(gen.result.len(), 2); - let testcase = &gen.result[1]; - if let TaskInputEntry::Testcase(testcase) = testcase { - assert_eq!(testcase.id, 0); - if let InputValidator::Custom(source, args) = &testcase.input_validator { - assert_eq!(source.name(), "val.py"); - assert_eq!( - args, - &vec!["1", "2", "3", TM_VALIDATION_FILE_NAME, "0", "0"] - ); - } else { - panic!( - "Expecting a custom validator, got: {:?}", - testcase.input_validator - ); - } - } else { - panic!("Expecting a testcase, got: {:?}", testcase); - } - } - #[test] fn test_testcase_valid_constraints() { let gen = TestHelper::new().add_file("gen/generator.py").cases_gen( diff --git a/task-maker-format/src/ioi/format/italian_yaml/gen_gen.rs b/task-maker-format/src/ioi/format/italian_yaml/gen_gen.rs index 834859e94..679ae69a9 100644 --- a/task-maker-format/src/ioi/format/italian_yaml/gen_gen.rs +++ b/task-maker-format/src/ioi/format/italian_yaml/gen_gen.rs @@ -105,6 +105,7 @@ where ) .ok(), is_default: false, + input_validator: get_validator(Some(subtask_id)), ..Default::default() })); subtask_id += 1; @@ -143,7 +144,6 @@ where entries.push(TaskInputEntry::Testcase(TestcaseInfo::new( testcase_count, InputGenerator::StaticFile(task_dir.join(what)), - get_validator(Some(subtask_id - 1)), get_output_gen(testcase_count), ))); testcase_count += 1; @@ -162,7 +162,6 @@ where entries.push(TaskInputEntry::Testcase(TestcaseInfo::new( testcase_count, InputGenerator::Custom(generator.clone(), cmd), - get_validator(Some(subtask_id - 1)), output_generator, ))); testcase_count += 1; diff --git a/task-maker-format/src/ioi/format/italian_yaml/static_inputs.rs b/task-maker-format/src/ioi/format/italian_yaml/static_inputs.rs index e5af9a52b..6826182f3 100644 --- a/task-maker-format/src/ioi/format/italian_yaml/static_inputs.rs +++ b/task-maker-format/src/ioi/format/italian_yaml/static_inputs.rs @@ -40,6 +40,7 @@ where name: Some("static-testcases".into()), max_score: 100.0, is_default: true, + input_validator: (self.get_validator)(Some(0)), ..Default::default() })); } @@ -50,7 +51,6 @@ where Some(TaskInputEntry::Testcase(TestcaseInfo::new( id, InputGenerator::StaticFile(path), - (self.get_validator)(Some(0)), (self.get_output_gen)(id), ))) } else { diff --git a/task-maker-format/src/ioi/mod.rs b/task-maker-format/src/ioi/mod.rs index 86062fad7..3d1e411f5 100644 --- a/task-maker-format/src/ioi/mod.rs +++ b/task-maker-format/src/ioi/mod.rs @@ -165,6 +165,8 @@ pub struct SubtaskInfo { pub max_score: f64, /// The testcases inside this subtask. pub testcases: HashMap, + /// The validator for the input files of this subtask. + pub input_validator: InputValidator, /// The span of the definition of this subtask. pub span: Option, /// Whether this subtask was created automatically since no subtask was present in gen/GEN. @@ -181,8 +183,6 @@ pub struct TestcaseInfo { pub id: TestcaseId, /// The generator of the input file for this testcase. pub input_generator: InputGenerator, - /// The validator of the input file for this testcase. - pub input_validator: InputValidator, /// The generator of the output file for this testcase. pub output_generator: OutputGenerator, /// The generated input file UUID. This is set only after the DAG is built. @@ -313,7 +313,7 @@ impl IOITask { .input_generator .generate_and_bind(eval, subtask.id, testcase.id) .context("Failed to bind input generator")?; - let val_handle = testcase + let val_handle = subtask .input_validator .validate_and_bind( eval, @@ -514,13 +514,11 @@ impl TestcaseInfo { pub fn new( id: TestcaseId, input_generator: InputGenerator, - input_validator: InputValidator, output_generator: OutputGenerator, ) -> Self { Self { id, input_generator, - input_validator, output_generator, input_file: None, official_output_file: None, diff --git a/task-maker-format/tests/ioi_task_execute.rs b/task-maker-format/tests/ioi_task_execute.rs index 7624856e2..b6bfc83d7 100644 --- a/task-maker-format/tests/ioi_task_execute.rs +++ b/task-maker-format/tests/ioi_task_execute.rs @@ -52,9 +52,6 @@ fn test_ioi_task_execute_val() { SourceFile::new(tmpdir.path().join("val.py"), "", "", None, None::).unwrap(); let val = InputValidator::Custom(Arc::new(source), vec![]); task.subtasks - .get_mut(&0) - .unwrap() - .testcases .get_mut(&0) .unwrap() .input_validator = val; diff --git a/task-maker-format/tests/utils.rs b/task-maker-format/tests/utils.rs index e2ea31828..055fb3734 100644 --- a/task-maker-format/tests/utils.rs +++ b/task-maker-format/tests/utils.rs @@ -50,7 +50,6 @@ pub fn new_task_with_context(path: &Path) -> IOITask { TestcaseInfo::new( 0, InputGenerator::StaticFile(p.clone()), - InputValidator::AssumeValid, OutputGenerator::StaticFile(p.clone()), ) }); @@ -64,7 +63,6 @@ pub fn new_task_with_context(path: &Path) -> IOITask { TestcaseInfo::new( 1, InputGenerator::StaticFile(p.clone()), - InputValidator::AssumeValid, OutputGenerator::StaticFile(p.clone()), ) }); @@ -72,7 +70,6 @@ pub fn new_task_with_context(path: &Path) -> IOITask { TestcaseInfo::new( 2, InputGenerator::StaticFile(p.clone()), - InputValidator::AssumeValid, OutputGenerator::StaticFile(p), ) });