diff --git a/pom.xml b/pom.xml index 3448e7b3841..cb62623b3d7 100644 --- a/pom.xml +++ b/pom.xml @@ -73,12 +73,12 @@ api_validation dist - rapids-4-spark-tools integration_tests shims shuffle-plugin sql-plugin tests + tools udf-compiler udf-examples diff --git a/rapids-4-spark-tools/.gitignore b/tools/.gitignore similarity index 100% rename from rapids-4-spark-tools/.gitignore rename to tools/.gitignore diff --git a/rapids-4-spark-tools/README.md b/tools/README.md similarity index 99% rename from rapids-4-spark-tools/README.md rename to tools/README.md index 68957c6216e..5ad430c8f96 100644 --- a/rapids-4-spark-tools/README.md +++ b/tools/README.md @@ -31,7 +31,7 @@ Here are 2 options: ```bash git clone https://github.com/NVIDIA/spark-rapids.git cd spark-rapids -mvn -pl .,rapids-4-spark-tools clean verify -DskipTests +mvn -pl .,tools clean verify -DskipTests ``` The jar is generated in below directory : diff --git a/rapids-4-spark-tools/pom.xml b/tools/pom.xml similarity index 100% rename from rapids-4-spark-tools/pom.xml rename to tools/pom.xml diff --git a/rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Analysis.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Analysis.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Analysis.scala rename to tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Analysis.scala diff --git a/rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ClassWarehouse.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ClassWarehouse.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ClassWarehouse.scala rename to tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ClassWarehouse.scala diff --git a/rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CollectInformation.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CollectInformation.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CollectInformation.scala rename to tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CollectInformation.scala diff --git a/rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CompareApplications.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CompareApplications.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CompareApplications.scala rename to tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CompareApplications.scala diff --git a/rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/GenerateDot.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/GenerateDot.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/GenerateDot.scala rename to tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/GenerateDot.scala diff --git a/rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileArgs.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileArgs.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileArgs.scala rename to tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileArgs.scala diff --git a/rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileMain.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileMain.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileMain.scala rename to tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileMain.scala diff --git a/rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileUtils.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileUtils.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileUtils.scala rename to tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileUtils.scala diff --git a/rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/Qualification.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/Qualification.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/Qualification.scala rename to tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/Qualification.scala diff --git a/rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualificationArgs.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualificationArgs.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualificationArgs.scala rename to tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualificationArgs.scala diff --git a/rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualificationMain.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualificationMain.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualificationMain.scala rename to tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualificationMain.scala diff --git a/rapids-4-spark-tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/ApplicationInfo.scala b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/ApplicationInfo.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/ApplicationInfo.scala rename to tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/ApplicationInfo.scala diff --git a/rapids-4-spark-tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/EventsProcessor.scala b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/EventsProcessor.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/EventsProcessor.scala rename to tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/EventsProcessor.scala diff --git a/rapids-4-spark-tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/ToolUtils.scala b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/ToolUtils.scala similarity index 100% rename from rapids-4-spark-tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/ToolUtils.scala rename to tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/ToolUtils.scala diff --git a/rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobandstagemetrics2_expectation.csv b/tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobandstagemetrics2_expectation.csv similarity index 100% rename from rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobandstagemetrics2_expectation.csv rename to tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobandstagemetrics2_expectation.csv diff --git a/rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobandstagemetrics_expectation.csv b/tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobandstagemetrics_expectation.csv similarity index 100% rename from rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobandstagemetrics_expectation.csv rename to tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobandstagemetrics_expectation.csv diff --git a/rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobandstagemetricsmulti_expectation.csv b/tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobandstagemetricsmulti_expectation.csv similarity index 100% rename from rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobandstagemetricsmulti_expectation.csv rename to tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobandstagemetricsmulti_expectation.csv diff --git a/rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetrics_expectation.csv b/tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetrics_expectation.csv similarity index 100% rename from rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetrics_expectation.csv rename to tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetrics_expectation.csv diff --git a/rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg2_expectation.csv b/tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg2_expectation.csv similarity index 100% rename from rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg2_expectation.csv rename to tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg2_expectation.csv diff --git a/rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg_expectation.csv b/tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg_expectation.csv similarity index 100% rename from rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg_expectation.csv rename to tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg_expectation.csv diff --git a/rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsaggmulti_expectation.csv b/tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsaggmulti_expectation.csv similarity index 100% rename from rapids-4-spark-tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsaggmulti_expectation.csv rename to tools/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsaggmulti_expectation.csv diff --git a/rapids-4-spark-tools/src/test/resources/QualificationExpectations/nds_q86_test_expectation.csv b/tools/src/test/resources/QualificationExpectations/nds_q86_test_expectation.csv similarity index 100% rename from rapids-4-spark-tools/src/test/resources/QualificationExpectations/nds_q86_test_expectation.csv rename to tools/src/test/resources/QualificationExpectations/nds_q86_test_expectation.csv diff --git a/rapids-4-spark-tools/src/test/resources/QualificationExpectations/qual_test_missing_sql_end_expectation.csv b/tools/src/test/resources/QualificationExpectations/qual_test_missing_sql_end_expectation.csv similarity index 100% rename from rapids-4-spark-tools/src/test/resources/QualificationExpectations/qual_test_missing_sql_end_expectation.csv rename to tools/src/test/resources/QualificationExpectations/qual_test_missing_sql_end_expectation.csv diff --git a/rapids-4-spark-tools/src/test/resources/QualificationExpectations/qual_test_simple_expectation.csv b/tools/src/test/resources/QualificationExpectations/qual_test_simple_expectation.csv similarity index 100% rename from rapids-4-spark-tools/src/test/resources/QualificationExpectations/qual_test_simple_expectation.csv rename to tools/src/test/resources/QualificationExpectations/qual_test_simple_expectation.csv diff --git a/rapids-4-spark-tools/src/test/resources/QualificationExpectations/truncated_1_end_expectation.csv b/tools/src/test/resources/QualificationExpectations/truncated_1_end_expectation.csv similarity index 100% rename from rapids-4-spark-tools/src/test/resources/QualificationExpectations/truncated_1_end_expectation.csv rename to tools/src/test/resources/QualificationExpectations/truncated_1_end_expectation.csv diff --git a/rapids-4-spark-tools/src/test/resources/log4j.properties b/tools/src/test/resources/log4j.properties similarity index 100% rename from rapids-4-spark-tools/src/test/resources/log4j.properties rename to tools/src/test/resources/log4j.properties diff --git a/rapids-4-spark-tools/src/test/resources/spark-events-profiling/eventlog_minimal_events b/tools/src/test/resources/spark-events-profiling/eventlog_minimal_events similarity index 100% rename from rapids-4-spark-tools/src/test/resources/spark-events-profiling/eventlog_minimal_events rename to tools/src/test/resources/spark-events-profiling/eventlog_minimal_events diff --git a/rapids-4-spark-tools/src/test/resources/spark-events-profiling/rapids_join_eventlog b/tools/src/test/resources/spark-events-profiling/rapids_join_eventlog similarity index 100% rename from rapids-4-spark-tools/src/test/resources/spark-events-profiling/rapids_join_eventlog rename to tools/src/test/resources/spark-events-profiling/rapids_join_eventlog diff --git a/rapids-4-spark-tools/src/test/resources/spark-events-profiling/rapids_join_eventlog2 b/tools/src/test/resources/spark-events-profiling/rapids_join_eventlog2 similarity index 100% rename from rapids-4-spark-tools/src/test/resources/spark-events-profiling/rapids_join_eventlog2 rename to tools/src/test/resources/spark-events-profiling/rapids_join_eventlog2 diff --git a/rapids-4-spark-tools/src/test/resources/spark-events-qualification/dataset_eventlog b/tools/src/test/resources/spark-events-qualification/dataset_eventlog similarity index 100% rename from rapids-4-spark-tools/src/test/resources/spark-events-qualification/dataset_eventlog rename to tools/src/test/resources/spark-events-qualification/dataset_eventlog diff --git a/rapids-4-spark-tools/src/test/resources/spark-events-qualification/dsAndDf_eventlog b/tools/src/test/resources/spark-events-qualification/dsAndDf_eventlog similarity index 100% rename from rapids-4-spark-tools/src/test/resources/spark-events-qualification/dsAndDf_eventlog rename to tools/src/test/resources/spark-events-qualification/dsAndDf_eventlog diff --git a/rapids-4-spark-tools/src/test/resources/spark-events-qualification/join_missing_sql_end b/tools/src/test/resources/spark-events-qualification/join_missing_sql_end similarity index 100% rename from rapids-4-spark-tools/src/test/resources/spark-events-qualification/join_missing_sql_end rename to tools/src/test/resources/spark-events-qualification/join_missing_sql_end diff --git a/rapids-4-spark-tools/src/test/resources/spark-events-qualification/nds_q86_test b/tools/src/test/resources/spark-events-qualification/nds_q86_test similarity index 100% rename from rapids-4-spark-tools/src/test/resources/spark-events-qualification/nds_q86_test rename to tools/src/test/resources/spark-events-qualification/nds_q86_test diff --git a/rapids-4-spark-tools/src/test/resources/spark-events-qualification/truncated_eventlog b/tools/src/test/resources/spark-events-qualification/truncated_eventlog similarity index 100% rename from rapids-4-spark-tools/src/test/resources/spark-events-qualification/truncated_eventlog rename to tools/src/test/resources/spark-events-qualification/truncated_eventlog diff --git a/rapids-4-spark-tools/src/test/resources/spark-events-qualification/udf_dataset_eventlog b/tools/src/test/resources/spark-events-qualification/udf_dataset_eventlog similarity index 100% rename from rapids-4-spark-tools/src/test/resources/spark-events-qualification/udf_dataset_eventlog rename to tools/src/test/resources/spark-events-qualification/udf_dataset_eventlog diff --git a/rapids-4-spark-tools/src/test/resources/spark-events-qualification/udf_func_eventlog b/tools/src/test/resources/spark-events-qualification/udf_func_eventlog similarity index 100% rename from rapids-4-spark-tools/src/test/resources/spark-events-qualification/udf_func_eventlog rename to tools/src/test/resources/spark-events-qualification/udf_func_eventlog diff --git a/rapids-4-spark-tools/src/test/scala/com/nvidia/spark/rapids/tool/ToolTestUtils.scala b/tools/src/test/scala/com/nvidia/spark/rapids/tool/ToolTestUtils.scala similarity index 100% rename from rapids-4-spark-tools/src/test/scala/com/nvidia/spark/rapids/tool/ToolTestUtils.scala rename to tools/src/test/scala/com/nvidia/spark/rapids/tool/ToolTestUtils.scala diff --git a/rapids-4-spark-tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/AnalysisSuite.scala b/tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/AnalysisSuite.scala similarity index 100% rename from rapids-4-spark-tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/AnalysisSuite.scala rename to tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/AnalysisSuite.scala diff --git a/rapids-4-spark-tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/ApplicationInfoSuite.scala b/tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/ApplicationInfoSuite.scala similarity index 100% rename from rapids-4-spark-tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/ApplicationInfoSuite.scala rename to tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/ApplicationInfoSuite.scala diff --git a/rapids-4-spark-tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/GenerateDotSuite.scala b/tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/GenerateDotSuite.scala similarity index 100% rename from rapids-4-spark-tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/GenerateDotSuite.scala rename to tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/GenerateDotSuite.scala diff --git a/rapids-4-spark-tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/QualificationInfoUtils.scala b/tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/QualificationInfoUtils.scala similarity index 100% rename from rapids-4-spark-tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/QualificationInfoUtils.scala rename to tools/src/test/scala/com/nvidia/spark/rapids/tool/profiling/QualificationInfoUtils.scala diff --git a/rapids-4-spark-tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala b/tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala similarity index 100% rename from rapids-4-spark-tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala rename to tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala diff --git a/rapids-4-spark-tools/src/test/scala/org/apache/spark/sql/TrampolineUtil.scala b/tools/src/test/scala/org/apache/spark/sql/TrampolineUtil.scala similarity index 100% rename from rapids-4-spark-tools/src/test/scala/org/apache/spark/sql/TrampolineUtil.scala rename to tools/src/test/scala/org/apache/spark/sql/TrampolineUtil.scala