Skip to content

Commit

Permalink
Refactor file_io_s3_test.rs (apache#455)
Browse files Browse the repository at this point in the history
  • Loading branch information
fqaiser94 authored and shaeqahmed committed Dec 9, 2024
1 parent 9138aa7 commit 27bd2d4
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 54 deletions.
1 change: 1 addition & 0 deletions crates/iceberg/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ url = { workspace = true }
uuid = { workspace = true }

[dev-dependencies]
ctor = { workspace = true }
iceberg_test_utils = { path = "../test_utils", features = ["tests"] }
pretty_assertions = { workspace = true }
tempfile = { workspace = true }
Expand Down
101 changes: 47 additions & 54 deletions crates/iceberg/tests/file_io_s3_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,86 +17,79 @@

//! Integration tests for FileIO S3.
use ctor::{ctor, dtor};
use iceberg::io::{
FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY,
};
use iceberg_test_utils::docker::DockerCompose;
use iceberg_test_utils::{normalize_test_name, set_up};
use std::sync::RwLock;

struct MinIOFixture {
_docker_compose: DockerCompose,
file_io: FileIO,
static DOCKER_COMPOSE_ENV: RwLock<Option<DockerCompose>> = RwLock::new(None);

#[ctor]
fn before_all() {
let mut guard = DOCKER_COMPOSE_ENV.write().unwrap();
let docker_compose = DockerCompose::new(
normalize_test_name(module_path!()),
format!("{}/testdata/file_io_s3", env!("CARGO_MANIFEST_DIR")),
);
docker_compose.run();
guard.replace(docker_compose);
}
impl MinIOFixture {
async fn new(project_name: impl ToString) -> Self {
// Start the Docker container for the test fixture
let docker = DockerCompose::new(
project_name.to_string(),
format!("{}/testdata/file_io_s3", env!("CARGO_MANIFEST_DIR")),
);
docker.run();
let container_ip = docker.get_container_ip("minio");
let read_port = format!("{}:{}", container_ip, 9000);

MinIOFixture {
_docker_compose: docker,
file_io: FileIOBuilder::new("s3")
.with_props(vec![
(S3_ENDPOINT, format!("http://{}", read_port)),
(S3_ACCESS_KEY_ID, "admin".to_string()),
(S3_SECRET_ACCESS_KEY, "password".to_string()),
(S3_REGION, "us-east-1".to_string()),
])
.build()
.unwrap(),
}
}
#[dtor]
fn after_all() {
let mut guard = DOCKER_COMPOSE_ENV.write().unwrap();
guard.take();
}

async fn get_file_io() -> FileIO {
set_up();

let guard = DOCKER_COMPOSE_ENV.read().unwrap();
let docker_compose = guard.as_ref().unwrap();
let container_ip = docker_compose.get_container_ip("minio");
let read_port = format!("{}:{}", container_ip, 9000);

FileIOBuilder::new("s3")
.with_props(vec![
(S3_ENDPOINT, format!("http://{}", read_port)),
(S3_ACCESS_KEY_ID, "admin".to_string()),
(S3_SECRET_ACCESS_KEY, "password".to_string()),
(S3_REGION, "us-east-1".to_string()),
])
.build()
.unwrap()
}

#[tokio::test]
async fn test_file_io_s3_is_exist() {
let fixture = MinIOFixture::new("test_file_io_s3_is_exist").await;
assert!(!fixture.file_io.is_exist("s3://bucket2/any").await.unwrap());
assert!(fixture.file_io.is_exist("s3://bucket1/").await.unwrap());
let file_io = get_file_io().await;
assert!(!file_io.is_exist("s3://bucket2/any").await.unwrap());
assert!(file_io.is_exist("s3://bucket1/").await.unwrap());
}

#[tokio::test]
async fn test_file_io_s3_output() {
// Start the Docker container for the test fixture
let fixture = MinIOFixture::new("test_file_io_s3_output").await;
assert!(!fixture
.file_io
.is_exist("s3://bucket1/test_output")
.await
.unwrap());
let output_file = fixture
.file_io
.new_output("s3://bucket1/test_output")
.unwrap();
let file_io = get_file_io().await;
assert!(!file_io.is_exist("s3://bucket1/test_output").await.unwrap());
let output_file = file_io.new_output("s3://bucket1/test_output").unwrap();
{
output_file.write("123".into()).await.unwrap();
}
assert!(fixture
.file_io
.is_exist("s3://bucket1/test_output")
.await
.unwrap());
assert!(file_io.is_exist("s3://bucket1/test_output").await.unwrap());
}

#[tokio::test]
async fn test_file_io_s3_input() {
let fixture = MinIOFixture::new("test_file_io_s3_input").await;
let output_file = fixture
.file_io
.new_output("s3://bucket1/test_input")
.unwrap();
let file_io = get_file_io().await;
let output_file = file_io.new_output("s3://bucket1/test_input").unwrap();
{
output_file.write("test_input".into()).await.unwrap();
}

let input_file = fixture
.file_io
.new_input("s3://bucket1/test_input")
.unwrap();
let input_file = file_io.new_input("s3://bucket1/test_input").unwrap();

{
let buffer = input_file.read().await.unwrap();
Expand Down

0 comments on commit 27bd2d4

Please sign in to comment.