Skip to content

Commit

Permalink
Only require PROTOC for source build (apache#3947)
Browse files Browse the repository at this point in the history
  • Loading branch information
tustvold committed Oct 25, 2022
1 parent 318b4ad commit 17af511
Show file tree
Hide file tree
Showing 8 changed files with 14,807 additions and 44 deletions.
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
datafusion/proto/src/generated/prost.rs linguist-generated
datafusion/proto/src/generated/pbjson.rs linguist-generated
8 changes: 4 additions & 4 deletions datafusion/proto/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,16 @@ keywords = ["arrow", "query", "sql"]
edition = "2021"
rust-version = "1.62"

[package.metadata.docs.rs]
rustc-args = ["--cfg", "docsrs"]
# Exclude proto files so crates.io consumers don't need protoc
exclude = ["*.proto"]

[lib]
name = "datafusion_proto"
path = "src/lib.rs"

[features]
default = []
json = ["pbjson", "pbjson-build", "serde", "serde_json"]
json = ["pbjson", "serde", "serde_json"]

[dependencies]
arrow = "25.0.0"
Expand All @@ -55,5 +55,5 @@ doc-comment = "0.3"
tokio = "1.18"

[build-dependencies]
pbjson-build = { version = "0.5", optional = true }
pbjson-build = { version = "0.5" }
prost-build = { version = "0.11.1" }
42 changes: 15 additions & 27 deletions datafusion/proto/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,25 +15,29 @@
// specific language governing permissions and limitations
// under the License.

use std::path::{Path, PathBuf};

type Error = Box<dyn std::error::Error>;
type Result<T, E = Error> = std::result::Result<T, E>;

fn main() -> Result<(), String> {
// for use in docker build where file changes can be wonky
println!("cargo:rerun-if-env-changed=FORCE_REBUILD");
println!("cargo:rerun-if-changed=proto/datafusion.proto");

build()?;
// We don't include the proto files in releases so that downstreams
// do not need to have PROTOC included
if Path::new("proto/datafusion.proto").exists() {
println!("cargo:rerun-if-changed=proto/datafusion.proto");
build()?
}

Ok(())
}

fn build() -> Result<(), String> {
use std::io::Write;

let out = std::path::PathBuf::from(
std::env::var("OUT_DIR").expect("Cannot find OUT_DIR environment variable"),
);
let out: PathBuf = std::env::var("OUT_DIR")
.expect("Cannot find OUT_DIR environment variable")
.into();
let descriptor_path = out.join("proto_descriptor.bin");

prost_build::Config::new()
Expand All @@ -57,27 +61,11 @@ fn build() -> Result<(), String> {
.build(&[".datafusion"])
.map_err(|e| format!("pbjson compilation failed: {}", e))?;

// .serde.rs is not a valid package name, so append to datafusion.rs so we can treat it normally
let proto = std::fs::read_to_string(out.join("datafusion.rs")).unwrap();

#[cfg(feature = "json")]
let json = std::fs::read_to_string(out.join("datafusion.serde.rs")).unwrap();
let prost = out.join("datafusion.rs");
let pbjson = out.join("datafusion.serde.rs");

#[cfg(feature = "docsrs")]
let path = out.join("datafusion.rs");
#[cfg(not(feature = "docsrs"))]
let path = "src/generated/datafusion.rs";

let mut file = std::fs::OpenOptions::new()
.write(true)
.truncate(true)
.create(true)
.open(path)
.unwrap();
file.write_all(proto.as_str().as_ref()).unwrap();

#[cfg(feature = "json")]
file.write_all(json.as_str().as_ref()).unwrap();
std::fs::copy(prost, "src/generated/prost.rs").unwrap();
std::fs::copy(pbjson, "src/generated/pbjson.rs").unwrap();

Ok(())
}
4 changes: 0 additions & 4 deletions datafusion/proto/src/generated/.gitignore

This file was deleted.

11 changes: 2 additions & 9 deletions datafusion/proto/src/generated/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,9 @@
// specific language governing permissions and limitations
// under the License.

#[allow(clippy::all)]
#[rustfmt::skip]
#[cfg(not(docsrs))]
pub mod datafusion;

#[cfg(docsrs)]
#[allow(clippy::all)]
pub mod datafusion {
include!(concat!(env!("OUT_DIR"), "/datafusion.rs"));
include!("prost.rs");

#[cfg(feature = "json")]
include!(concat!(env!("OUT_DIR"), "/datafusion.serde.rs"));
include!("pbjson.rs");
}
Loading

0 comments on commit 17af511

Please sign in to comment.