-
Notifications
You must be signed in to change notification settings - Fork 211
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: update merge_insert to add statistics for inserted, updated, de…
- Loading branch information
Showing
12 changed files
with
249 additions
and
104 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -20,7 +20,6 @@ | |
from subprocess import check_output | ||
|
||
import lance | ||
import pyarrow as pa | ||
|
||
|
||
def main(): | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,6 @@ | ||
# Configuration file for the Sphinx documentation builder. | ||
|
||
import shutil | ||
import subprocess | ||
|
||
|
||
def run_apidoc(_): | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,25 +1,29 @@ | ||
# | ||
# | ||
# Lance example loading a dataset from Google Cloud Storage | ||
# | ||
# You need to set one of the following environment variables in order to authenticate with GS | ||
# - GOOGLE_SERVICE_ACCOUNT: location of service account file | ||
# - GOOGLE_SERVICE_ACCOUNT_KEY: JSON serialized service account key | ||
# | ||
# Follow this doc in order to create an service key: https://cloud.google.com/iam/docs/keys-create-delete | ||
# Follow this doc in order to create an service key: https://cloud.google.com/iam/docs/keys-create-delete | ||
# | ||
|
||
import lance | ||
import pandas as pd | ||
|
||
ds = lance.dataset("gs://eto-public/datasets/oxford_pet/oxford_pet.lance") | ||
count = ds.count_rows() | ||
print(f"There are {count} pets") | ||
|
||
# You can also write to GCS | ||
import pandas as pd | ||
|
||
uri = "gs://eto-public/datasets/oxford_pet/example.lance" | ||
lance.write_dataset(pd.DataFrame({"a": pd.array([10], dtype="Int32")}), uri, mode='create') | ||
lance.write_dataset( | ||
pd.DataFrame({"a": pd.array([10], dtype="Int32")}), uri, mode="create" | ||
) | ||
assert lance.dataset(uri).version == 1 | ||
|
||
lance.write_dataset(pd.DataFrame({"a": pd.array([5], dtype="Int32")}), uri, mode='append') | ||
lance.write_dataset( | ||
pd.DataFrame({"a": pd.array([5], dtype="Int32")}), uri, mode="append" | ||
) | ||
assert lance.dataset(uri).version == 2 | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.