From dda36f536ee541e50dd13fa287f4f99f66e5e223 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Fri, 11 Mar 2016 11:49:42 -0800 Subject: [PATCH] Moving all Bigtable row filters into their own module. In the process also lowering the line limits to levels before they were raised to accomodate the final changes to the Bigtable Row implementations. --- docs/bigtable-row-filters.rst | 68 ++ docs/bigtable-row.rst | 60 -- docs/index.rst | 1 + gcloud/bigquery/test_job.py | 1 - gcloud/bigtable/happybase/batch.py | 2 +- gcloud/bigtable/happybase/table.py | 16 +- gcloud/bigtable/happybase/test_batch.py | 2 +- gcloud/bigtable/happybase/test_table.py | 34 +- gcloud/bigtable/row.py | 746 +---------------- gcloud/bigtable/row_filters.py | 764 +++++++++++++++++ gcloud/bigtable/test_row.py | 993 +--------------------- gcloud/bigtable/test_row_filters.py | 1005 +++++++++++++++++++++++ gcloud/bigtable/test_table.py | 2 +- scripts/pylintrc_default | 2 +- scripts/run_pylint.py | 2 +- system_tests/bigtable.py | 8 +- 16 files changed, 1875 insertions(+), 1831 deletions(-) create mode 100644 docs/bigtable-row-filters.rst create mode 100644 gcloud/bigtable/row_filters.py create mode 100644 gcloud/bigtable/test_row_filters.py diff --git a/docs/bigtable-row-filters.rst b/docs/bigtable-row-filters.rst new file mode 100644 index 000000000000..b5e99eab6575 --- /dev/null +++ b/docs/bigtable-row-filters.rst @@ -0,0 +1,68 @@ +Bigtable Row Filters +==================== + +It is possible to use a +:class:`RowFilter ` +when adding mutations to a +:class:`ConditionalRow ` and when +reading row data with :meth:`read_row() ` +:meth:`read_rows() `. + +As laid out in the `RowFilter definition`_, the following basic filters +are provided: + +* :class:`SinkFilter <.row_filters.SinkFilter>` +* :class:`PassAllFilter <.row_filters.PassAllFilter>` +* :class:`BlockAllFilter <.row_filters.BlockAllFilter>` +* :class:`RowKeyRegexFilter <.row_filters.RowKeyRegexFilter>` +* :class:`RowSampleFilter <.row_filters.RowSampleFilter>` +* :class:`FamilyNameRegexFilter <.row_filters.FamilyNameRegexFilter>` +* :class:`ColumnQualifierRegexFilter <.row_filters.ColumnQualifierRegexFilter>` +* :class:`TimestampRangeFilter <.row_filters.TimestampRangeFilter>` +* :class:`ColumnRangeFilter <.row_filters.ColumnRangeFilter>` +* :class:`ValueRegexFilter <.row_filters.ValueRegexFilter>` +* :class:`ValueRangeFilter <.row_filters.ValueRangeFilter>` +* :class:`CellsRowOffsetFilter <.row_filters.CellsRowOffsetFilter>` +* :class:`CellsRowLimitFilter <.row_filters.CellsRowLimitFilter>` +* :class:`CellsColumnLimitFilter <.row_filters.CellsColumnLimitFilter>` +* :class:`StripValueTransformerFilter <.row_filters.StripValueTransformerFilter>` +* :class:`ApplyLabelFilter <.row_filters.ApplyLabelFilter>` + +In addition, these filters can be combined into composite filters with + +* :class:`RowFilterChain <.row_filters.RowFilterChain>` +* :class:`RowFilterUnion <.row_filters.RowFilterUnion>` +* :class:`ConditionalRowFilter <.row_filters.ConditionalRowFilter>` + +These rules can be nested arbitrarily, with a basic filter at the lowest +level. For example: + +.. code:: python + + # Filter in a specified column (matching any column family). + col1_filter = ColumnQualifierRegexFilter(b'columnbia') + + # Create a filter to label results. + label1 = u'label-red' + label1_filter = ApplyLabelFilter(label1) + + # Combine the filters to label all the cells in columnbia. + chain1 = RowFilterChain(filters=[col1_filter, label1_filter]) + + # Create a similar filter to label cells blue. + col2_filter = ColumnQualifierRegexFilter(b'columnseeya') + label2 = u'label-blue' + label2_filter = ApplyLabelFilter(label2) + chain2 = RowFilterChain(filters=[col2_filter, label2_filter]) + + # Bring our two labeled columns together. + row_filter = RowFilterUnion(filters=[chain1, chain2]) + +---- + +.. automodule:: gcloud.bigtable.row_filters + :members: + :undoc-members: + :show-inheritance: + +.. _RowFilter definition: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/1ff247c2e3b7cd0a2dd49071b2d95beaf6563092/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_data.proto#L195 diff --git a/docs/bigtable-row.rst b/docs/bigtable-row.rst index fae56b76f3d5..fbd96352be24 100644 --- a/docs/bigtable-row.rst +++ b/docs/bigtable-row.rst @@ -1,67 +1,7 @@ Bigtable Row ============ -It is possible to use a :class:`RowFilter ` -when adding mutations to a -:class:`ConditionalRow ` and when -reading row data with :meth:`read_row() ` -:meth:`read_rows() `. - -As laid out in the `RowFilter definition`_, the following basic filters -are provided: - -* :class:`SinkFilter <.row.SinkFilter>` -* :class:`PassAllFilter <.row.PassAllFilter>` -* :class:`BlockAllFilter <.row.BlockAllFilter>` -* :class:`RowKeyRegexFilter <.row.RowKeyRegexFilter>` -* :class:`RowSampleFilter <.row.RowSampleFilter>` -* :class:`FamilyNameRegexFilter <.row.FamilyNameRegexFilter>` -* :class:`ColumnQualifierRegexFilter <.row.ColumnQualifierRegexFilter>` -* :class:`TimestampRangeFilter <.row.TimestampRangeFilter>` -* :class:`ColumnRangeFilter <.row.ColumnRangeFilter>` -* :class:`ValueRegexFilter <.row.ValueRegexFilter>` -* :class:`ValueRangeFilter <.row.ValueRangeFilter>` -* :class:`CellsRowOffsetFilter <.row.CellsRowOffsetFilter>` -* :class:`CellsRowLimitFilter <.row.CellsRowLimitFilter>` -* :class:`CellsColumnLimitFilter <.row.CellsColumnLimitFilter>` -* :class:`StripValueTransformerFilter <.row.StripValueTransformerFilter>` -* :class:`ApplyLabelFilter <.row.ApplyLabelFilter>` - -In addition, these filters can be combined into composite filters with - -* :class:`RowFilterChain <.row.RowFilterChain>` -* :class:`RowFilterUnion <.row.RowFilterUnion>` -* :class:`ConditionalRowFilter <.row.ConditionalRowFilter>` - -These rules can be nested arbitrarily, with a basic filter at the lowest -level. For example: - -.. code:: python - - # Filter in a specified column (matching any column family). - col1_filter = ColumnQualifierRegexFilter(b'columnbia') - - # Create a filter to label results. - label1 = u'label-red' - label1_filter = ApplyLabelFilter(label1) - - # Combine the filters to label all the cells in columnbia. - chain1 = RowFilterChain(filters=[col1_filter, label1_filter]) - - # Create a similar filter to label cells blue. - col2_filter = ColumnQualifierRegexFilter(b'columnseeya') - label2 = u'label-blue' - label2_filter = ApplyLabelFilter(label2) - chain2 = RowFilterChain(filters=[col2_filter, label2_filter]) - - # Bring our two labeled columns together. - row_filter = RowFilterUnion(filters=[chain1, chain2]) - ----- - .. automodule:: gcloud.bigtable.row :members: :undoc-members: :show-inheritance: - -.. _RowFilter definition: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/1ff247c2e3b7cd0a2dd49071b2d95beaf6563092/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_data.proto#L195 diff --git a/docs/index.rst b/docs/index.rst index 84afe9522fc8..dd0e6e90bf70 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -69,6 +69,7 @@ bigtable-table bigtable-column-family bigtable-row + bigtable-row-filters bigtable-row-data happybase-connection happybase-pool diff --git a/gcloud/bigquery/test_job.py b/gcloud/bigquery/test_job.py index 58953dbc81d5..64660706be2e 100644 --- a/gcloud/bigquery/test_job.py +++ b/gcloud/bigquery/test_job.py @@ -1,4 +1,3 @@ -# pylint: disable=C0302 # Copyright 2015 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/gcloud/bigtable/happybase/batch.py b/gcloud/bigtable/happybase/batch.py index 663b7f892c77..25e6d073fc66 100644 --- a/gcloud/bigtable/happybase/batch.py +++ b/gcloud/bigtable/happybase/batch.py @@ -21,7 +21,7 @@ import six from gcloud._helpers import _datetime_from_microseconds -from gcloud.bigtable.row import TimestampRange +from gcloud.bigtable.row_filters import TimestampRange _WAL_SENTINEL = object() diff --git a/gcloud/bigtable/happybase/table.py b/gcloud/bigtable/happybase/table.py index dcbf45163015..bf018ad1647f 100644 --- a/gcloud/bigtable/happybase/table.py +++ b/gcloud/bigtable/happybase/table.py @@ -30,14 +30,14 @@ from gcloud.bigtable.happybase.batch import _get_column_pairs from gcloud.bigtable.happybase.batch import _WAL_SENTINEL from gcloud.bigtable.happybase.batch import Batch -from gcloud.bigtable.row import CellsColumnLimitFilter -from gcloud.bigtable.row import ColumnQualifierRegexFilter -from gcloud.bigtable.row import FamilyNameRegexFilter -from gcloud.bigtable.row import RowFilterChain -from gcloud.bigtable.row import RowFilterUnion -from gcloud.bigtable.row import RowKeyRegexFilter -from gcloud.bigtable.row import TimestampRange -from gcloud.bigtable.row import TimestampRangeFilter +from gcloud.bigtable.row_filters import CellsColumnLimitFilter +from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter +from gcloud.bigtable.row_filters import FamilyNameRegexFilter +from gcloud.bigtable.row_filters import RowFilterChain +from gcloud.bigtable.row_filters import RowFilterUnion +from gcloud.bigtable.row_filters import RowKeyRegexFilter +from gcloud.bigtable.row_filters import TimestampRange +from gcloud.bigtable.row_filters import TimestampRangeFilter from gcloud.bigtable.table import Table as _LowLevelTable diff --git a/gcloud/bigtable/happybase/test_batch.py b/gcloud/bigtable/happybase/test_batch.py index 40cc84ae4c67..cf2156f226b4 100644 --- a/gcloud/bigtable/happybase/test_batch.py +++ b/gcloud/bigtable/happybase/test_batch.py @@ -46,7 +46,7 @@ def test_constructor_defaults(self): def test_constructor_explicit(self): from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row import TimestampRange + from gcloud.bigtable.row_filters import TimestampRange table = object() timestamp = 144185290431 diff --git a/gcloud/bigtable/happybase/test_table.py b/gcloud/bigtable/happybase/test_table.py index cf95896a7696..9c5382a9e7fe 100644 --- a/gcloud/bigtable/happybase/test_table.py +++ b/gcloud/bigtable/happybase/test_table.py @@ -1113,7 +1113,7 @@ def test_invalid_type(self): def test_success(self): from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row import TimestampRange + from gcloud.bigtable.row_filters import TimestampRange timestamp = 1441928298571 ts_dt = _datetime_from_microseconds(1000 * timestamp) @@ -1219,7 +1219,7 @@ def test_no_filters(self): self._callFUT() def test_single_filter(self): - from gcloud.bigtable.row import CellsColumnLimitFilter + from gcloud.bigtable.row_filters import CellsColumnLimitFilter versions = 1337 result = self._callFUT(versions=versions) @@ -1229,7 +1229,7 @@ def test_single_filter(self): self.assertEqual(result.num_cells, versions) def test_existing_filters(self): - from gcloud.bigtable.row import CellsColumnLimitFilter + from gcloud.bigtable.row_filters import CellsColumnLimitFilter filters = [] versions = 1337 @@ -1244,9 +1244,9 @@ def test_existing_filters(self): def _column_helper(self, num_filters, versions=None, timestamp=None, column=None, col_fam=None, qual=None): - from gcloud.bigtable.row import ColumnQualifierRegexFilter - from gcloud.bigtable.row import FamilyNameRegexFilter - from gcloud.bigtable.row import RowFilterChain + from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter + from gcloud.bigtable.row_filters import FamilyNameRegexFilter + from gcloud.bigtable.row_filters import RowFilterChain if col_fam is None: col_fam = 'cf1' @@ -1282,7 +1282,7 @@ def test_column_unicode(self): col_fam=u'cfU', qual=u'qualN') def test_with_versions(self): - from gcloud.bigtable.row import CellsColumnLimitFilter + from gcloud.bigtable.row_filters import CellsColumnLimitFilter versions = 11 result = self._column_helper(num_filters=3, versions=versions) @@ -1295,8 +1295,8 @@ def test_with_versions(self): def test_with_timestamp(self): from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row import TimestampRange - from gcloud.bigtable.row import TimestampRangeFilter + from gcloud.bigtable.row_filters import TimestampRange + from gcloud.bigtable.row_filters import TimestampRangeFilter timestamp = 1441928298571 result = self._column_helper(num_filters=3, timestamp=timestamp) @@ -1330,7 +1330,7 @@ def test_no_columns(self): self._callFUT(columns) def test_single_column(self): - from gcloud.bigtable.row import FamilyNameRegexFilter + from gcloud.bigtable.row_filters import FamilyNameRegexFilter col_fam = 'cf1' columns = [col_fam] @@ -1339,10 +1339,10 @@ def test_single_column(self): self.assertEqual(result, expected_result) def test_column_and_column_families(self): - from gcloud.bigtable.row import ColumnQualifierRegexFilter - from gcloud.bigtable.row import FamilyNameRegexFilter - from gcloud.bigtable.row import RowFilterChain - from gcloud.bigtable.row import RowFilterUnion + from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter + from gcloud.bigtable.row_filters import FamilyNameRegexFilter + from gcloud.bigtable.row_filters import RowFilterChain + from gcloud.bigtable.row_filters import RowFilterUnion col_fam1 = 'cf1' col_fam2 = 'cf2' @@ -1378,7 +1378,7 @@ def test_no_rows(self): self._callFUT(row_keys) def test_single_row(self): - from gcloud.bigtable.row import RowKeyRegexFilter + from gcloud.bigtable.row_filters import RowKeyRegexFilter row_key = b'row-key' row_keys = [row_key] @@ -1387,8 +1387,8 @@ def test_single_row(self): self.assertEqual(result, expected_result) def test_many_rows(self): - from gcloud.bigtable.row import RowFilterUnion - from gcloud.bigtable.row import RowKeyRegexFilter + from gcloud.bigtable.row_filters import RowFilterUnion + from gcloud.bigtable.row_filters import RowKeyRegexFilter row_key1 = b'row-key1' row_key2 = b'row-key2' diff --git a/gcloud/bigtable/row.py b/gcloud/bigtable/row.py index 276f009b7bb1..6666a62e9d8a 100644 --- a/gcloud/bigtable/row.py +++ b/gcloud/bigtable/row.py @@ -397,7 +397,7 @@ class ConditionalRow(DirectRow): :type table: :class:`Table ` :param table: The table that owns the row. - :type filter_: :class:`RowFilter` + :type filter_: :class:`.RowFilter` :param filter_: Filter to be used for conditional mutations. """ def __init__(self, row_key, table, filter_): @@ -771,750 +771,6 @@ def commit(self): return _parse_rmw_row_response(row_response) -class RowFilter(object): - """Basic filter to apply to cells in a row. - - These values can be combined via :class:`RowFilterChain`, - :class:`RowFilterUnion` and :class:`ConditionalRowFilter`. - - .. note:: - - This class is a do-nothing base class for all row filters. - """ - - def __ne__(self, other): - return not self.__eq__(other) - - -class _BoolFilter(RowFilter): - """Row filter that uses a boolean flag. - - :type flag: bool - :param flag: An indicator if a setting is turned on or off. - """ - - def __init__(self, flag): - self.flag = flag - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.flag == self.flag - - -class SinkFilter(_BoolFilter): - """Advanced row filter to skip parent filters. - - :type flag: bool - :param flag: ADVANCED USE ONLY. Hook for introspection into the row filter. - Outputs all cells directly to the output of the read rather - than to any parent filter. Cannot be used within the - ``predicate_filter``, ``true_filter``, or ``false_filter`` - of a :class:`ConditionalRowFilter`. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(sink=self.flag) - - -class PassAllFilter(_BoolFilter): - """Row filter equivalent to not filtering at all. - - :type flag: bool - :param flag: Matches all cells, regardless of input. Functionally - equivalent to leaving ``filter`` unset, but included for - completeness. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(pass_all_filter=self.flag) - - -class BlockAllFilter(_BoolFilter): - """Row filter that doesn't match any cells. - - :type flag: bool - :param flag: Does not match any cells, regardless of input. Useful for - temporarily disabling just part of a filter. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(block_all_filter=self.flag) - - -class _RegexFilter(RowFilter): - """Row filter that uses a regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - :type regex: bytes or str - :param regex: A regular expression (RE2) for some row filter. - """ - - def __init__(self, regex): - self.regex = regex - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.regex == self.regex - - -class RowKeyRegexFilter(_RegexFilter): - """Row filter for a row key regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - Special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. - - :type regex: bytes - :param regex: A regular expression (RE2) to match cells from rows with row - keys that satisfy this regex. For a - ``CheckAndMutateRowRequest``, this filter is unnecessary - since the row key is already specified. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(row_key_regex_filter=self.regex) - - -class RowSampleFilter(RowFilter): - """Matches all cells from a row with probability p. - - :type sample: float - :param sample: The probability of matching a cell (must be in the - interval ``[0, 1]``). - """ - - def __init__(self, sample): - self.sample = sample - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.sample == self.sample - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(row_sample_filter=self.sample) - - -class FamilyNameRegexFilter(_RegexFilter): - """Row filter for a family name regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - :type regex: str - :param regex: A regular expression (RE2) to match cells from columns in a - given column family. For technical reasons, the regex must - not contain the ``':'`` character, even if it is not being - used as a literal. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(family_name_regex_filter=self.regex) - - -class ColumnQualifierRegexFilter(_RegexFilter): - """Row filter for a column qualifier regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - Special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. - - :type regex: bytes - :param regex: A regular expression (RE2) to match cells from column that - match this regex (irrespective of column family). - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(column_qualifier_regex_filter=self.regex) - - -class TimestampRange(object): - """Range of time with inclusive lower and exclusive upper bounds. - - :type start: :class:`datetime.datetime` - :param start: (Optional) The (inclusive) lower bound of the timestamp - range. If omitted, defaults to Unix epoch. - - :type end: :class:`datetime.datetime` - :param end: (Optional) The (exclusive) upper bound of the timestamp - range. If omitted, no upper bound is used. - """ - - def __init__(self, start=None, end=None): - self.start = start - self.end = end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.start == self.start and - other.end == self.end) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the :class:`TimestampRange` to a protobuf. - - :rtype: :class:`.data_pb2.TimestampRange` - :returns: The converted current object. - """ - timestamp_range_kwargs = {} - if self.start is not None: - timestamp_range_kwargs['start_timestamp_micros'] = ( - _microseconds_from_datetime(self.start)) - if self.end is not None: - timestamp_range_kwargs['end_timestamp_micros'] = ( - _microseconds_from_datetime(self.end)) - return data_pb2.TimestampRange(**timestamp_range_kwargs) - - -class TimestampRangeFilter(RowFilter): - """Row filter that limits cells to a range of time. - - :type range_: :class:`TimestampRange` - :param range_: Range of time that cells should match against. - """ - - def __init__(self, range_): - self.range_ = range_ - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.range_ == self.range_ - - def to_pb(self): - """Converts the row filter to a protobuf. - - First converts the ``range_`` on the current object to a protobuf and - then uses it in the ``timestamp_range_filter`` field. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(timestamp_range_filter=self.range_.to_pb()) - - -class ColumnRangeFilter(RowFilter): - """A row filter to restrict to a range of columns. - - Both the start and end column can be included or excluded in the range. - By default, we include them both, but this can be changed with optional - flags. - - :type column_family_id: str - :param column_family_id: The column family that contains the columns. Must - be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type start_column: bytes - :param start_column: The start of the range of columns. If no value is - used, the backend applies no upper bound to the - values. - - :type end_column: bytes - :param end_column: The end of the range of columns. If no value is used, - the backend applies no upper bound to the values. - - :type inclusive_start: bool - :param inclusive_start: Boolean indicating if the start column should be - included in the range (or excluded). Defaults - to :data:`True` if ``start_column`` is passed and - no ``inclusive_start`` was given. - - :type inclusive_end: bool - :param inclusive_end: Boolean indicating if the end column should be - included in the range (or excluded). Defaults - to :data:`True` if ``end_column`` is passed and - no ``inclusive_end`` was given. - - :raises: :class:`ValueError ` if ``inclusive_start`` - is set but no ``start_column`` is given or if ``inclusive_end`` - is set but no ``end_column`` is given - """ - - def __init__(self, column_family_id, start_column=None, end_column=None, - inclusive_start=None, inclusive_end=None): - self.column_family_id = column_family_id - - if inclusive_start is None: - inclusive_start = True - elif start_column is None: - raise ValueError('Inclusive start was specified but no ' - 'start column was given.') - self.start_column = start_column - self.inclusive_start = inclusive_start - - if inclusive_end is None: - inclusive_end = True - elif end_column is None: - raise ValueError('Inclusive end was specified but no ' - 'end column was given.') - self.end_column = end_column - self.inclusive_end = inclusive_end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.column_family_id == self.column_family_id and - other.start_column == self.start_column and - other.end_column == self.end_column and - other.inclusive_start == self.inclusive_start and - other.inclusive_end == self.inclusive_end) - - def to_pb(self): - """Converts the row filter to a protobuf. - - First converts to a :class:`.data_pb2.ColumnRange` and then uses it - in the ``column_range_filter`` field. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - column_range_kwargs = {'family_name': self.column_family_id} - if self.start_column is not None: - if self.inclusive_start: - key = 'start_qualifier_inclusive' - else: - key = 'start_qualifier_exclusive' - column_range_kwargs[key] = _to_bytes(self.start_column) - if self.end_column is not None: - if self.inclusive_end: - key = 'end_qualifier_inclusive' - else: - key = 'end_qualifier_exclusive' - column_range_kwargs[key] = _to_bytes(self.end_column) - - column_range = data_pb2.ColumnRange(**column_range_kwargs) - return data_pb2.RowFilter(column_range_filter=column_range) - - -class ValueRegexFilter(_RegexFilter): - """Row filter for a value regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - Special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. - - :type regex: bytes - :param regex: A regular expression (RE2) to match cells with values that - match this regex. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(value_regex_filter=self.regex) - - -class ValueRangeFilter(RowFilter): - """A range of values to restrict to in a row filter. - - Will only match cells that have values in this range. - - Both the start and end value can be included or excluded in the range. - By default, we include them both, but this can be changed with optional - flags. - - :type start_value: bytes - :param start_value: The start of the range of values. If no value is used, - the backend applies no lower bound to the values. - - :type end_value: bytes - :param end_value: The end of the range of values. If no value is used, - the backend applies no upper bound to the values. - - :type inclusive_start: bool - :param inclusive_start: Boolean indicating if the start value should be - included in the range (or excluded). Defaults - to :data:`True` if ``start_value`` is passed and - no ``inclusive_start`` was given. - - :type inclusive_end: bool - :param inclusive_end: Boolean indicating if the end value should be - included in the range (or excluded). Defaults - to :data:`True` if ``end_value`` is passed and - no ``inclusive_end`` was given. - - :raises: :class:`ValueError ` if ``inclusive_start`` - is set but no ``start_value`` is given or if ``inclusive_end`` - is set but no ``end_value`` is given - """ - - def __init__(self, start_value=None, end_value=None, - inclusive_start=None, inclusive_end=None): - if inclusive_start is None: - inclusive_start = True - elif start_value is None: - raise ValueError('Inclusive start was specified but no ' - 'start value was given.') - self.start_value = start_value - self.inclusive_start = inclusive_start - - if inclusive_end is None: - inclusive_end = True - elif end_value is None: - raise ValueError('Inclusive end was specified but no ' - 'end value was given.') - self.end_value = end_value - self.inclusive_end = inclusive_end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.start_value == self.start_value and - other.end_value == self.end_value and - other.inclusive_start == self.inclusive_start and - other.inclusive_end == self.inclusive_end) - - def to_pb(self): - """Converts the row filter to a protobuf. - - First converts to a :class:`.data_pb2.ValueRange` and then uses - it to create a row filter protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - value_range_kwargs = {} - if self.start_value is not None: - if self.inclusive_start: - key = 'start_value_inclusive' - else: - key = 'start_value_exclusive' - value_range_kwargs[key] = _to_bytes(self.start_value) - if self.end_value is not None: - if self.inclusive_end: - key = 'end_value_inclusive' - else: - key = 'end_value_exclusive' - value_range_kwargs[key] = _to_bytes(self.end_value) - - value_range = data_pb2.ValueRange(**value_range_kwargs) - return data_pb2.RowFilter(value_range_filter=value_range) - - -class _CellCountFilter(RowFilter): - """Row filter that uses an integer count of cells. - - The cell count is used as an offset or a limit for the number - of results returned. - - :type num_cells: int - :param num_cells: An integer count / offset / limit. - """ - - def __init__(self, num_cells): - self.num_cells = num_cells - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.num_cells == self.num_cells - - -class CellsRowOffsetFilter(_CellCountFilter): - """Row filter to skip cells in a row. - - :type num_cells: int - :param num_cells: Skips the first N cells of the row. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(cells_per_row_offset_filter=self.num_cells) - - -class CellsRowLimitFilter(_CellCountFilter): - """Row filter to limit cells in a row. - - :type num_cells: int - :param num_cells: Matches only the first N cells of the row. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(cells_per_row_limit_filter=self.num_cells) - - -class CellsColumnLimitFilter(_CellCountFilter): - """Row filter to limit cells in a column. - - :type num_cells: int - :param num_cells: Matches only the most recent N cells within each column. - This filters a (family name, column) pair, based on - timestamps of each cell. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(cells_per_column_limit_filter=self.num_cells) - - -class StripValueTransformerFilter(_BoolFilter): - """Row filter that transforms cells into empty string (0 bytes). - - :type flag: bool - :param flag: If :data:`True`, replaces each cell's value with the empty - string. As the name indicates, this is more useful as a - transformer than a generic query / filter. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(strip_value_transformer=self.flag) - - -class ApplyLabelFilter(RowFilter): - """Filter to apply labels to cells. - - Intended to be used as an intermediate filter on a pre-existing filtered - result set. This way if two sets are combined, the label can tell where - the cell(s) originated.This allows the client to determine which results - were produced from which part of the filter. - - .. note:: - - Due to a technical limitation of the backend, it is not currently - possible to apply multiple labels to a cell. - - :type label: str - :param label: Label to apply to cells in the output row. Values must be - at most 15 characters long, and match the pattern - ``[a-z0-9\\-]+``. - """ - - def __init__(self, label): - self.label = label - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.label == self.label - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(apply_label_transformer=self.label) - - -class _FilterCombination(RowFilter): - """Chain of row filters. - - Sends rows through several filters in sequence. The filters are "chained" - together to process a row. After the first filter is applied, the second - is applied to the filtered output and so on for subsequent filters. - - :type filters: list - :param filters: List of :class:`RowFilter` - """ - - def __init__(self, filters=None): - if filters is None: - filters = [] - self.filters = filters - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.filters == self.filters - - -class RowFilterChain(_FilterCombination): - """Chain of row filters. - - Sends rows through several filters in sequence. The filters are "chained" - together to process a row. After the first filter is applied, the second - is applied to the filtered output and so on for subsequent filters. - - :type filters: list - :param filters: List of :class:`RowFilter` - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - chain = data_pb2.RowFilter.Chain( - filters=[row_filter.to_pb() for row_filter in self.filters]) - return data_pb2.RowFilter(chain=chain) - - -class RowFilterUnion(_FilterCombination): - """Union of row filters. - - Sends rows through several filters simultaneously, then - merges / interleaves all the filtered results together. - - If multiple cells are produced with the same column and timestamp, - they will all appear in the output row in an unspecified mutual order. - - :type filters: list - :param filters: List of :class:`RowFilter` - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - interleave = data_pb2.RowFilter.Interleave( - filters=[row_filter.to_pb() for row_filter in self.filters]) - return data_pb2.RowFilter(interleave=interleave) - - -class ConditionalRowFilter(RowFilter): - """Conditional row filter which exhibits ternary behavior. - - Executes one of two filters based on another filter. If the ``base_filter`` - returns any cells in the row, then ``true_filter`` is executed. If not, - then ``false_filter`` is executed. - - .. note:: - - The ``base_filter`` does not execute atomically with the true and false - filters, which may lead to inconsistent or unexpected results. - - Additionally, executing a :class:`ConditionalRowFilter` has poor - performance on the server, especially when ``false_filter`` is set. - - :type base_filter: :class:`RowFilter` - :param base_filter: The filter to condition on before executing the - true/false filters. - - :type true_filter: :class:`RowFilter` - :param true_filter: (Optional) The filter to execute if there are any cells - matching ``base_filter``. If not provided, no results - will be returned in the true case. - - :type false_filter: :class:`RowFilter` - :param false_filter: (Optional) The filter to execute if there are no cells - matching ``base_filter``. If not provided, no results - will be returned in the false case. - """ - - def __init__(self, base_filter, true_filter=None, false_filter=None): - self.base_filter = base_filter - self.true_filter = true_filter - self.false_filter = false_filter - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.base_filter == self.base_filter and - other.true_filter == self.true_filter and - other.false_filter == self.false_filter) - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - condition_kwargs = {'predicate_filter': self.base_filter.to_pb()} - if self.true_filter is not None: - condition_kwargs['true_filter'] = self.true_filter.to_pb() - if self.false_filter is not None: - condition_kwargs['false_filter'] = self.false_filter.to_pb() - condition = data_pb2.RowFilter.Condition(**condition_kwargs) - return data_pb2.RowFilter(condition=condition) - - def _parse_rmw_row_response(row_response): """Parses the response to a ``ReadModifyWriteRow`` request. diff --git a/gcloud/bigtable/row_filters.py b/gcloud/bigtable/row_filters.py new file mode 100644 index 000000000000..be220b6308f2 --- /dev/null +++ b/gcloud/bigtable/row_filters.py @@ -0,0 +1,764 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Filters for Google Cloud Bigtable Row classes.""" + + +from gcloud._helpers import _microseconds_from_datetime +from gcloud._helpers import _to_bytes +from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + +class RowFilter(object): + """Basic filter to apply to cells in a row. + + These values can be combined via :class:`RowFilterChain`, + :class:`RowFilterUnion` and :class:`ConditionalRowFilter`. + + .. note:: + + This class is a do-nothing base class for all row filters. + """ + + def __ne__(self, other): + return not self.__eq__(other) + + +class _BoolFilter(RowFilter): + """Row filter that uses a boolean flag. + + :type flag: bool + :param flag: An indicator if a setting is turned on or off. + """ + + def __init__(self, flag): + self.flag = flag + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.flag == self.flag + + +class SinkFilter(_BoolFilter): + """Advanced row filter to skip parent filters. + + :type flag: bool + :param flag: ADVANCED USE ONLY. Hook for introspection into the row filter. + Outputs all cells directly to the output of the read rather + than to any parent filter. Cannot be used within the + ``predicate_filter``, ``true_filter``, or ``false_filter`` + of a :class:`ConditionalRowFilter`. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(sink=self.flag) + + +class PassAllFilter(_BoolFilter): + """Row filter equivalent to not filtering at all. + + :type flag: bool + :param flag: Matches all cells, regardless of input. Functionally + equivalent to leaving ``filter`` unset, but included for + completeness. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(pass_all_filter=self.flag) + + +class BlockAllFilter(_BoolFilter): + """Row filter that doesn't match any cells. + + :type flag: bool + :param flag: Does not match any cells, regardless of input. Useful for + temporarily disabling just part of a filter. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(block_all_filter=self.flag) + + +class _RegexFilter(RowFilter): + """Row filter that uses a regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + :type regex: bytes or str + :param regex: A regular expression (RE2) for some row filter. + """ + + def __init__(self, regex): + self.regex = regex + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.regex == self.regex + + +class RowKeyRegexFilter(_RegexFilter): + """Row filter for a row key regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + .. note:: + + Special care need be used with the expression used. Since + each of these properties can contain arbitrary bytes, the ``\\C`` + escape sequence must be used if a true wildcard is desired. The ``.`` + character will not match the new line character ``\\n``, which may be + present in a binary value. + + :type regex: bytes + :param regex: A regular expression (RE2) to match cells from rows with row + keys that satisfy this regex. For a + ``CheckAndMutateRowRequest``, this filter is unnecessary + since the row key is already specified. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(row_key_regex_filter=self.regex) + + +class RowSampleFilter(RowFilter): + """Matches all cells from a row with probability p. + + :type sample: float + :param sample: The probability of matching a cell (must be in the + interval ``[0, 1]``). + """ + + def __init__(self, sample): + self.sample = sample + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.sample == self.sample + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(row_sample_filter=self.sample) + + +class FamilyNameRegexFilter(_RegexFilter): + """Row filter for a family name regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + :type regex: str + :param regex: A regular expression (RE2) to match cells from columns in a + given column family. For technical reasons, the regex must + not contain the ``':'`` character, even if it is not being + used as a literal. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(family_name_regex_filter=self.regex) + + +class ColumnQualifierRegexFilter(_RegexFilter): + """Row filter for a column qualifier regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + .. note:: + + Special care need be used with the expression used. Since + each of these properties can contain arbitrary bytes, the ``\\C`` + escape sequence must be used if a true wildcard is desired. The ``.`` + character will not match the new line character ``\\n``, which may be + present in a binary value. + + :type regex: bytes + :param regex: A regular expression (RE2) to match cells from column that + match this regex (irrespective of column family). + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(column_qualifier_regex_filter=self.regex) + + +class TimestampRange(object): + """Range of time with inclusive lower and exclusive upper bounds. + + :type start: :class:`datetime.datetime` + :param start: (Optional) The (inclusive) lower bound of the timestamp + range. If omitted, defaults to Unix epoch. + + :type end: :class:`datetime.datetime` + :param end: (Optional) The (exclusive) upper bound of the timestamp + range. If omitted, no upper bound is used. + """ + + def __init__(self, start=None, end=None): + self.start = start + self.end = end + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.start == self.start and + other.end == self.end) + + def __ne__(self, other): + return not self.__eq__(other) + + def to_pb(self): + """Converts the :class:`TimestampRange` to a protobuf. + + :rtype: :class:`.data_pb2.TimestampRange` + :returns: The converted current object. + """ + timestamp_range_kwargs = {} + if self.start is not None: + timestamp_range_kwargs['start_timestamp_micros'] = ( + _microseconds_from_datetime(self.start)) + if self.end is not None: + timestamp_range_kwargs['end_timestamp_micros'] = ( + _microseconds_from_datetime(self.end)) + return data_pb2.TimestampRange(**timestamp_range_kwargs) + + +class TimestampRangeFilter(RowFilter): + """Row filter that limits cells to a range of time. + + :type range_: :class:`TimestampRange` + :param range_: Range of time that cells should match against. + """ + + def __init__(self, range_): + self.range_ = range_ + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.range_ == self.range_ + + def to_pb(self): + """Converts the row filter to a protobuf. + + First converts the ``range_`` on the current object to a protobuf and + then uses it in the ``timestamp_range_filter`` field. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(timestamp_range_filter=self.range_.to_pb()) + + +class ColumnRangeFilter(RowFilter): + """A row filter to restrict to a range of columns. + + Both the start and end column can be included or excluded in the range. + By default, we include them both, but this can be changed with optional + flags. + + :type column_family_id: str + :param column_family_id: The column family that contains the columns. Must + be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. + + :type start_column: bytes + :param start_column: The start of the range of columns. If no value is + used, the backend applies no upper bound to the + values. + + :type end_column: bytes + :param end_column: The end of the range of columns. If no value is used, + the backend applies no upper bound to the values. + + :type inclusive_start: bool + :param inclusive_start: Boolean indicating if the start column should be + included in the range (or excluded). Defaults + to :data:`True` if ``start_column`` is passed and + no ``inclusive_start`` was given. + + :type inclusive_end: bool + :param inclusive_end: Boolean indicating if the end column should be + included in the range (or excluded). Defaults + to :data:`True` if ``end_column`` is passed and + no ``inclusive_end`` was given. + + :raises: :class:`ValueError ` if ``inclusive_start`` + is set but no ``start_column`` is given or if ``inclusive_end`` + is set but no ``end_column`` is given + """ + + def __init__(self, column_family_id, start_column=None, end_column=None, + inclusive_start=None, inclusive_end=None): + self.column_family_id = column_family_id + + if inclusive_start is None: + inclusive_start = True + elif start_column is None: + raise ValueError('Inclusive start was specified but no ' + 'start column was given.') + self.start_column = start_column + self.inclusive_start = inclusive_start + + if inclusive_end is None: + inclusive_end = True + elif end_column is None: + raise ValueError('Inclusive end was specified but no ' + 'end column was given.') + self.end_column = end_column + self.inclusive_end = inclusive_end + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.column_family_id == self.column_family_id and + other.start_column == self.start_column and + other.end_column == self.end_column and + other.inclusive_start == self.inclusive_start and + other.inclusive_end == self.inclusive_end) + + def to_pb(self): + """Converts the row filter to a protobuf. + + First converts to a :class:`.data_pb2.ColumnRange` and then uses it + in the ``column_range_filter`` field. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + column_range_kwargs = {'family_name': self.column_family_id} + if self.start_column is not None: + if self.inclusive_start: + key = 'start_qualifier_inclusive' + else: + key = 'start_qualifier_exclusive' + column_range_kwargs[key] = _to_bytes(self.start_column) + if self.end_column is not None: + if self.inclusive_end: + key = 'end_qualifier_inclusive' + else: + key = 'end_qualifier_exclusive' + column_range_kwargs[key] = _to_bytes(self.end_column) + + column_range = data_pb2.ColumnRange(**column_range_kwargs) + return data_pb2.RowFilter(column_range_filter=column_range) + + +class ValueRegexFilter(_RegexFilter): + """Row filter for a value regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + .. note:: + + Special care need be used with the expression used. Since + each of these properties can contain arbitrary bytes, the ``\\C`` + escape sequence must be used if a true wildcard is desired. The ``.`` + character will not match the new line character ``\\n``, which may be + present in a binary value. + + :type regex: bytes + :param regex: A regular expression (RE2) to match cells with values that + match this regex. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(value_regex_filter=self.regex) + + +class ValueRangeFilter(RowFilter): + """A range of values to restrict to in a row filter. + + Will only match cells that have values in this range. + + Both the start and end value can be included or excluded in the range. + By default, we include them both, but this can be changed with optional + flags. + + :type start_value: bytes + :param start_value: The start of the range of values. If no value is used, + the backend applies no lower bound to the values. + + :type end_value: bytes + :param end_value: The end of the range of values. If no value is used, + the backend applies no upper bound to the values. + + :type inclusive_start: bool + :param inclusive_start: Boolean indicating if the start value should be + included in the range (or excluded). Defaults + to :data:`True` if ``start_value`` is passed and + no ``inclusive_start`` was given. + + :type inclusive_end: bool + :param inclusive_end: Boolean indicating if the end value should be + included in the range (or excluded). Defaults + to :data:`True` if ``end_value`` is passed and + no ``inclusive_end`` was given. + + :raises: :class:`ValueError ` if ``inclusive_start`` + is set but no ``start_value`` is given or if ``inclusive_end`` + is set but no ``end_value`` is given + """ + + def __init__(self, start_value=None, end_value=None, + inclusive_start=None, inclusive_end=None): + if inclusive_start is None: + inclusive_start = True + elif start_value is None: + raise ValueError('Inclusive start was specified but no ' + 'start value was given.') + self.start_value = start_value + self.inclusive_start = inclusive_start + + if inclusive_end is None: + inclusive_end = True + elif end_value is None: + raise ValueError('Inclusive end was specified but no ' + 'end value was given.') + self.end_value = end_value + self.inclusive_end = inclusive_end + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.start_value == self.start_value and + other.end_value == self.end_value and + other.inclusive_start == self.inclusive_start and + other.inclusive_end == self.inclusive_end) + + def to_pb(self): + """Converts the row filter to a protobuf. + + First converts to a :class:`.data_pb2.ValueRange` and then uses + it to create a row filter protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + value_range_kwargs = {} + if self.start_value is not None: + if self.inclusive_start: + key = 'start_value_inclusive' + else: + key = 'start_value_exclusive' + value_range_kwargs[key] = _to_bytes(self.start_value) + if self.end_value is not None: + if self.inclusive_end: + key = 'end_value_inclusive' + else: + key = 'end_value_exclusive' + value_range_kwargs[key] = _to_bytes(self.end_value) + + value_range = data_pb2.ValueRange(**value_range_kwargs) + return data_pb2.RowFilter(value_range_filter=value_range) + + +class _CellCountFilter(RowFilter): + """Row filter that uses an integer count of cells. + + The cell count is used as an offset or a limit for the number + of results returned. + + :type num_cells: int + :param num_cells: An integer count / offset / limit. + """ + + def __init__(self, num_cells): + self.num_cells = num_cells + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.num_cells == self.num_cells + + +class CellsRowOffsetFilter(_CellCountFilter): + """Row filter to skip cells in a row. + + :type num_cells: int + :param num_cells: Skips the first N cells of the row. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(cells_per_row_offset_filter=self.num_cells) + + +class CellsRowLimitFilter(_CellCountFilter): + """Row filter to limit cells in a row. + + :type num_cells: int + :param num_cells: Matches only the first N cells of the row. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(cells_per_row_limit_filter=self.num_cells) + + +class CellsColumnLimitFilter(_CellCountFilter): + """Row filter to limit cells in a column. + + :type num_cells: int + :param num_cells: Matches only the most recent N cells within each column. + This filters a (family name, column) pair, based on + timestamps of each cell. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(cells_per_column_limit_filter=self.num_cells) + + +class StripValueTransformerFilter(_BoolFilter): + """Row filter that transforms cells into empty string (0 bytes). + + :type flag: bool + :param flag: If :data:`True`, replaces each cell's value with the empty + string. As the name indicates, this is more useful as a + transformer than a generic query / filter. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(strip_value_transformer=self.flag) + + +class ApplyLabelFilter(RowFilter): + """Filter to apply labels to cells. + + Intended to be used as an intermediate filter on a pre-existing filtered + result set. This way if two sets are combined, the label can tell where + the cell(s) originated.This allows the client to determine which results + were produced from which part of the filter. + + .. note:: + + Due to a technical limitation of the backend, it is not currently + possible to apply multiple labels to a cell. + + :type label: str + :param label: Label to apply to cells in the output row. Values must be + at most 15 characters long, and match the pattern + ``[a-z0-9\\-]+``. + """ + + def __init__(self, label): + self.label = label + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.label == self.label + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(apply_label_transformer=self.label) + + +class _FilterCombination(RowFilter): + """Chain of row filters. + + Sends rows through several filters in sequence. The filters are "chained" + together to process a row. After the first filter is applied, the second + is applied to the filtered output and so on for subsequent filters. + + :type filters: list + :param filters: List of :class:`RowFilter` + """ + + def __init__(self, filters=None): + if filters is None: + filters = [] + self.filters = filters + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.filters == self.filters + + +class RowFilterChain(_FilterCombination): + """Chain of row filters. + + Sends rows through several filters in sequence. The filters are "chained" + together to process a row. After the first filter is applied, the second + is applied to the filtered output and so on for subsequent filters. + + :type filters: list + :param filters: List of :class:`RowFilter` + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + chain = data_pb2.RowFilter.Chain( + filters=[row_filter.to_pb() for row_filter in self.filters]) + return data_pb2.RowFilter(chain=chain) + + +class RowFilterUnion(_FilterCombination): + """Union of row filters. + + Sends rows through several filters simultaneously, then + merges / interleaves all the filtered results together. + + If multiple cells are produced with the same column and timestamp, + they will all appear in the output row in an unspecified mutual order. + + :type filters: list + :param filters: List of :class:`RowFilter` + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + interleave = data_pb2.RowFilter.Interleave( + filters=[row_filter.to_pb() for row_filter in self.filters]) + return data_pb2.RowFilter(interleave=interleave) + + +class ConditionalRowFilter(RowFilter): + """Conditional row filter which exhibits ternary behavior. + + Executes one of two filters based on another filter. If the ``base_filter`` + returns any cells in the row, then ``true_filter`` is executed. If not, + then ``false_filter`` is executed. + + .. note:: + + The ``base_filter`` does not execute atomically with the true and false + filters, which may lead to inconsistent or unexpected results. + + Additionally, executing a :class:`ConditionalRowFilter` has poor + performance on the server, especially when ``false_filter`` is set. + + :type base_filter: :class:`RowFilter` + :param base_filter: The filter to condition on before executing the + true/false filters. + + :type true_filter: :class:`RowFilter` + :param true_filter: (Optional) The filter to execute if there are any cells + matching ``base_filter``. If not provided, no results + will be returned in the true case. + + :type false_filter: :class:`RowFilter` + :param false_filter: (Optional) The filter to execute if there are no cells + matching ``base_filter``. If not provided, no results + will be returned in the false case. + """ + + def __init__(self, base_filter, true_filter=None, false_filter=None): + self.base_filter = base_filter + self.true_filter = true_filter + self.false_filter = false_filter + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.base_filter == self.base_filter and + other.true_filter == self.true_filter and + other.false_filter == self.false_filter) + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + condition_kwargs = {'predicate_filter': self.base_filter.to_pb()} + if self.true_filter is not None: + condition_kwargs['true_filter'] = self.true_filter.to_pb() + if self.false_filter is not None: + condition_kwargs['false_filter'] = self.false_filter.to_pb() + condition = data_pb2.RowFilter.Condition(**condition_kwargs) + return data_pb2.RowFilter(condition=condition) diff --git a/gcloud/bigtable/test_row.py b/gcloud/bigtable/test_row.py index 4caa6668e2b2..71efbb2cd57c 100644 --- a/gcloud/bigtable/test_row.py +++ b/gcloud/bigtable/test_row.py @@ -237,7 +237,7 @@ def test_delete_cells_no_time_range(self): def test_delete_cells_with_time_range(self): import datetime from gcloud._helpers import _EPOCH - from gcloud.bigtable.row import TimestampRange + from gcloud.bigtable.row_filters import TimestampRange microseconds = 30871000 # Makes sure already milliseconds granularity start = _EPOCH + datetime.timedelta(microseconds=microseconds) @@ -410,7 +410,7 @@ def test_commit(self): from gcloud.bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) from gcloud.bigtable._testing import _FakeStub - from gcloud.bigtable.row import RowSampleFilter + from gcloud.bigtable.row_filters import RowSampleFilter row_key = b'row_key' table_name = 'projects/more-stuff' @@ -670,995 +670,6 @@ def test_commit_too_many_mutations(self): row.commit() -class Test_BoolFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import _BoolFilter - return _BoolFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - flag = object() - row_filter = self._makeOne(flag) - self.assertTrue(row_filter.flag is flag) - - def test___eq__type_differ(self): - flag = object() - row_filter1 = self._makeOne(flag) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - flag = object() - row_filter1 = self._makeOne(flag) - row_filter2 = self._makeOne(flag) - self.assertEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - flag = object() - row_filter1 = self._makeOne(flag) - row_filter2 = self._makeOne(flag) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - -class TestSinkFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import SinkFilter - return SinkFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(sink=flag) - self.assertEqual(pb_val, expected_pb) - - -class TestPassAllFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import PassAllFilter - return PassAllFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(pass_all_filter=flag) - self.assertEqual(pb_val, expected_pb) - - -class TestBlockAllFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import BlockAllFilter - return BlockAllFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(block_all_filter=flag) - self.assertEqual(pb_val, expected_pb) - - -class Test_RegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import _RegexFilter - return _RegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - regex = object() - row_filter = self._makeOne(regex) - self.assertTrue(row_filter.regex is regex) - - def test___eq__type_differ(self): - regex = object() - row_filter1 = self._makeOne(regex) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - regex = object() - row_filter1 = self._makeOne(regex) - row_filter2 = self._makeOne(regex) - self.assertEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - regex = object() - row_filter1 = self._makeOne(regex) - row_filter2 = self._makeOne(regex) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - -class TestRowKeyRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import RowKeyRegexFilter - return RowKeyRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = b'row-key-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(row_key_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestRowSampleFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import RowSampleFilter - return RowSampleFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - sample = object() - row_filter = self._makeOne(sample) - self.assertTrue(row_filter.sample is sample) - - def test___eq__type_differ(self): - sample = object() - row_filter1 = self._makeOne(sample) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - sample = object() - row_filter1 = self._makeOne(sample) - row_filter2 = self._makeOne(sample) - self.assertEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - sample = 0.25 - row_filter = self._makeOne(sample) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(row_sample_filter=sample) - self.assertEqual(pb_val, expected_pb) - - -class TestFamilyNameRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import FamilyNameRegexFilter - return FamilyNameRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = u'family-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(family_name_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestColumnQualifierRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ColumnQualifierRegexFilter - return ColumnQualifierRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = b'column-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(column_qualifier_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestTimestampRange(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import TimestampRange - return TimestampRange - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - start = object() - end = object() - time_range = self._makeOne(start=start, end=end) - self.assertTrue(time_range.start is start) - self.assertTrue(time_range.end is end) - - def test___eq__(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = self._makeOne(start=start, end=end) - self.assertEqual(time_range1, time_range2) - - def test___eq__type_differ(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = object() - self.assertNotEqual(time_range1, time_range2) - - def test___ne__same_value(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = self._makeOne(start=start, end=end) - comparison_val = (time_range1 != time_range2) - self.assertFalse(comparison_val) - - def _to_pb_helper(self, start_micros=None, end_micros=None): - import datetime - from gcloud._helpers import _EPOCH - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - pb_kwargs = {} - - start = None - if start_micros is not None: - start = _EPOCH + datetime.timedelta(microseconds=start_micros) - pb_kwargs['start_timestamp_micros'] = start_micros - end = None - if end_micros is not None: - end = _EPOCH + datetime.timedelta(microseconds=end_micros) - pb_kwargs['end_timestamp_micros'] = end_micros - time_range = self._makeOne(start=start, end=end) - - expected_pb = data_pb2.TimestampRange(**pb_kwargs) - self.assertEqual(time_range.to_pb(), expected_pb) - - def test_to_pb(self): - # Makes sure already milliseconds granularity - start_micros = 30871000 - end_micros = 12939371000 - self._to_pb_helper(start_micros=start_micros, - end_micros=end_micros) - - def test_to_pb_start_only(self): - # Makes sure already milliseconds granularity - start_micros = 30871000 - self._to_pb_helper(start_micros=start_micros) - - def test_to_pb_end_only(self): - # Makes sure already milliseconds granularity - end_micros = 12939371000 - self._to_pb_helper(end_micros=end_micros) - - -class TestTimestampRangeFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import TimestampRangeFilter - return TimestampRangeFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - range_ = object() - row_filter = self._makeOne(range_) - self.assertTrue(row_filter.range_ is range_) - - def test___eq__type_differ(self): - range_ = object() - row_filter1 = self._makeOne(range_) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - range_ = object() - row_filter1 = self._makeOne(range_) - row_filter2 = self._makeOne(range_) - self.assertEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import TimestampRange - - range_ = TimestampRange() - row_filter = self._makeOne(range_) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter( - timestamp_range_filter=data_pb2.TimestampRange()) - self.assertEqual(pb_val, expected_pb) - - -class TestColumnRangeFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ColumnRangeFilter - return ColumnRangeFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - column_family_id = object() - row_filter = self._makeOne(column_family_id) - self.assertTrue(row_filter.column_family_id is column_family_id) - self.assertEqual(row_filter.start_column, None) - self.assertEqual(row_filter.end_column, None) - self.assertTrue(row_filter.inclusive_start) - self.assertTrue(row_filter.inclusive_end) - - def test_constructor_explicit(self): - column_family_id = object() - start_column = object() - end_column = object() - inclusive_start = object() - inclusive_end = object() - row_filter = self._makeOne(column_family_id, start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertTrue(row_filter.column_family_id is column_family_id) - self.assertTrue(row_filter.start_column is start_column) - self.assertTrue(row_filter.end_column is end_column) - self.assertTrue(row_filter.inclusive_start is inclusive_start) - self.assertTrue(row_filter.inclusive_end is inclusive_end) - - def test_constructor_bad_start(self): - column_family_id = object() - self.assertRaises(ValueError, self._makeOne, - column_family_id, inclusive_start=True) - - def test_constructor_bad_end(self): - column_family_id = object() - self.assertRaises(ValueError, self._makeOne, - column_family_id, inclusive_end=True) - - def test___eq__(self): - column_family_id = object() - start_column = object() - end_column = object() - inclusive_start = object() - inclusive_end = object() - row_filter1 = self._makeOne(column_family_id, - start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - row_filter2 = self._makeOne(column_family_id, - start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - column_family_id = object() - row_filter1 = self._makeOne(column_family_id) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - row_filter = self._makeOne(column_family_id) - col_range_pb = data_pb2.ColumnRange(family_name=column_family_id) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_start(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, start_column=column) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - start_qualifier_inclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_start(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, start_column=column, - inclusive_start=False) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - start_qualifier_exclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_end(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, end_column=column) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - end_qualifier_inclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_end(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, end_column=column, - inclusive_end=False) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - end_qualifier_exclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - -class TestValueRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ValueRegexFilter - return ValueRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = b'value-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(value_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestValueRangeFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ValueRangeFilter - return ValueRangeFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - row_filter = self._makeOne() - self.assertEqual(row_filter.start_value, None) - self.assertEqual(row_filter.end_value, None) - self.assertTrue(row_filter.inclusive_start) - self.assertTrue(row_filter.inclusive_end) - - def test_constructor_explicit(self): - start_value = object() - end_value = object() - inclusive_start = object() - inclusive_end = object() - row_filter = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertTrue(row_filter.start_value is start_value) - self.assertTrue(row_filter.end_value is end_value) - self.assertTrue(row_filter.inclusive_start is inclusive_start) - self.assertTrue(row_filter.inclusive_end is inclusive_end) - - def test_constructor_bad_start(self): - self.assertRaises(ValueError, self._makeOne, inclusive_start=True) - - def test_constructor_bad_end(self): - self.assertRaises(ValueError, self._makeOne, inclusive_end=True) - - def test___eq__(self): - start_value = object() - end_value = object() - inclusive_start = object() - inclusive_end = object() - row_filter1 = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - row_filter2 = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - row_filter1 = self._makeOne() - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - row_filter = self._makeOne() - expected_pb = data_pb2.RowFilter( - value_range_filter=data_pb2.ValueRange()) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_start(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(start_value=value) - val_range_pb = data_pb2.ValueRange(start_value_inclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_start(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(start_value=value, inclusive_start=False) - val_range_pb = data_pb2.ValueRange(start_value_exclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_end(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(end_value=value) - val_range_pb = data_pb2.ValueRange(end_value_inclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_end(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(end_value=value, inclusive_end=False) - val_range_pb = data_pb2.ValueRange(end_value_exclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - -class Test_CellCountFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import _CellCountFilter - return _CellCountFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - num_cells = object() - row_filter = self._makeOne(num_cells) - self.assertTrue(row_filter.num_cells is num_cells) - - def test___eq__type_differ(self): - num_cells = object() - row_filter1 = self._makeOne(num_cells) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - num_cells = object() - row_filter1 = self._makeOne(num_cells) - row_filter2 = self._makeOne(num_cells) - self.assertEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - num_cells = object() - row_filter1 = self._makeOne(num_cells) - row_filter2 = self._makeOne(num_cells) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - -class TestCellsRowOffsetFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import CellsRowOffsetFilter - return CellsRowOffsetFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - num_cells = 76 - row_filter = self._makeOne(num_cells) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(cells_per_row_offset_filter=num_cells) - self.assertEqual(pb_val, expected_pb) - - -class TestCellsRowLimitFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import CellsRowLimitFilter - return CellsRowLimitFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - num_cells = 189 - row_filter = self._makeOne(num_cells) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(cells_per_row_limit_filter=num_cells) - self.assertEqual(pb_val, expected_pb) - - -class TestCellsColumnLimitFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import CellsColumnLimitFilter - return CellsColumnLimitFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - num_cells = 10 - row_filter = self._makeOne(num_cells) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter( - cells_per_column_limit_filter=num_cells) - self.assertEqual(pb_val, expected_pb) - - -class TestStripValueTransformerFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import StripValueTransformerFilter - return StripValueTransformerFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(strip_value_transformer=flag) - self.assertEqual(pb_val, expected_pb) - - -class TestApplyLabelFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ApplyLabelFilter - return ApplyLabelFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - label = object() - row_filter = self._makeOne(label) - self.assertTrue(row_filter.label is label) - - def test___eq__type_differ(self): - label = object() - row_filter1 = self._makeOne(label) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - label = object() - row_filter1 = self._makeOne(label) - row_filter2 = self._makeOne(label) - self.assertEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - label = u'label' - row_filter = self._makeOne(label) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(apply_label_transformer=label) - self.assertEqual(pb_val, expected_pb) - - -class Test_FilterCombination(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import _FilterCombination - return _FilterCombination - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - row_filter = self._makeOne() - self.assertEqual(row_filter.filters, []) - - def test_constructor_explicit(self): - filters = object() - row_filter = self._makeOne(filters=filters) - self.assertTrue(row_filter.filters is filters) - - def test___eq__(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = self._makeOne(filters=filters) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - -class TestRowFilterChain(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import RowFilterChain - return RowFilterChain - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - chain=data_pb2.RowFilter.Chain( - filters=[row_filter1_pb, row_filter2_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_nested(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import CellsRowLimitFilter - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter2 = RowSampleFilter(0.25) - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = CellsRowLimitFilter(11) - row_filter4_pb = row_filter4.to_pb() - - row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) - filter_pb = row_filter5.to_pb() - - expected_pb = data_pb2.RowFilter( - chain=data_pb2.RowFilter.Chain( - filters=[row_filter3_pb, row_filter4_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - -class TestRowFilterUnion(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import RowFilterUnion - return RowFilterUnion - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - interleave=data_pb2.RowFilter.Interleave( - filters=[row_filter1_pb, row_filter2_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_nested(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import CellsRowLimitFilter - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter2 = RowSampleFilter(0.25) - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = CellsRowLimitFilter(11) - row_filter4_pb = row_filter4.to_pb() - - row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) - filter_pb = row_filter5.to_pb() - - expected_pb = data_pb2.RowFilter( - interleave=data_pb2.RowFilter.Interleave( - filters=[row_filter3_pb, row_filter4_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - -class TestConditionalRowFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ConditionalRowFilter - return ConditionalRowFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - self.assertTrue(cond_filter.base_filter is base_filter) - self.assertTrue(cond_filter.true_filter is true_filter) - self.assertTrue(cond_filter.false_filter is false_filter) - - def test___eq__(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter1 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - cond_filter2 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - self.assertEqual(cond_filter1, cond_filter2) - - def test___eq__type_differ(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter1 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - cond_filter2 = object() - self.assertNotEqual(cond_filter1, cond_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import CellsRowOffsetFilter - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = CellsRowOffsetFilter(11) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = self._makeOne(row_filter1, true_filter=row_filter2, - false_filter=row_filter3) - filter_pb = row_filter4.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - true_filter=row_filter2_pb, - false_filter=row_filter3_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_true_only(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(row_filter1, true_filter=row_filter2) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - true_filter=row_filter2_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_false_only(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(row_filter1, false_filter=row_filter2) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - false_filter=row_filter2_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - class Test__parse_rmw_row_response(unittest2.TestCase): def _callFUT(self, row_response): diff --git a/gcloud/bigtable/test_row_filters.py b/gcloud/bigtable/test_row_filters.py new file mode 100644 index 000000000000..90a79dceb52d --- /dev/null +++ b/gcloud/bigtable/test_row_filters.py @@ -0,0 +1,1005 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest2 + + +class Test_BoolFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import _BoolFilter + return _BoolFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + flag = object() + row_filter = self._makeOne(flag) + self.assertTrue(row_filter.flag is flag) + + def test___eq__type_differ(self): + flag = object() + row_filter1 = self._makeOne(flag) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + flag = object() + row_filter1 = self._makeOne(flag) + row_filter2 = self._makeOne(flag) + self.assertEqual(row_filter1, row_filter2) + + def test___ne__same_value(self): + flag = object() + row_filter1 = self._makeOne(flag) + row_filter2 = self._makeOne(flag) + comparison_val = (row_filter1 != row_filter2) + self.assertFalse(comparison_val) + + +class TestSinkFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import SinkFilter + return SinkFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(sink=flag) + self.assertEqual(pb_val, expected_pb) + + +class TestPassAllFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import PassAllFilter + return PassAllFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(pass_all_filter=flag) + self.assertEqual(pb_val, expected_pb) + + +class TestBlockAllFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import BlockAllFilter + return BlockAllFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(block_all_filter=flag) + self.assertEqual(pb_val, expected_pb) + + +class Test_RegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import _RegexFilter + return _RegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + regex = object() + row_filter = self._makeOne(regex) + self.assertTrue(row_filter.regex is regex) + + def test___eq__type_differ(self): + regex = object() + row_filter1 = self._makeOne(regex) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + regex = object() + row_filter1 = self._makeOne(regex) + row_filter2 = self._makeOne(regex) + self.assertEqual(row_filter1, row_filter2) + + def test___ne__same_value(self): + regex = object() + row_filter1 = self._makeOne(regex) + row_filter2 = self._makeOne(regex) + comparison_val = (row_filter1 != row_filter2) + self.assertFalse(comparison_val) + + +class TestRowKeyRegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import RowKeyRegexFilter + return RowKeyRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = b'row-key-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(row_key_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) + + +class TestRowSampleFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import RowSampleFilter + return RowSampleFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + sample = object() + row_filter = self._makeOne(sample) + self.assertTrue(row_filter.sample is sample) + + def test___eq__type_differ(self): + sample = object() + row_filter1 = self._makeOne(sample) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + sample = object() + row_filter1 = self._makeOne(sample) + row_filter2 = self._makeOne(sample) + self.assertEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + sample = 0.25 + row_filter = self._makeOne(sample) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(row_sample_filter=sample) + self.assertEqual(pb_val, expected_pb) + + +class TestFamilyNameRegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import FamilyNameRegexFilter + return FamilyNameRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = u'family-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(family_name_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) + + +class TestColumnQualifierRegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter + return ColumnQualifierRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = b'column-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(column_qualifier_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) + + +class TestTimestampRange(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import TimestampRange + return TimestampRange + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + start = object() + end = object() + time_range = self._makeOne(start=start, end=end) + self.assertTrue(time_range.start is start) + self.assertTrue(time_range.end is end) + + def test___eq__(self): + start = object() + end = object() + time_range1 = self._makeOne(start=start, end=end) + time_range2 = self._makeOne(start=start, end=end) + self.assertEqual(time_range1, time_range2) + + def test___eq__type_differ(self): + start = object() + end = object() + time_range1 = self._makeOne(start=start, end=end) + time_range2 = object() + self.assertNotEqual(time_range1, time_range2) + + def test___ne__same_value(self): + start = object() + end = object() + time_range1 = self._makeOne(start=start, end=end) + time_range2 = self._makeOne(start=start, end=end) + comparison_val = (time_range1 != time_range2) + self.assertFalse(comparison_val) + + def _to_pb_helper(self, start_micros=None, end_micros=None): + import datetime + from gcloud._helpers import _EPOCH + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + pb_kwargs = {} + + start = None + if start_micros is not None: + start = _EPOCH + datetime.timedelta(microseconds=start_micros) + pb_kwargs['start_timestamp_micros'] = start_micros + end = None + if end_micros is not None: + end = _EPOCH + datetime.timedelta(microseconds=end_micros) + pb_kwargs['end_timestamp_micros'] = end_micros + time_range = self._makeOne(start=start, end=end) + + expected_pb = data_pb2.TimestampRange(**pb_kwargs) + self.assertEqual(time_range.to_pb(), expected_pb) + + def test_to_pb(self): + # Makes sure already milliseconds granularity + start_micros = 30871000 + end_micros = 12939371000 + self._to_pb_helper(start_micros=start_micros, + end_micros=end_micros) + + def test_to_pb_start_only(self): + # Makes sure already milliseconds granularity + start_micros = 30871000 + self._to_pb_helper(start_micros=start_micros) + + def test_to_pb_end_only(self): + # Makes sure already milliseconds granularity + end_micros = 12939371000 + self._to_pb_helper(end_micros=end_micros) + + +class TestTimestampRangeFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import TimestampRangeFilter + return TimestampRangeFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + range_ = object() + row_filter = self._makeOne(range_) + self.assertTrue(row_filter.range_ is range_) + + def test___eq__type_differ(self): + range_ = object() + row_filter1 = self._makeOne(range_) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + range_ = object() + row_filter1 = self._makeOne(range_) + row_filter2 = self._makeOne(range_) + self.assertEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import TimestampRange + + range_ = TimestampRange() + row_filter = self._makeOne(range_) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter( + timestamp_range_filter=data_pb2.TimestampRange()) + self.assertEqual(pb_val, expected_pb) + + +class TestColumnRangeFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ColumnRangeFilter + return ColumnRangeFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor_defaults(self): + column_family_id = object() + row_filter = self._makeOne(column_family_id) + self.assertTrue(row_filter.column_family_id is column_family_id) + self.assertEqual(row_filter.start_column, None) + self.assertEqual(row_filter.end_column, None) + self.assertTrue(row_filter.inclusive_start) + self.assertTrue(row_filter.inclusive_end) + + def test_constructor_explicit(self): + column_family_id = object() + start_column = object() + end_column = object() + inclusive_start = object() + inclusive_end = object() + row_filter = self._makeOne(column_family_id, start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertTrue(row_filter.column_family_id is column_family_id) + self.assertTrue(row_filter.start_column is start_column) + self.assertTrue(row_filter.end_column is end_column) + self.assertTrue(row_filter.inclusive_start is inclusive_start) + self.assertTrue(row_filter.inclusive_end is inclusive_end) + + def test_constructor_bad_start(self): + column_family_id = object() + self.assertRaises(ValueError, self._makeOne, + column_family_id, inclusive_start=True) + + def test_constructor_bad_end(self): + column_family_id = object() + self.assertRaises(ValueError, self._makeOne, + column_family_id, inclusive_end=True) + + def test___eq__(self): + column_family_id = object() + start_column = object() + end_column = object() + inclusive_start = object() + inclusive_end = object() + row_filter1 = self._makeOne(column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + row_filter2 = self._makeOne(column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertEqual(row_filter1, row_filter2) + + def test___eq__type_differ(self): + column_family_id = object() + row_filter1 = self._makeOne(column_family_id) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + row_filter = self._makeOne(column_family_id) + col_range_pb = data_pb2.ColumnRange(family_name=column_family_id) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_start(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, start_column=column) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + start_qualifier_inclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_start(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, start_column=column, + inclusive_start=False) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + start_qualifier_exclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_end(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, end_column=column) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + end_qualifier_inclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_end(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, end_column=column, + inclusive_end=False) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + end_qualifier_exclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + +class TestValueRegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ValueRegexFilter + return ValueRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = b'value-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(value_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) + + +class TestValueRangeFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ValueRangeFilter + return ValueRangeFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor_defaults(self): + row_filter = self._makeOne() + self.assertEqual(row_filter.start_value, None) + self.assertEqual(row_filter.end_value, None) + self.assertTrue(row_filter.inclusive_start) + self.assertTrue(row_filter.inclusive_end) + + def test_constructor_explicit(self): + start_value = object() + end_value = object() + inclusive_start = object() + inclusive_end = object() + row_filter = self._makeOne(start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertTrue(row_filter.start_value is start_value) + self.assertTrue(row_filter.end_value is end_value) + self.assertTrue(row_filter.inclusive_start is inclusive_start) + self.assertTrue(row_filter.inclusive_end is inclusive_end) + + def test_constructor_bad_start(self): + self.assertRaises(ValueError, self._makeOne, inclusive_start=True) + + def test_constructor_bad_end(self): + self.assertRaises(ValueError, self._makeOne, inclusive_end=True) + + def test___eq__(self): + start_value = object() + end_value = object() + inclusive_start = object() + inclusive_end = object() + row_filter1 = self._makeOne(start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + row_filter2 = self._makeOne(start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertEqual(row_filter1, row_filter2) + + def test___eq__type_differ(self): + row_filter1 = self._makeOne() + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + row_filter = self._makeOne() + expected_pb = data_pb2.RowFilter( + value_range_filter=data_pb2.ValueRange()) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_start(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + value = b'some-value' + row_filter = self._makeOne(start_value=value) + val_range_pb = data_pb2.ValueRange(start_value_inclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_start(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + value = b'some-value' + row_filter = self._makeOne(start_value=value, inclusive_start=False) + val_range_pb = data_pb2.ValueRange(start_value_exclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_end(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + value = b'some-value' + row_filter = self._makeOne(end_value=value) + val_range_pb = data_pb2.ValueRange(end_value_inclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_end(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + value = b'some-value' + row_filter = self._makeOne(end_value=value, inclusive_end=False) + val_range_pb = data_pb2.ValueRange(end_value_exclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + +class Test_CellCountFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import _CellCountFilter + return _CellCountFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + num_cells = object() + row_filter = self._makeOne(num_cells) + self.assertTrue(row_filter.num_cells is num_cells) + + def test___eq__type_differ(self): + num_cells = object() + row_filter1 = self._makeOne(num_cells) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + num_cells = object() + row_filter1 = self._makeOne(num_cells) + row_filter2 = self._makeOne(num_cells) + self.assertEqual(row_filter1, row_filter2) + + def test___ne__same_value(self): + num_cells = object() + row_filter1 = self._makeOne(num_cells) + row_filter2 = self._makeOne(num_cells) + comparison_val = (row_filter1 != row_filter2) + self.assertFalse(comparison_val) + + +class TestCellsRowOffsetFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import CellsRowOffsetFilter + return CellsRowOffsetFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + num_cells = 76 + row_filter = self._makeOne(num_cells) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(cells_per_row_offset_filter=num_cells) + self.assertEqual(pb_val, expected_pb) + + +class TestCellsRowLimitFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import CellsRowLimitFilter + return CellsRowLimitFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + num_cells = 189 + row_filter = self._makeOne(num_cells) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(cells_per_row_limit_filter=num_cells) + self.assertEqual(pb_val, expected_pb) + + +class TestCellsColumnLimitFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import CellsColumnLimitFilter + return CellsColumnLimitFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + num_cells = 10 + row_filter = self._makeOne(num_cells) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter( + cells_per_column_limit_filter=num_cells) + self.assertEqual(pb_val, expected_pb) + + +class TestStripValueTransformerFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import StripValueTransformerFilter + return StripValueTransformerFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(strip_value_transformer=flag) + self.assertEqual(pb_val, expected_pb) + + +class TestApplyLabelFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ApplyLabelFilter + return ApplyLabelFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + label = object() + row_filter = self._makeOne(label) + self.assertTrue(row_filter.label is label) + + def test___eq__type_differ(self): + label = object() + row_filter1 = self._makeOne(label) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + label = object() + row_filter1 = self._makeOne(label) + row_filter2 = self._makeOne(label) + self.assertEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + label = u'label' + row_filter = self._makeOne(label) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(apply_label_transformer=label) + self.assertEqual(pb_val, expected_pb) + + +class Test_FilterCombination(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import _FilterCombination + return _FilterCombination + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor_defaults(self): + row_filter = self._makeOne() + self.assertEqual(row_filter.filters, []) + + def test_constructor_explicit(self): + filters = object() + row_filter = self._makeOne(filters=filters) + self.assertTrue(row_filter.filters is filters) + + def test___eq__(self): + filters = object() + row_filter1 = self._makeOne(filters=filters) + row_filter2 = self._makeOne(filters=filters) + self.assertEqual(row_filter1, row_filter2) + + def test___eq__type_differ(self): + filters = object() + row_filter1 = self._makeOne(filters=filters) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + +class TestRowFilterChain(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import RowFilterChain + return RowFilterChain + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) + filter_pb = row_filter3.to_pb() + + expected_pb = data_pb2.RowFilter( + chain=data_pb2.RowFilter.Chain( + filters=[row_filter1_pb, row_filter2_pb], + ), + ) + self.assertEqual(filter_pb, expected_pb) + + def test_to_pb_nested(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import CellsRowLimitFilter + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) + row_filter3_pb = row_filter3.to_pb() + + row_filter4 = CellsRowLimitFilter(11) + row_filter4_pb = row_filter4.to_pb() + + row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) + filter_pb = row_filter5.to_pb() + + expected_pb = data_pb2.RowFilter( + chain=data_pb2.RowFilter.Chain( + filters=[row_filter3_pb, row_filter4_pb], + ), + ) + self.assertEqual(filter_pb, expected_pb) + + +class TestRowFilterUnion(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import RowFilterUnion + return RowFilterUnion + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) + filter_pb = row_filter3.to_pb() + + expected_pb = data_pb2.RowFilter( + interleave=data_pb2.RowFilter.Interleave( + filters=[row_filter1_pb, row_filter2_pb], + ), + ) + self.assertEqual(filter_pb, expected_pb) + + def test_to_pb_nested(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import CellsRowLimitFilter + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) + row_filter3_pb = row_filter3.to_pb() + + row_filter4 = CellsRowLimitFilter(11) + row_filter4_pb = row_filter4.to_pb() + + row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) + filter_pb = row_filter5.to_pb() + + expected_pb = data_pb2.RowFilter( + interleave=data_pb2.RowFilter.Interleave( + filters=[row_filter3_pb, row_filter4_pb], + ), + ) + self.assertEqual(filter_pb, expected_pb) + + +class TestConditionalRowFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ConditionalRowFilter + return ConditionalRowFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + base_filter = object() + true_filter = object() + false_filter = object() + cond_filter = self._makeOne(base_filter, + true_filter=true_filter, + false_filter=false_filter) + self.assertTrue(cond_filter.base_filter is base_filter) + self.assertTrue(cond_filter.true_filter is true_filter) + self.assertTrue(cond_filter.false_filter is false_filter) + + def test___eq__(self): + base_filter = object() + true_filter = object() + false_filter = object() + cond_filter1 = self._makeOne(base_filter, + true_filter=true_filter, + false_filter=false_filter) + cond_filter2 = self._makeOne(base_filter, + true_filter=true_filter, + false_filter=false_filter) + self.assertEqual(cond_filter1, cond_filter2) + + def test___eq__type_differ(self): + base_filter = object() + true_filter = object() + false_filter = object() + cond_filter1 = self._makeOne(base_filter, + true_filter=true_filter, + false_filter=false_filter) + cond_filter2 = object() + self.assertNotEqual(cond_filter1, cond_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import CellsRowOffsetFilter + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = CellsRowOffsetFilter(11) + row_filter3_pb = row_filter3.to_pb() + + row_filter4 = self._makeOne(row_filter1, true_filter=row_filter2, + false_filter=row_filter3) + filter_pb = row_filter4.to_pb() + + expected_pb = data_pb2.RowFilter( + condition=data_pb2.RowFilter.Condition( + predicate_filter=row_filter1_pb, + true_filter=row_filter2_pb, + false_filter=row_filter3_pb, + ), + ) + self.assertEqual(filter_pb, expected_pb) + + def test_to_pb_true_only(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = self._makeOne(row_filter1, true_filter=row_filter2) + filter_pb = row_filter3.to_pb() + + expected_pb = data_pb2.RowFilter( + condition=data_pb2.RowFilter.Condition( + predicate_filter=row_filter1_pb, + true_filter=row_filter2_pb, + ), + ) + self.assertEqual(filter_pb, expected_pb) + + def test_to_pb_false_only(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = self._makeOne(row_filter1, false_filter=row_filter2) + filter_pb = row_filter3.to_pb() + + expected_pb = data_pb2.RowFilter( + condition=data_pb2.RowFilter.Condition( + predicate_filter=row_filter1_pb, + false_filter=row_filter2_pb, + ), + ) + self.assertEqual(filter_pb, expected_pb) diff --git a/gcloud/bigtable/test_table.py b/gcloud/bigtable/test_table.py index 65cfe0d87fe2..9fcdf21593b0 100644 --- a/gcloud/bigtable/test_table.py +++ b/gcloud/bigtable/test_table.py @@ -582,7 +582,7 @@ def test_row_range_both_keys(self): def test_with_filter(self): from gcloud.bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) - from gcloud.bigtable.row import RowSampleFilter + from gcloud.bigtable.row_filters import RowSampleFilter table_name = 'table_name' row_filter = RowSampleFilter(0.33) diff --git a/scripts/pylintrc_default b/scripts/pylintrc_default index 4a76e193baae..a06fd338f61d 100644 --- a/scripts/pylintrc_default +++ b/scripts/pylintrc_default @@ -200,7 +200,7 @@ no-space-check = # Maximum number of lines in a module # DEFAULT: max-module-lines=1000 # RATIONALE: API-mapping -max-module-lines=1593 +max-module-lines=1200 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). diff --git a/scripts/run_pylint.py b/scripts/run_pylint.py index 1e92422fc45c..7bc418b5cdac 100644 --- a/scripts/run_pylint.py +++ b/scripts/run_pylint.py @@ -66,7 +66,7 @@ } TEST_RC_REPLACEMENTS = { 'FORMAT': { - 'max-module-lines': 1825, + 'max-module-lines': 1700, }, } diff --git a/system_tests/bigtable.py b/system_tests/bigtable.py index 5066e2e710de..39d60e0658f6 100644 --- a/system_tests/bigtable.py +++ b/system_tests/bigtable.py @@ -24,10 +24,10 @@ from gcloud._helpers import UTC from gcloud.bigtable.client import Client from gcloud.bigtable.column_family import MaxVersionsGCRule -from gcloud.bigtable.row import ApplyLabelFilter -from gcloud.bigtable.row import ColumnQualifierRegexFilter -from gcloud.bigtable.row import RowFilterChain -from gcloud.bigtable.row import RowFilterUnion +from gcloud.bigtable.row_filters import ApplyLabelFilter +from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter +from gcloud.bigtable.row_filters import RowFilterChain +from gcloud.bigtable.row_filters import RowFilterUnion from gcloud.bigtable.row_data import Cell from gcloud.bigtable.row_data import PartialRowData from gcloud.environment_vars import TESTS_PROJECT