[explore view] add partition as adhoc filter option (#9637)

* [explore view] add partition as adhoc option * use adhocFilter Simple Tab * simplify conditional check for custom adhoc filter operator * add simple unit tests
2020-04-28 23:09:44 -07:00 · 2020-04-28 23:09:44 -07:00 · 735dcd2002
parent 8d2165d96d
commit 735dcd2002
7 changed files with 176 additions and 24 deletions
--- a/superset-frontend/spec/javascripts/explore/components/AdhocFilterEditPopoverSimpleTabContent_spec.jsx
+++ b/superset-frontend/spec/javascripts/explore/components/AdhocFilterEditPopoverSimpleTabContent_spec.jsx
@ -52,6 +52,12 @@ const sumValueAdhocMetric = new AdhocMetric({
  aggregate: AGGREGATES.SUM,
 });

+const simpleCustomFilter = new AdhocFilter({
+  expressionType: EXPRESSION_TYPES.SIMPLE,
+  subject: 'ds',
+  operator: 'LATEST PARTITION',
+});
+
 const options = [
  { type: 'VARCHAR(255)', column_name: 'source' },
  { type: 'VARCHAR(255)', column_name: 'target' },
@ -155,6 +161,56 @@ describe('AdhocFilterEditPopoverSimpleTabContent', () => {
    expect(wrapper.instance().isOperatorRelevant('LIKE')).toBe(false);
  });

+  it('will show LATEST PARTITION operator', () => {
+    const { wrapper } = setup({
+      datasource: {
+        type: 'table',
+        datasource_name: 'table1',
+        schema: 'schema',
+      },
+      adhocFilter: simpleCustomFilter,
+      partitionColumn: 'ds',
+    });
+
+    expect(
+      wrapper.instance().isOperatorRelevant('LATEST PARTITION', 'ds'),
+    ).toBe(true);
+    expect(
+      wrapper.instance().isOperatorRelevant('LATEST PARTITION', 'value'),
+    ).toBe(false);
+  });
+
+  it('will generate custom sqlExpression for LATEST PARTITION operator', () => {
+    const testAdhocFilter = new AdhocFilter({
+      expressionType: EXPRESSION_TYPES.SIMPLE,
+      subject: 'ds',
+    });
+    const { wrapper, onChange } = setup({
+      datasource: {
+        type: 'table',
+        datasource_name: 'table1',
+        schema: 'schema',
+      },
+      adhocFilter: testAdhocFilter,
+      partitionColumn: 'ds',
+    });
+
+    wrapper.instance().onOperatorChange({ operator: 'LATEST PARTITION' });
+    expect(
+      onChange.lastCall.args[0].equals(
+        testAdhocFilter.duplicateWith({
+          subject: 'ds',
+          operator: 'LATEST PARTITION',
+          comparator: null,
+          clause: 'WHERE',
+          expressionType: 'SQL',
+          sqlExpression:
+            "ds = '{{ presto.latest_partition('schema.table1') }}' ",
+        }),
+      ),
+    ).toBe(true);
+  });
+
  it('expands when its multi comparator input field expands', () => {
    const { wrapper, onHeightChange } = setup();

--- a/superset-frontend/src/explore/AdhocFilter.js
+++ b/superset-frontend/src/explore/AdhocFilter.js
@ -16,7 +16,7 @@
 * specific language governing permissions and limitations
 * under the License.
 */
-import { MULTI_OPERATORS } from './constants';
+import { MULTI_OPERATORS, CUSTOM_OPERATORS } from './constants';

 export const EXPRESSION_TYPES = {
  SIMPLE: 'SIMPLE',
@ -41,16 +41,22 @@ const OPERATORS_TO_SQL = {
  regex: 'regex',
  'IS NOT NULL': 'IS NOT NULL',
  'IS NULL': 'IS NULL',
+  'LATEST PARTITION': ({ datasource }) => {
+    return `= '{{ presto.latest_partition('${datasource.schema}.${datasource.datasource_name}') }}'`;
+  },
 };

 function translateToSql(adhocMetric, { useSimple } = {}) {
  if (adhocMetric.expressionType === EXPRESSION_TYPES.SIMPLE || useSimple) {
    const isMulti = MULTI_OPERATORS.indexOf(adhocMetric.operator) >= 0;
    const subject = adhocMetric.subject;
-    const operator = OPERATORS_TO_SQL[adhocMetric.operator];
+    const operator =
+      adhocMetric.operator && CUSTOM_OPERATORS.includes(adhocMetric.operator)
+        ? OPERATORS_TO_SQL[adhocMetric.operator](adhocMetric)
+        : OPERATORS_TO_SQL[adhocMetric.operator];
    const comparator = Array.isArray(adhocMetric.comparator)
      ? adhocMetric.comparator.join("','")
-      : adhocMetric.comparator;
+      : adhocMetric.comparator || '';
    return `${subject} ${operator} ${isMulti ? "('" : ''}${comparator}${
      isMulti ? "')" : ''
    }`;
@ -75,8 +81,16 @@ export default class AdhocFilter {
          ? adhocFilter.sqlExpression
          : translateToSql(adhocFilter, { useSimple: true });
      this.clause = adhocFilter.clause;
-      this.subject = null;
-      this.operator = null;
+      if (
+        adhocFilter.operator &&
+        CUSTOM_OPERATORS.includes(adhocFilter.operator)
+      ) {
+        this.subject = adhocFilter.subject;
+        this.operator = adhocFilter.operator;
+      } else {
+        this.subject = null;
+        this.operator = null;
+      }
      this.comparator = null;
    }
    this.isExtra = !!adhocFilter.isExtra;
--- a/superset-frontend/src/explore/components/AdhocFilterEditPopover.jsx
+++ b/superset-frontend/src/explore/components/AdhocFilterEditPopover.jsx
@ -39,6 +39,7 @@ const propTypes = {
    ]),
  ).isRequired,
  datasource: PropTypes.object,
+  partitionColumn: PropTypes.string,
 };

 const startingWidth = 300;
@ -117,6 +118,7 @@ export default class AdhocFilterEditPopover extends React.Component {
      onClose,
      onResize,
      datasource,
+      partitionColumn,
      ...popoverProps
    } = this.props;

@ -141,9 +143,10 @@ export default class AdhocFilterEditPopover extends React.Component {
            <AdhocFilterEditPopoverSimpleTabContent
              adhocFilter={this.state.adhocFilter}
              onChange={this.onAdhocFilterChange}
-              options={this.props.options}
-              datasource={this.props.datasource}
+              options={options}
+              datasource={datasource}
              onHeightChange={this.adjustHeight}
+              partitionColumn={partitionColumn}
            />
          </Tab>
          <Tab
--- a/superset-frontend/src/explore/components/AdhocFilterEditPopoverSimpleTabContent.jsx
+++ b/superset-frontend/src/explore/components/AdhocFilterEditPopoverSimpleTabContent.jsx
@ -32,6 +32,8 @@ import {
  DRUID_ONLY_OPERATORS,
  HAVING_OPERATORS,
  MULTI_OPERATORS,
+  CUSTOM_OPERATORS,
+  DISABLE_INPUT_OPERATORS,
 } from '../constants';
 import FilterDefinitionOption from './FilterDefinitionOption';
 import OnPasteSelect from '../../components/OnPasteSelect';
@ -50,6 +52,7 @@ const propTypes = {
  ).isRequired,
  onHeightChange: PropTypes.func.isRequired,
  datasource: PropTypes.object,
+  partitionColumn: PropTypes.string,
 };

 const defaultProps = {
@ -63,6 +66,8 @@ function translateOperator(operator) {
    return 'not equal to';
  } else if (operator === OPERATORS.LIKE) {
    return 'like';
+  } else if (operator === OPERATORS['LATEST PARTITION']) {
+    return 'use latest_partition template';
  }
  return operator;
 }
@ -124,10 +129,15 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon
      subject = option.saved_metric_name || option.label;
      clause = CLAUSES.HAVING;
    }
+    const { operator } = this.props.adhocFilter;
    this.props.onChange(
      this.props.adhocFilter.duplicateWith({
        subject,
        clause,
+        operator:
+          operator && this.isOperatorRelevant(operator, subject)
+            ? operator
+            : null,
        expressionType: EXPRESSION_TYPES.SIMPLE,
      }),
    );
@ -147,13 +157,26 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon
        ? currentComparator[0]
        : currentComparator;
    }
-    this.props.onChange(
-      this.props.adhocFilter.duplicateWith({
-        operator: operator && operator.operator,
-        comparator: newComparator,
-        expressionType: EXPRESSION_TYPES.SIMPLE,
-      }),
-    );
+
+    if (operator && CUSTOM_OPERATORS.includes(operator.operator)) {
+      this.props.onChange(
+        this.props.adhocFilter.duplicateWith({
+          subject: this.props.adhocFilter.subject,
+          clause: CLAUSES.WHERE,
+          operator: operator && operator.operator,
+          expressionType: EXPRESSION_TYPES.SQL,
+          datasource: this.props.datasource,
+        }),
+      );
+    } else {
+      this.props.onChange(
+        this.props.adhocFilter.duplicateWith({
+          operator: operator && operator.operator,
+          comparator: newComparator,
+          expressionType: EXPRESSION_TYPES.SIMPLE,
+        }),
+      );
+    }
  }

  onInputComparatorChange(event) {
@ -220,7 +243,12 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon
    }
  }

-  isOperatorRelevant(operator) {
+  isOperatorRelevant(operator, subject) {
+    if (operator && CUSTOM_OPERATORS.includes(operator)) {
+      const { partitionColumn } = this.props;
+      return partitionColumn && subject && subject === partitionColumn;
+    }
+
    return !(
      (this.props.datasource.type === 'druid' &&
        TABLE_ONLY_OPERATORS.indexOf(operator) >= 0) ||
@ -282,7 +310,9 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon
    const operatorSelectProps = {
      placeholder: t('%s operators(s)', Object.keys(OPERATORS).length),
      options: Object.keys(OPERATORS)
-        .filter(this.isOperatorRelevant)
+        .filter(operator =>
+          this.isOperatorRelevant(operator, adhocFilter.subject),
+        )
        .map(operator => ({ operator })),
      value: adhocFilter.operator,
      onChange: this.onOperatorChange,
@ -317,10 +347,7 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon
              showHeader={false}
              noResultsText={t('type a value here')}
              refFunc={this.multiComparatorRef}
-              disabled={
-                adhocFilter.operator === 'IS NOT NULL' ||
-                adhocFilter.operator === 'IS NULL'
-              }
+              disabled={DISABLE_INPUT_OPERATORS.includes(adhocFilter.operator)}
            />
          ) : (
            <input
@ -330,10 +357,7 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon
              value={adhocFilter.comparator || ''}
              className="form-control input-sm"
              placeholder={t('Filter value')}
-              disabled={
-                adhocFilter.operator === 'IS NOT NULL' ||
-                adhocFilter.operator === 'IS NULL'
-              }
+              disabled={DISABLE_INPUT_OPERATORS.includes(adhocFilter.operator)}
            />
          )}
        </FormGroup>
--- a/superset-frontend/src/explore/components/AdhocFilterOption.jsx
+++ b/superset-frontend/src/explore/components/AdhocFilterOption.jsx
@ -38,6 +38,7 @@ const propTypes = {
    ]),
  ).isRequired,
  datasource: PropTypes.object,
+  partitionColumn: PropTypes.string,
 };

 export default class AdhocFilterOption extends React.PureComponent {
@ -80,6 +81,7 @@ export default class AdhocFilterOption extends React.PureComponent {
        onClose={this.closeFilterEditOverlay}
        options={this.props.options}
        datasource={this.props.datasource}
+        partitionColumn={this.props.partitionColumn}
      />
    );
    return (
--- a/superset-frontend/src/explore/components/controls/AdhocFilterControl.jsx
+++ b/superset-frontend/src/explore/components/controls/AdhocFilterControl.jsx
@ -21,6 +21,8 @@ import PropTypes from 'prop-types';
 import VirtualizedSelect from 'react-virtualized-select';

 import { t } from '@superset-ui/translation';
+import { SupersetClient } from '@superset-ui/connection';
+
 import ControlHeader from '../ControlHeader';
 import adhocFilterType from '../../propTypes/adhocFilterType';
 import adhocMetricType from '../../propTypes/adhocMetricType';
@ -90,6 +92,46 @@ export default class AdhocFilterControl extends React.Component {
    };
  }

+  componentDidMount() {
+    const { datasource } = this.props;
+    if (datasource && datasource.type === 'table') {
+      const dbId = datasource.database ? datasource.database.id : null;
+      const datasourceName = datasource.datasource_name;
+      const datasourceSchema = datasource.schema;
+
+      if (dbId && datasourceName && datasourceSchema) {
+        SupersetClient.get({
+          endpoint: `/superset/extra_table_metadata/${dbId}/${datasourceName}/${datasourceSchema}/`,
+        }).then(
+          ({ json }) => {
+            if (json && json.partitions) {
+              const partitions = json.partitions;
+              // for now only show latest_partition option
+              // when table datasource has only 1 partition key.
+              if (
+                partitions &&
+                partitions.cols &&
+                Object.keys(partitions.cols).length === 1
+              ) {
+                const partitionColumn = partitions.cols[0];
+                this.valueRenderer = adhocFilter => (
+                  <AdhocFilterOption
+                    adhocFilter={adhocFilter}
+                    onFilterEdit={this.onFilterEdit}
+                    options={this.state.options}
+                    datasource={this.props.datasource}
+                    partitionColumn={partitionColumn}
+                  />
+                );
+              }
+            }
+          },
+          // no error handler, in case of error do not show partition option
+        );
+      }
+    }
+  }
+
  UNSAFE_componentWillReceiveProps(nextProps) {
    if (
      this.props.columns !== nextProps.columns ||
--- a/superset-frontend/src/explore/constants.js
+++ b/superset-frontend/src/explore/constants.js
@ -40,6 +40,7 @@ export const OPERATORS = {
  regex: 'regex',
  'IS NOT NULL': 'IS NOT NULL',
  'IS NULL': 'IS NULL',
+  'LATEST PARTITION': 'LATEST PARTITION',
 };

 export const TABLE_ONLY_OPERATORS = [OPERATORS.LIKE];
@ -53,6 +54,16 @@ export const HAVING_OPERATORS = [
  OPERATORS['<='],
 ];
 export const MULTI_OPERATORS = [OPERATORS.in, OPERATORS['not in']];
+// CUSTOM_OPERATORS will show operator in simple mode,
+// but will generate customized sqlExpression
+export const CUSTOM_OPERATORS = [OPERATORS['LATEST PARTITION']];
+// DISABLE_INPUT_OPERATORS will disable filter value input
+// in adhocFilter control
+export const DISABLE_INPUT_OPERATORS = [
+  OPERATORS['IS NOT NULL'],
+  OPERATORS['IS NULL'],
+  OPERATORS['LATEST PARTITION'],
+];

 export const sqlaAutoGeneratedMetricNameRegex = /^(sum|min|max|avg|count|count_distinct)__.*$/i;
 export const sqlaAutoGeneratedMetricRegex = /^(LONG|DOUBLE|FLOAT)?(SUM|AVG|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i;