fix(virtual dataset sync): Sync virtual dataset columns when changing the SQL query (#30903)

Co-authored-by: Kamil Gabryjelski <kamil.gabryjelski@gmail.com>

superset-frontend/cypress-base/cypress/e2e/explore/control.test.ts

@@ -51,8 +51,8 @@ describe('Datasource control', () => {
     )
       .first()
       .focus();
-    cy.focused().clear();
+    cy.focused().clear({ force: true });
-    cy.focused().type(`${newMetricName}{enter}`);
+    cy.focused().type(`${newMetricName}{enter}`, { force: true });
 
     cy.get('[data-test="datasource-modal-save"]').click();
     cy.get('.antd5-modal-confirm-btns button').contains('OK').click();

superset-frontend/jest.config.js

@@ -75,4 +75,5 @@ module.exports = {
       },
     ],
   ],
+  testTimeout: 10000,
 };

superset-frontend/packages/superset-ui-chart-controls/src/types.ts

@@ -84,6 +84,12 @@ export interface Dataset {
   filter_select?: boolean;
   filter_select_enabled?: boolean;
   column_names?: string[];
+  catalog?: string;
+  schema?: string;
+  table_name?: string;
+  database?: Record<string, unknown>;
+  normalize_columns?: boolean;
+  always_filter_main_dttm?: boolean;
 }
 
 export interface ControlPanelState {

superset-frontend/packages/superset-ui-core/src/query/types/Metric.ts

@@ -17,7 +17,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import { Maybe, QueryFormMetric } from '../../types';
+import { Currency, Maybe, QueryFormMetric } from '../../types';
 import { Column } from './Column';
 
 export type Aggregate =
@@ -65,7 +65,7 @@ export interface Metric {
   certification_details?: Maybe<string>;
   certified_by?: Maybe<string>;
   d3format?: Maybe<string>;
-  currency?: Maybe<string>;
+  currency?: Maybe<Currency>;
   description?: Maybe<string>;
   is_certified?: boolean;
   verbose_name?: string;

superset-frontend/plugins/plugin-chart-echarts/test/BigNumber/transformProps.test.ts

@@ -173,7 +173,7 @@ describe('BigNumberWithTrendline', () => {
               label: 'value',
               metric_name: 'value',
               d3format: '.2f',
-              currency: `{symbol: 'USD', symbolPosition: 'prefix' }`,
+              currency: { symbol: 'USD', symbolPosition: 'prefix' },
             },
           ],
         },

superset-frontend/src/components/Datasource/DatasourceEditor.jsx

@@ -23,7 +23,6 @@ import { Radio } from 'src/components/Radio';
 import Card from 'src/components/Card';
 import Alert from 'src/components/Alert';
 import Badge from 'src/components/Badge';
-import { nanoid } from 'nanoid';
 import {
   css,
   isFeatureEnabled,
@@ -57,6 +56,7 @@ import CurrencyControl from 'src/explore/components/controls/CurrencyControl';
 import CollectionTable from './CollectionTable';
 import Fieldset from './Fieldset';
 import Field from './Field';
+import { fetchSyncedColumns, updateColumns } from './utils';
 
 const DatasourceContainer = styled.div`
   .change-warning {
@@ -140,6 +140,14 @@ const StyledButtonWrapper = styled.span`
   `}
 `;
 
+const sqlTooltipOptions = {
+  placement: 'topRight',
+  title: t(
+    'If changes are made to your SQL query, ' +
+      'columns in your dataset will be synced when saving the dataset.',
+  ),
+};
+
 const checkboxGenerator = (d, onChange) => (
   <CheckboxControl value={d} onChange={onChange} />
 );
@@ -694,116 +702,27 @@ class DatasourceEditor extends PureComponent {
     });
   }
 
-  updateColumns(cols) {
+  async syncMetadata() {
-    // cols: Array<{column_name: string; is_dttm: boolean; type: string;}>
-    const { databaseColumns } = this.state;
-    const databaseColumnNames = cols.map(col => col.column_name);
-    const currentCols = databaseColumns.reduce(
-      (agg, col) => ({
-        ...agg,
-        [col.column_name]: col,
-      }),
-      {},
-    );
-    const finalColumns = [];
-    const results = {
-      added: [],
-      modified: [],
-      removed: databaseColumns
-        .map(col => col.column_name)
-        .filter(col => !databaseColumnNames.includes(col)),
-    };
-    cols.forEach(col => {
-      const currentCol = currentCols[col.column_name];
-      if (!currentCol) {
-        // new column
-        finalColumns.push({
-          id: nanoid(),
-          column_name: col.column_name,
-          type: col.type,
-          groupby: true,
-          filterable: true,
-          is_dttm: col.is_dttm,
-        });
-        results.added.push(col.column_name);
-      } else if (
-        currentCol.type !== col.type ||
-        (!currentCol.is_dttm && col.is_dttm)
-      ) {
-        // modified column
-        finalColumns.push({
-          ...currentCol,
-          type: col.type,
-          is_dttm: currentCol.is_dttm || col.is_dttm,
-        });
-        results.modified.push(col.column_name);
-      } else {
-        // unchanged
-        finalColumns.push(currentCol);
-      }
-    });
-    if (
-      results.added.length ||
-      results.modified.length ||
-      results.removed.length
-    ) {
-      this.setColumns({ databaseColumns: finalColumns });
-    }
-    return results;
-  }
-
-  syncMetadata() {
     const { datasource } = this.state;
-    const params = {
-      datasource_type: datasource.type || datasource.datasource_type,
-      database_name:
-        datasource.database.database_name || datasource.database.name,
-      catalog_name: datasource.catalog,
-      schema_name: datasource.schema,
-      table_name: datasource.table_name,
-      normalize_columns: datasource.normalize_columns,
-      always_filter_main_dttm: datasource.always_filter_main_dttm,
-    };
-    Object.entries(params).forEach(([key, value]) => {
-      // rison can't encode the undefined value
-      if (value === undefined) {
-        params[key] = null;
-      }
-    });
-    const endpoint = `/datasource/external_metadata_by_name/?q=${rison.encode_uri(
-      params,
-    )}`;
     this.setState({ metadataLoading: true });
-
+    try {
-    SupersetClient.get({ endpoint })
+      const newCols = await fetchSyncedColumns(datasource);
-      .then(({ json }) => {
+      const columnChanges = updateColumns(
-        const results = this.updateColumns(json);
+        datasource.columns,
-        if (results.modified.length) {
+        newCols,
-          this.props.addSuccessToast(
+        this.props.addSuccessToast,
-            t('Modified columns: %s', results.modified.join(', ')),
-          );
-        }
-        if (results.removed.length) {
-          this.props.addSuccessToast(
-            t('Removed columns: %s', results.removed.join(', ')),
-          );
-        }
-        if (results.added.length) {
-          this.props.addSuccessToast(
-            t('New columns added: %s', results.added.join(', ')),
-          );
-        }
-        this.props.addSuccessToast(t('Metadata has been synced'));
-        this.setState({ metadataLoading: false });
-      })
-      .catch(response =>
-        getClientErrorObject(response).then(({ error, statusText }) => {
-          this.props.addDangerToast(
-            error || statusText || t('An error has occurred'),
-          );
-          this.setState({ metadataLoading: false });
-        }),
       );
+      this.setColumns({ databaseColumns: columnChanges.finalColumns });
+      this.props.addSuccessToast(t('Metadata has been synced'));
+      this.setState({ metadataLoading: false });
+    } catch (error) {
+      const { error: clientError, statusText } =
+        await getClientErrorObject(error);
+      this.props.addDangerToast(
+        clientError || statusText || t('An error has occurred'),
+      );
+      this.setState({ metadataLoading: false });
+    }
   }
 
   findDuplicates(arr, accessor) {
@@ -1146,6 +1065,7 @@ class DatasourceEditor extends PureComponent {
                         maxLines={Infinity}
                         readOnly={!this.state.isEditMode}
                         resize="both"
+                        tooltipOptions={sqlTooltipOptions}
                       />
                     }
                   />

superset-frontend/src/components/Datasource/DatasourceModal.tsx

@@ -17,11 +17,11 @@
  * under the License.
  */
 import { FunctionComponent, useState, useRef } from 'react';
+import { useDispatch, useSelector } from 'react-redux';
 import Alert from 'src/components/Alert';
 import Button from 'src/components/Button';
 import {
   isDefined,
-  Metric,
   styled,
   SupersetClient,
   getClientErrorObject,
@@ -33,7 +33,16 @@ import Modal from 'src/components/Modal';
 import AsyncEsmComponent from 'src/components/AsyncEsmComponent';
 import ErrorMessageWithStackTrace from 'src/components/ErrorMessage/ErrorMessageWithStackTrace';
 import withToasts from 'src/components/MessageToasts/withToasts';
-import { useSelector } from 'react-redux';
+import {
+  startMetaDataLoading,
+  stopMetaDataLoading,
+  syncDatasourceMetadata,
+} from 'src/explore/actions/exploreActions';
+import {
+  fetchSyncedColumns,
+  updateColumns,
+} from 'src/components/Datasource/utils';
+import { DatasetObject } from '../../features/datasets/types';
 
 const DatasourceEditor = AsyncEsmComponent(() => import('./DatasourceEditor'));
 
@@ -60,14 +69,17 @@ const StyledDatasourceModal = styled(Modal)`
 
 interface DatasourceModalProps {
   addSuccessToast: (msg: string) => void;
-  datasource: any;
+  addDangerToast: (msg: string) => void;
+  datasource: DatasetObject;
   onChange: () => {};
   onDatasourceSave: (datasource: object, errors?: Array<any>) => {};
   onHide: () => {};
   show: boolean;
 }
 
-function buildExtraJsonObject(item: Record<string, unknown>) {
+function buildExtraJsonObject(
+  item: DatasetObject['metrics'][0] | DatasetObject['columns'][0],
+) {
   const certification =
     item?.certified_by || item?.certification_details
       ? {
@@ -83,18 +95,14 @@ function buildExtraJsonObject(item: Record<string, unknown>) {
 
 const DatasourceModal: FunctionComponent<DatasourceModalProps> = ({
   addSuccessToast,
+  addDangerToast,
   datasource,
   onDatasourceSave,
   onHide,
   show,
 }) => {
-  const [currentDatasource, setCurrentDatasource] = useState({
+  const dispatch = useDispatch();
-    ...datasource,
+  const [currentDatasource, setCurrentDatasource] = useState(datasource);
-    metrics: datasource?.metrics?.map((metric: Metric) => ({
-      ...metric,
-      currency: JSON.parse(metric.currency || 'null'),
-    })),
-  });
   const currencies = useSelector<
     {
       common: {
@@ -108,130 +116,145 @@ const DatasourceModal: FunctionComponent<DatasourceModalProps> = ({
   const [isEditing, setIsEditing] = useState<boolean>(false);
   const dialog = useRef<any>(null);
   const [modal, contextHolder] = Modal.useModal();
-
+  const buildPayload = (datasource: Record<string, any>) => ({
-  const onConfirmSave = () => {
+    table_name: datasource.table_name,
+    database_id: datasource.database?.id,
+    sql: datasource.sql,
+    filter_select_enabled: datasource.filter_select_enabled,
+    fetch_values_predicate: datasource.fetch_values_predicate,
+    schema:
+      datasource.tableSelector?.schema ||
+      datasource.databaseSelector?.schema ||
+      datasource.schema,
+    description: datasource.description,
+    main_dttm_col: datasource.main_dttm_col,
+    normalize_columns: datasource.normalize_columns,
+    always_filter_main_dttm: datasource.always_filter_main_dttm,
+    offset: datasource.offset,
+    default_endpoint: datasource.default_endpoint,
+    cache_timeout:
+      datasource.cache_timeout === '' ? null : datasource.cache_timeout,
+    is_sqllab_view: datasource.is_sqllab_view,
+    template_params: datasource.template_params,
+    extra: datasource.extra,
+    is_managed_externally: datasource.is_managed_externally,
+    external_url: datasource.external_url,
+    metrics: datasource?.metrics?.map((metric: DatasetObject['metrics'][0]) => {
+      const metricBody: any = {
+        expression: metric.expression,
+        description: metric.description,
+        metric_name: metric.metric_name,
+        metric_type: metric.metric_type,
+        d3format: metric.d3format || null,
+        currency: !isDefined(metric.currency)
+          ? null
+          : JSON.stringify(metric.currency),
+        verbose_name: metric.verbose_name,
+        warning_text: metric.warning_text,
+        uuid: metric.uuid,
+        extra: buildExtraJsonObject(metric),
+      };
+      if (!Number.isNaN(Number(metric.id))) {
+        metricBody.id = metric.id;
+      }
+      return metricBody;
+    }),
+    columns: datasource?.columns?.map(
+      (column: DatasetObject['columns'][0]) => ({
+        id: typeof column.id === 'number' ? column.id : undefined,
+        column_name: column.column_name,
+        type: column.type,
+        advanced_data_type: column.advanced_data_type,
+        verbose_name: column.verbose_name,
+        description: column.description,
+        expression: column.expression,
+        filterable: column.filterable,
+        groupby: column.groupby,
+        is_active: column.is_active,
+        is_dttm: column.is_dttm,
+        python_date_format: column.python_date_format || null,
+        uuid: column.uuid,
+        extra: buildExtraJsonObject(column),
+      }),
+    ),
+    owners: datasource.owners.map(
+      (o: Record<string, number>) => o.value || o.id,
+    ),
+  });
+  const onConfirmSave = async () => {
     // Pull out extra fields into the extra object
-    const schema =
-      currentDatasource.tableSelector?.schema ||
-      currentDatasource.databaseSelector?.schema ||
-      currentDatasource.schema;
-
     setIsSaving(true);
-    SupersetClient.put({
+    try {
-      endpoint: `/api/v1/dataset/${currentDatasource.id}`,
+      await SupersetClient.put({
-      jsonPayload: {
+        endpoint: `/api/v1/dataset/${currentDatasource.id}`,
-        table_name: currentDatasource.table_name,
+        jsonPayload: buildPayload(currentDatasource),
-        database_id: currentDatasource.database?.id,
-        sql: currentDatasource.sql,
-        filter_select_enabled: currentDatasource.filter_select_enabled,
-        fetch_values_predicate: currentDatasource.fetch_values_predicate,
-        schema,
-        description: currentDatasource.description,
-        main_dttm_col: currentDatasource.main_dttm_col,
-        normalize_columns: currentDatasource.normalize_columns,
-        always_filter_main_dttm: currentDatasource.always_filter_main_dttm,
-        offset: currentDatasource.offset,
-        default_endpoint: currentDatasource.default_endpoint,
-        cache_timeout:
-          currentDatasource.cache_timeout === ''
-            ? null
-            : currentDatasource.cache_timeout,
-        is_sqllab_view: currentDatasource.is_sqllab_view,
-        template_params: currentDatasource.template_params,
-        extra: currentDatasource.extra,
-        is_managed_externally: currentDatasource.is_managed_externally,
-        external_url: currentDatasource.external_url,
-        metrics: currentDatasource?.metrics?.map(
-          (metric: Record<string, unknown>) => {
-            const metricBody: any = {
-              expression: metric.expression,
-              description: metric.description,
-              metric_name: metric.metric_name,
-              metric_type: metric.metric_type,
-              d3format: metric.d3format || null,
-              currency: !isDefined(metric.currency)
-                ? null
-                : JSON.stringify(metric.currency),
-              verbose_name: metric.verbose_name,
-              warning_text: metric.warning_text,
-              uuid: metric.uuid,
-              extra: buildExtraJsonObject(metric),
-            };
-            if (!Number.isNaN(Number(metric.id))) {
-              metricBody.id = metric.id;
-            }
-            return metricBody;
-          },
-        ),
-        columns: currentDatasource?.columns?.map(
-          (column: Record<string, unknown>) => ({
-            id: typeof column.id === 'number' ? column.id : undefined,
-            column_name: column.column_name,
-            type: column.type,
-            advanced_data_type: column.advanced_data_type,
-            verbose_name: column.verbose_name,
-            description: column.description,
-            expression: column.expression,
-            filterable: column.filterable,
-            groupby: column.groupby,
-            is_active: column.is_active,
-            is_dttm: column.is_dttm,
-            python_date_format: column.python_date_format || null,
-            uuid: column.uuid,
-            extra: buildExtraJsonObject(column),
-          }),
-        ),
-        owners: currentDatasource.owners.map(
-          (o: Record<string, number>) => o.value || o.id,
-        ),
-      },
-    })
-      .then(() => {
-        addSuccessToast(t('The dataset has been saved'));
-        return SupersetClient.get({
-          endpoint: `/api/v1/dataset/${currentDatasource?.id}`,
-        });
-      })
-      .then(({ json }) => {
-        // eslint-disable-next-line no-param-reassign
-        json.result.type = 'table';
-        onDatasourceSave({
-          ...json.result,
-          owners: currentDatasource.owners,
-        });
-        onHide();
-      })
-      .catch(response => {
-        setIsSaving(false);
-        getClientErrorObject(response).then(error => {
-          let errorResponse: SupersetError | undefined;
-          let errorText: string | undefined;
-          // sip-40 error response
-          if (error?.errors?.length) {
-            errorResponse = error.errors[0];
-          } else if (typeof error.error === 'string') {
-            // backward compatible with old error messages
-            errorText = error.error;
-          }
-          modal.error({
-            title: t('Error saving dataset'),
-            okButtonProps: { danger: true, className: 'btn-danger' },
-            content: (
-              <ErrorMessageWithStackTrace
-                error={errorResponse}
-                source="crud"
-                fallback={errorText}
-              />
-            ),
-          });
-        });
       });
+      if (datasource.sql !== currentDatasource.sql) {
+        // if sql has changed, save a second time with synced columns
+        dispatch(startMetaDataLoading());
+        try {
+          const columnJson = await fetchSyncedColumns(currentDatasource);
+          const columnChanges = updateColumns(
+            currentDatasource.columns,
+            columnJson,
+            addSuccessToast,
+          );
+          currentDatasource.columns = columnChanges.finalColumns;
+          dispatch(syncDatasourceMetadata(currentDatasource));
+          dispatch(stopMetaDataLoading());
+          addSuccessToast(t('Metadata has been synced'));
+        } catch (error) {
+          dispatch(stopMetaDataLoading());
+          addDangerToast(
+            t('An error has occurred while syncing virtual dataset columns'),
+          );
+        }
+        await SupersetClient.put({
+          endpoint: `/api/v1/dataset/${currentDatasource.id}`,
+          jsonPayload: buildPayload(currentDatasource),
+        });
+      }
+      const { json } = await SupersetClient.get({
+        endpoint: `/api/v1/dataset/${currentDatasource?.id}`,
+      });
+      addSuccessToast(t('The dataset has been saved'));
+      // eslint-disable-next-line no-param-reassign
+      json.result.type = 'table';
+      onDatasourceSave({
+        ...json.result,
+        owners: currentDatasource.owners,
+      });
+      onHide();
+    } catch (response) {
+      setIsSaving(false);
+      const error = await getClientErrorObject(response);
+      let errorResponse: SupersetError | undefined;
+      let errorText: string | undefined;
+      // sip-40 error response
+      if (error?.errors?.length) {
+        errorResponse = error.errors[0];
+      } else if (typeof error.error === 'string') {
+        // backward compatible with old error messages
+        errorText = error.error;
+      }
+      modal.error({
+        title: t('Error saving dataset'),
+        okButtonProps: { danger: true, className: 'btn-danger' },
+        content: (
+          <ErrorMessageWithStackTrace
+            error={errorResponse}
+            source="crud"
+            fallback={errorText}
+          />
+        ),
+      });
+    }
   };
 
-  const onDatasourceChange = (data: Record<string, any>, err: Array<any>) => {
+  const onDatasourceChange = (data: DatasetObject, err: Array<any>) => {
     setCurrentDatasource({
       ...data,
-      metrics: data?.metrics.map((metric: Record<string, unknown>) => ({
+      metrics: data?.metrics.map((metric: DatasetObject['metrics'][0]) => ({
         ...metric,
         is_certified: metric?.certified_by || metric?.certification_details,
       })),

superset-frontend/src/components/Datasource/utils.js

@@ -17,6 +17,9 @@
  * under the License.
  */
 import { Children, cloneElement } from 'react';
+import { nanoid } from 'nanoid';
+import { SupersetClient, tn } from '@superset-ui/core';
+import rison from 'rison';
 
 export function recurseReactClone(children, type, propExtender) {
   /**
@@ -40,3 +43,102 @@ export function recurseReactClone(children, type, propExtender) {
     return newChild;
   });
 }
+
+export function updateColumns(prevCols, newCols, addSuccessToast) {
+  // cols: Array<{column_name: string; is_dttm: boolean; type: string;}>
+  const databaseColumnNames = newCols.map(col => col.column_name);
+  const currentCols = prevCols.reduce((agg, col) => {
+    // eslint-disable-next-line no-param-reassign
+    agg[col.column_name] = col;
+    return agg;
+  }, {});
+  const columnChanges = {
+    added: [],
+    modified: [],
+    removed: prevCols
+      .map(col => col.column_name)
+      .filter(col => !databaseColumnNames.includes(col)),
+    finalColumns: [],
+  };
+  newCols.forEach(col => {
+    const currentCol = currentCols[col.column_name];
+    if (!currentCol) {
+      // new column
+      columnChanges.finalColumns.push({
+        id: nanoid(),
+        column_name: col.column_name,
+        type: col.type,
+        groupby: true,
+        filterable: true,
+        is_dttm: col.is_dttm,
+      });
+      columnChanges.added.push(col.column_name);
+    } else if (
+      currentCol.type !== col.type ||
+      currentCol.is_dttm !== col.is_dttm
+    ) {
+      // modified column
+      columnChanges.finalColumns.push({
+        ...currentCol,
+        type: col.type,
+        is_dttm: currentCol.is_dttm || col.is_dttm,
+      });
+      columnChanges.modified.push(col.column_name);
+    } else {
+      // unchanged
+      columnChanges.finalColumns.push(currentCol);
+    }
+  });
+  if (columnChanges.modified.length) {
+    addSuccessToast(
+      tn(
+        'Modified 1 column in the virtual dataset',
+        'Modified %s columns in the virtual dataset',
+        columnChanges.modified.length,
+      ),
+    );
+  }
+  if (columnChanges.removed.length) {
+    addSuccessToast(
+      tn(
+        'Removed 1 column from the virtual dataset',
+        'Removed %s columns from the virtual dataset',
+        columnChanges.removed.length,
+      ),
+    );
+  }
+  if (columnChanges.added.length) {
+    addSuccessToast(
+      tn(
+        'Added 1 new column to the virtual dataset',
+        'Added %s new columns to the virtual dataset',
+        columnChanges.added.length,
+      ),
+    );
+  }
+  return columnChanges;
+}
+
+export async function fetchSyncedColumns(datasource) {
+  const params = {
+    datasource_type: datasource.type,
+    database_name:
+      datasource.database?.database_name || datasource.database?.name,
+    catalog_name: datasource.catalog,
+    schema_name: datasource.schema,
+    table_name: datasource.table_name,
+    normalize_columns: datasource.normalize_columns,
+    always_filter_main_dttm: datasource.always_filter_main_dttm,
+  };
+  Object.entries(params).forEach(([key, value]) => {
+    // rison can't encode the undefined value
+    if (value === undefined) {
+      params[key] = null;
+    }
+  });
+  const endpoint = `/datasource/external_metadata_by_name/?q=${rison.encode_uri(
+    params,
+  )}`;
+  const { json } = await SupersetClient.get({ endpoint });
+  return json;
+}

superset-frontend/src/components/Tooltip/index.tsx

@@ -18,13 +18,9 @@
  */
 import { supersetTheme } from '@superset-ui/core';
 import { Tooltip as AntdTooltip } from 'antd-v5';
-import {
+import { TooltipProps, TooltipPlacement } from 'antd-v5/lib/tooltip';
-  TooltipProps as AntdTooltipProps,
-  TooltipPlacement as AntdTooltipPlacement,
-} from 'antd-v5/lib/tooltip';
 
-export type TooltipPlacement = AntdTooltipPlacement;
+export { TooltipProps, TooltipPlacement };
-export type TooltipProps = AntdTooltipProps;
 
 export const Tooltip = ({ overlayStyle, ...props }: TooltipProps) => (
   <>

superset-frontend/src/explore/actions/exploreActions.ts

@@ -164,6 +164,21 @@ export function setStashFormData(
   };
 }
 
+export const START_METADATA_LOADING = 'START_METADATA_LOADING';
+export function startMetaDataLoading() {
+  return { type: START_METADATA_LOADING };
+}
+
+export const STOP_METADATA_LOADING = 'STOP_METADATA_LOADING';
+export function stopMetaDataLoading() {
+  return { type: STOP_METADATA_LOADING };
+}
+
+export const SYNC_DATASOURCE_METADATA = 'SYNC_DATASOURCE_METADATA';
+export function syncDatasourceMetadata(datasource: Dataset) {
+  return { type: SYNC_DATASOURCE_METADATA, datasource };
+}
+
 export const exploreActions = {
   ...toastActions,
   fetchDatasourcesStarted,
@@ -178,6 +193,7 @@ export const exploreActions = {
   createNewSlice,
   sliceUpdated,
   setForceQuery,
+  syncDatasourceMetadata,
 };
 
 export type ExploreActions = typeof exploreActions;

superset-frontend/src/explore/components/controls/TextAreaControl.jsx

@@ -19,6 +19,10 @@
 import { Component } from 'react';
 import PropTypes from 'prop-types';
 import { TextArea } from 'src/components/Input';
+import {
+  Tooltip,
+  TooltipProps as TooltipOptions,
+} from 'src/components/Tooltip';
 import { t, withTheme } from '@superset-ui/core';
 
 import Button from 'src/components/Button';
@@ -55,6 +59,7 @@ const propTypes = {
     'vertical',
   ]),
   textAreaStyles: PropTypes.object,
+  tooltipOptions: PropTypes.oneOf([null, TooltipOptions]),
 };
 
 const defaultProps = {
@@ -67,6 +72,7 @@ const defaultProps = {
   readOnly: false,
   resize: null,
   textAreaStyles: {},
+  tooltipOptions: {},
 };
 
 class TextAreaControl extends Component {
@@ -94,31 +100,44 @@ class TextAreaControl extends Component {
       if (this.props.readOnly) {
         style.backgroundColor = '#f2f2f2';
       }
-
+      const codeEditor = (
-      return (
+        <div>
-        <TextAreaEditor
+          <TextAreaEditor
-          mode={this.props.language}
+            mode={this.props.language}
-          style={style}
+            style={style}
-          minLines={minLines}
+            minLines={minLines}
-          maxLines={inModal ? 1000 : this.props.maxLines}
+            maxLines={inModal ? 1000 : this.props.maxLines}
-          editorProps={{ $blockScrolling: true }}
+            editorProps={{ $blockScrolling: true }}
-          defaultValue={this.props.initialValue}
+            defaultValue={this.props.initialValue}
-          readOnly={this.props.readOnly}
+            readOnly={this.props.readOnly}
-          key={this.props.name}
+            key={this.props.name}
-          {...this.props}
+            {...this.props}
-          onChange={this.onAreaEditorChange.bind(this)}
+            onChange={this.onAreaEditorChange.bind(this)}
-        />
+          />
+        </div>
       );
+
+      if (this.props.tooltipOptions) {
+        return <Tooltip {...this.props.tooltipOptions}>{codeEditor}</Tooltip>;
+      }
+      return codeEditor;
     }
-    return (
+
-      <TextArea
+    const textArea = (
-        placeholder={t('textarea')}
+      <div>
-        onChange={this.onControlChange.bind(this)}
+        <TextArea
-        defaultValue={this.props.initialValue}
+          placeholder={t('textarea')}
-        disabled={this.props.readOnly}
+          onChange={this.onControlChange.bind(this)}
-        style={{ height: this.props.height }}
+          defaultValue={this.props.initialValue}
-      />
+          disabled={this.props.readOnly}
+          style={{ height: this.props.height }}
+        />
+      </div>
     );
+    if (this.props.tooltipOptions) {
+      return <Tooltip {...this.props.tooltipOptions}>{textArea}</Tooltip>;
+    }
+    return textArea;
   }
 
   renderModalBody() {

superset-frontend/src/explore/reducers/exploreReducer.js

@@ -50,6 +50,24 @@ export default function exploreReducer(state = {}, action) {
         isDatasourceMetaLoading: true,
       };
     },
+    [actions.START_METADATA_LOADING]() {
+      return {
+        ...state,
+        isDatasourceMetaLoading: true,
+      };
+    },
+    [actions.STOP_METADATA_LOADING]() {
+      return {
+        ...state,
+        isDatasourceMetaLoading: false,
+      };
+    },
+    [actions.SYNC_DATASOURCE_METADATA]() {
+      return {
+        ...state,
+        datasource: action.datasource,
+      };
+    },
     [actions.UPDATE_FORM_DATA_BY_DATASOURCE]() {
       const newFormData = { ...state.form_data };
       const { prevDatasource, newDatasource } = action;

superset-frontend/src/features/databases/DatabaseModal/DatabaseConnectionForm/EncryptedField.tsx

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import { useState } from 'react';
+import { useRef, useState } from 'react';
 import { SupersetTheme, t } from '@superset-ui/core';
 import { Button, AntdSelect } from 'src/components';
 import InfoTooltip from 'src/components/InfoTooltip';
@@ -46,6 +46,7 @@ export const EncryptedField = ({
   db,
   editNewDb,
 }: FieldPropTypes) => {
+  const selectedFileInputRef = useRef<HTMLInputElement | null>(null);
   const [uploadOption, setUploadOption] = useState<number>(
     CredentialInfoOptions.JsonUpload.valueOf(),
   );
@@ -152,9 +153,7 @@ export const EncryptedField = ({
             {!fileToUpload && (
               <Button
                 className="input-upload-btn"
-                onClick={() =>
+                onClick={() => selectedFileInputRef.current?.click()}
-                  document?.getElementById('selectedFile')?.click()
-                }
               >
                 {t('Choose File')}
               </Button>
@@ -178,6 +177,7 @@ export const EncryptedField = ({
             )}
 
             <input
+              ref={selectedFileInputRef}
               id="selectedFile"
               accept=".json"
               className="input-upload"
@@ -196,9 +196,9 @@ export const EncryptedField = ({
                     checked: false,
                   },
                 });
-                (
+                if (selectedFileInputRef.current) {
-                  document.getElementById('selectedFile') as HTMLInputElement
+                  selectedFileInputRef.current.value = null as any;
-                ).value = null as any;
+                }
               }}
             />
           </div>

superset-frontend/src/features/databases/DatabaseModal/index.tsx

@@ -139,7 +139,7 @@ const SSHTunnelContainer = styled.div`
   `};
 `;
 
-interface DatabaseModalProps {
+export interface DatabaseModalProps {
   addDangerToast: (msg: string) => void;
   addSuccessToast: (msg: string) => void;
   onDatabaseAdd?: (database?: DatabaseObject) => void;
@@ -1335,7 +1335,7 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
   useEffect(() => {
     if (importingModal) {
       document
-        .getElementsByClassName('ant-upload-list-item-name')[0]
+        ?.getElementsByClassName('ant-upload-list-item-name')[0]
         .scrollIntoView();
     }
   }, [importingModal]);

superset-frontend/src/features/databases/UploadDataModel/UploadDataModal.test.tsx

@@ -622,7 +622,7 @@ test('CSV form post', async () => {
 
   // Select a file from the file dialog
   const file = new File(['test'], 'test.csv', { type: 'text' });
-  const inputElement = document.querySelector('input[type="file"]');
+  const inputElement = screen.getByTestId('model-file-input');
 
   if (inputElement) {
     userEvent.upload(inputElement as HTMLElement, file);
@@ -680,7 +680,7 @@ test('Excel form post', async () => {
 
   // Select a file from the file dialog
   const file = new File(['test'], 'test.xls', { type: 'text' });
-  const inputElement = document.querySelector('input[type="file"]');
+  const inputElement = screen.getByTestId('model-file-input');
 
   if (inputElement) {
     userEvent.upload(inputElement as HTMLElement, file);
@@ -738,7 +738,7 @@ test('Columnar form post', async () => {
 
   // Select a file from the file dialog
   const file = new File(['test'], 'test.parquet', { type: 'text' });
-  const inputElement = document.querySelector('input[type="file"]');
+  const inputElement = screen.getByTestId('model-file-input');
 
   if (inputElement) {
     userEvent.upload(inputElement as HTMLElement, file);

superset-frontend/src/features/datasets/types.ts

@@ -1,4 +1,5 @@
-import { Currency } from '@superset-ui/core';
+import { Currency, type DatasourceType } from '@superset-ui/core';
+import { Owner } from '@superset-ui/chart-controls';
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -32,37 +33,52 @@ export type ColumnObject = {
   python_date_format?: string;
   uuid?: string;
   extra?: string;
+  certified_by?: string;
+  certification_details?: string;
+  warning_markdown?: string;
+  advanced_data_type?: string;
 };
 
 type MetricObject = {
   id: number;
+  uuid: number;
   expression?: string;
   description?: string;
   metric_name: string;
+  verbose_name?: string;
   metric_type: string;
   d3format?: string;
   currency?: Currency;
   warning_text?: string;
+  certified_by?: string;
+  certification_details?: string;
+  warning_markdown?: string;
 };
 
 export type DatasetObject = {
+  id: number;
   table_name?: string;
   sql?: string;
   filter_select_enabled?: boolean;
   fetch_values_predicate?: string;
   schema?: string;
-  description?: string;
+  description: string | null;
-  main_dttm_col?: string;
+  main_dttm_col: string;
   offset?: number;
   default_endpoint?: string;
   cache_timeout?: number;
   is_sqllab_view?: boolean;
   template_params?: string;
-  owners: number[];
+  owners: Owner[];
   columns: ColumnObject[];
   metrics: MetricObject[];
   extra?: string;
   is_managed_externally: boolean;
   normalize_columns: boolean;
   always_filter_main_dttm: boolean;
+  type: DatasourceType;
+  column_formats: Record<string, string>;
+  currency_formats: Record<string, Currency>;
+  datasource_name: string | null;
+  verbose_map: Record<string, string>;
 };