Skip to content

Commit

Permalink
19943 Grid view status filters (#23392)
Browse files Browse the repository at this point in the history
* Move tree filtering inside react and add some filters

* Move filters from context to utils

* Fix tests for useTreeData

* Fix last tests.

* Add tests for useFilters

* Refact to use existing SimpleStatus component

* Additional fix after rebase.

* Update following bbovenzi code review

* Update following code review

* Fix tests.

* Fix page flickering issues from react-query

* Fix side panel and small changes.

* Use default_dag_run_display_number in the filter options

* Handle timezone

* Fix flaky test

Co-authored-by: Brent Bovenzi <brent.bovenzi@gmail.com>
  • Loading branch information
pierrejeambrun and bbovenzi authored May 9, 2022
1 parent 7354d2e commit 46c1c00
Show file tree
Hide file tree
Showing 22 changed files with 471 additions and 82 deletions.
2 changes: 1 addition & 1 deletion airflow/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@
"up_for_retry": "gold",
"up_for_reschedule": "turquoise",
"upstream_failed": "orange",
"skipped": "pink",
"skipped": "hotpink",
"scheduled": "tan",
"deferred": "mediumpurple",
}
Expand Down
2 changes: 1 addition & 1 deletion airflow/utils/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ class State:
TaskInstanceState.UP_FOR_RETRY: 'gold',
TaskInstanceState.UP_FOR_RESCHEDULE: 'turquoise',
TaskInstanceState.UPSTREAM_FAILED: 'orange',
TaskInstanceState.SKIPPED: 'pink',
TaskInstanceState.SKIPPED: 'hotpink',
TaskInstanceState.REMOVED: 'lightgrey',
TaskInstanceState.SCHEDULED: 'tan',
TaskInstanceState.DEFERRED: 'mediumpurple',
Expand Down
6 changes: 4 additions & 2 deletions airflow/www/jest-setup.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,18 @@

import '@testing-library/jest-dom';

// Mock a global object we use across the app
// Mock global objects we use across the app
global.stateColors = {
deferred: 'mediumpurple',
failed: 'red',
queued: 'gray',
running: 'lime',
scheduled: 'tan',
skipped: 'pink',
skipped: 'hotpink',
success: 'green',
up_for_reschedule: 'turquoise',
up_for_retry: 'gold',
upstream_failed: 'orange',
};

global.defaultDagRunDisplayNumber = 245;
1 change: 1 addition & 0 deletions airflow/www/static/js/datetime_utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

/* global moment, $, document */
export const defaultFormat = 'YYYY-MM-DD, HH:mm:ss';
export const isoFormatWithoutTZ = 'YYYY-MM-DDTHH:mm:ss.SSS';
export const defaultFormatWithTZ = 'YYYY-MM-DD, HH:mm:ss z';
export const defaultTZFormat = 'z (Z)';
export const dateTimeAttrFormat = 'YYYY-MM-DDThh:mm:ssTZD';
Expand Down
127 changes: 127 additions & 0 deletions airflow/www/static/js/grid/FilterBar.jsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

/* global filtersOptions, moment */

import {
Box,
Button,
Flex,
Input,
Select,
} from '@chakra-ui/react';
import React from 'react';
import { useTimezone } from './context/timezone';
import { isoFormatWithoutTZ } from '../datetime_utils';

import useFilters from './utils/useFilters';

const FilterBar = () => {
const {
filters,
onBaseDateChange,
onNumRunsChange,
onRunTypeChange,
onRunStateChange,
onTaskStateChange,
clearFilters,
} = useFilters();

const { timezone } = useTimezone();
const time = moment(filters.baseDate);
const formattedTime = time.tz(timezone).format(isoFormatWithoutTZ);

const inputStyles = { backgroundColor: 'white', size: 'lg' };

return (
<Flex backgroundColor="#f0f0f0" mt={0} mb={2} p={4}>
<Box px={2}>
<Input
{...inputStyles}
type="datetime-local"
value={formattedTime || ''}
onChange={onBaseDateChange}
/>
</Box>
<Box px={2}>
<Select
{...inputStyles}
placeholder="Runs"
value={filters.numRuns || ''}
onChange={onNumRunsChange}
>
{filtersOptions.numRuns.map((value) => (
<option value={value} key={value}>{value}</option>
))}
</Select>
</Box>
<Box px={2}>
<Select
{...inputStyles}
value={filters.runType || ''}
onChange={onRunTypeChange}
>
<option value="" key="all">All Run Types</option>
{filtersOptions.runTypes.map((value) => (
<option value={value} key={value}>{value}</option>
))}
</Select>
</Box>
<Box />
<Box px={2}>
<Select
{...inputStyles}
value={filters.runState || ''}
onChange={onRunStateChange}
>
<option value="" key="all">All Run States</option>
{filtersOptions.dagStates.map((value) => (
<option value={value} key={value}>{value}</option>
))}
</Select>
</Box>
<Box px={2}>
<Select
{...inputStyles}
value={filters.taskState || ''}
onChange={onTaskStateChange}
>
<option value="" key="all">All Task States</option>
{filtersOptions.taskStates.map((value) => (
<option value={value} key={value}>{value}</option>
))}
</Select>
</Box>
<Box px={2}>
<Button
colorScheme="cyan"
aria-label="Reset filters"
background="white"
variant="outline"
onClick={clearFilters}
size="lg"
>
Clear Filters
</Button>
</Box>
</Flex>
);
};

export default FilterBar;
14 changes: 11 additions & 3 deletions airflow/www/static/js/grid/Grid.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import {
Flex,
useDisclosure,
Button,
Divider,
} from '@chakra-ui/react';

import { useGridData } from './api';
Expand All @@ -42,17 +43,21 @@ import Details from './details';
import useSelection from './utils/useSelection';
import { useAutoRefresh } from './context/autorefresh';
import ToggleGroups from './ToggleGroups';
import FilterBar from './FilterBar';
import LegendRow from './LegendRow';

const sidePanelKey = 'hideSidePanel';

const Grid = () => {
const scrollRef = useRef();
const tableRef = useRef();

const { data: { groups, dagRuns } } = useGridData();
const dagRunIds = dagRuns.map((dr) => dr.runId);

const { isRefreshOn, toggleRefresh, isPaused } = useAutoRefresh();
const isPanelOpen = localStorage.getItem(sidePanelKey) !== 'true';
const { isOpen, onToggle } = useDisclosure({ defaultIsOpen: isPanelOpen });
const dagRunIds = dagRuns.map((dr) => dr.runId);

const { clearSelection } = useSelection();
const toggleSidePanel = () => {
Expand Down Expand Up @@ -86,7 +91,10 @@ const Grid = () => {
}, [tableRef, scrollOnResize]);

return (
<Box>
<Box mt={3}>
<FilterBar />
<LegendRow />
<Divider mb={5} borderBottomWidth={2} />
<Flex flexGrow={1} justifyContent="space-between" alignItems="center">
<Flex alignItems="center">
<FormControl display="flex" width="auto" mr={2}>
Expand Down Expand Up @@ -129,7 +137,7 @@ const Grid = () => {
<Thead display="block" pr="10px" position="sticky" top={0} zIndex={2} bg="white">
<DagRuns />
</Thead>
{/* TODO: remove hardcoded values. 665px is roughly the total heade+footer height */}
{/* TODO: remove hardcoded values. 665px is roughly the total header+footer height */}
<Tbody display="block" width="100%" maxHeight="calc(100vh - 665px)" minHeight="500px" ref={tableRef} pr="10px">
{renderTaskRows({
task: groups, dagRunIds,
Expand Down
42 changes: 42 additions & 0 deletions airflow/www/static/js/grid/LegendRow.jsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

/* global stateColors */

import {
Flex,
Text,
} from '@chakra-ui/react';
import React from 'react';
import { SimpleStatus } from './StatusBox';

const LegendRow = () => (
<Flex mt={0} mb={2} p={4} flexWrap="wrap">
{
Object.entries(stateColors).map(([state, stateColor]) => (
<Flex alignItems="center" mr={3} key={stateColor}>
<SimpleStatus mr={1} state={state} />
<Text fontSize="md">{state}</Text>
</Flex>
))
}
</Flex>
);

export default LegendRow;
3 changes: 3 additions & 0 deletions airflow/www/static/js/grid/StatusBox.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import {

import InstanceTooltip from './InstanceTooltip';
import { useContainerRef } from './context/containerRef';
import useFilters from './utils/useFilters';

export const boxSize = 10;
export const boxSizePx = `${boxSize}px`;
Expand All @@ -51,6 +52,7 @@ const StatusBox = ({
const { runId, taskId } = instance;
const { colors } = useTheme();
const hoverBlue = `${colors.blue[100]}50`;
const { filters } = useFilters();

// Fetch the corresponding column element and set its background color when hovering
const onMouseEnter = () => {
Expand Down Expand Up @@ -87,6 +89,7 @@ const StatusBox = ({
zIndex={1}
onMouseEnter={onMouseEnter}
onMouseLeave={onMouseLeave}
opacity={(filters.taskState && filters.taskState !== instance.state) ? 0.30 : 1}
/>
</Box>
</Tooltip>
Expand Down
41 changes: 26 additions & 15 deletions airflow/www/static/js/grid/api/useGridData.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,25 @@
* under the License.
*/

/* global gridData, autoRefreshInterval */
/* global autoRefreshInterval, gridData */

import { useQuery } from 'react-query';
import axios from 'axios';

import { getMetaValue } from '../../utils';
import { useAutoRefresh } from '../context/autorefresh';
import { formatData, areActiveRuns } from '../utils/gridData';
import { areActiveRuns, formatData } from '../utils/gridData';
import useErrorToast from '../utils/useErrorToast';
import useFilters, {
BASE_DATE_PARAM, NUM_RUNS_PARAM, RUN_STATE_PARAM, RUN_TYPE_PARAM, now,
} from '../utils/useFilters';

const DAG_ID_PARAM = 'dag_id';

// dagId comes from dag.html
const dagId = getMetaValue('dag_id');
const dagId = getMetaValue(DAG_ID_PARAM);
const gridDataUrl = getMetaValue('grid_data_url') || '';
const numRuns = getMetaValue('num_runs');
const urlRoot = getMetaValue('root');
const baseDate = getMetaValue('base_date');

const emptyData = {
dagRuns: [],
Expand All @@ -43,15 +46,22 @@ const useGridData = () => {
const initialData = formatData(gridData, emptyData);
const { isRefreshOn, stopRefresh } = useAutoRefresh();
const errorToast = useErrorToast();
return useQuery('gridData', async () => {
try {
const params = new URLSearchParams({
dag_id: dagId,
});
if (numRuns && numRuns !== 25) params.append('num_runs', numRuns);
if (urlRoot) params.append('root', urlRoot);
if (baseDate) params.append('base_date', baseDate);
const {
filters: {
baseDate, numRuns, runType, runState,
},
} = useFilters();

return useQuery(['gridData', baseDate, numRuns, runType, runState], async () => {
try {
const params = {
root: urlRoot || undefined,
[DAG_ID_PARAM]: dagId,
[BASE_DATE_PARAM]: baseDate === now ? undefined : baseDate,
[NUM_RUNS_PARAM]: numRuns,
[RUN_TYPE_PARAM]: runType,
[RUN_STATE_PARAM]: runState,
};
const newData = await axios.get(gridDataUrl, { params });
// turn off auto refresh if there are no active runs
if (!areActiveRuns(newData.dagRuns)) stopRefresh();
Expand All @@ -65,10 +75,11 @@ const useGridData = () => {
throw (error);
}
}, {
// only refetch if the refresh switch is on
refetchInterval: isRefreshOn && autoRefreshInterval * 1000,
initialData,
placeholderData: emptyData,
// only refetch if the refresh switch is on
refetchInterval: isRefreshOn && autoRefreshInterval * 1000,
keepPreviousData: true,
});
};

Expand Down
4 changes: 2 additions & 2 deletions airflow/www/static/js/grid/api/useGridData.test.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@
* under the License.
*/

/* global describe, test, expect, beforeAll */

import { renderHook } from '@testing-library/react-hooks';
import useGridData from './useGridData';
import { Wrapper } from '../utils/testUtils';

/* global describe, test, expect, beforeAll */

const pendingGridData = {
groups: {},
dag_runs: [
Expand Down
Loading

0 comments on commit 46c1c00

Please sign in to comment.