diff --git a/packages/material-ui/src/ButtonBase/ButtonBase.js b/packages/material-ui/src/ButtonBase/ButtonBase.js
index 4977ab46a75d77..b1f29cab226a53 100644
--- a/packages/material-ui/src/ButtonBase/ButtonBase.js
+++ b/packages/material-ui/src/ButtonBase/ButtonBase.js
@@ -3,12 +3,11 @@ import PropTypes from 'prop-types';
 import ReactDOM from 'react-dom';
 import clsx from 'clsx';
 import { elementTypeAcceptingRef } from '@material-ui/utils';
-import ownerWindow from '../utils/ownerWindow';
 import withForwardedRef from '../utils/withForwardedRef';
 import { setRef } from '../utils/reactHelpers';
 import withStyles from '../styles/withStyles';
 import NoSsr from '../NoSsr';
-import { listenForFocusKeys, detectFocusVisible } from './focusVisible';
+import { handleBlurVisible, isFocusVisible, prepare as prepareFocusVisible } from './focusVisible';
 import TouchRipple from './TouchRipple';
 import createRippleHandler from './createRippleHandler';
 
@@ -60,18 +59,7 @@ class ButtonBase extends React.Component {
 
   buttonRef = React.createRef();
 
-  keyDown = false; // Used to help track keyboard activation keyDown
-
-  focusVisibleCheckTime = 50;
-
-  focusVisibleMaxCheckTimes = 5;
-
-  handleMouseDown = createRippleHandler(this, 'MouseDown', 'start', () => {
-    clearTimeout(this.focusVisibleTimeout);
-    if (this.state.focusVisible) {
-      this.setState({ focusVisible: false });
-    }
-  });
+  handleMouseDown = createRippleHandler(this, 'MouseDown', 'start');
 
   handleMouseUp = createRippleHandler(this, 'MouseUp', 'stop');
 
@@ -89,16 +77,15 @@ class ButtonBase extends React.Component {
 
   handleContextMenu = createRippleHandler(this, 'ContextMenu', 'stop');
 
-  handleBlur = createRippleHandler(this, 'Blur', 'stop', () => {
-    clearTimeout(this.focusVisibleTimeout);
+  handleBlur = createRippleHandler(this, 'Blur', 'stop', event => {
     if (this.state.focusVisible) {
+      handleBlurVisible(event);
       this.setState({ focusVisible: false });
     }
   });
 
   componentDidMount() {
-    listenForFocusKeys(ownerWindow(this.getButtonNode()));
-
+    prepareFocusVisible(this.getButtonNode().ownerDocument);
     if (this.props.action) {
       this.props.action({
         focusVisible: () => {
@@ -120,10 +107,6 @@ class ButtonBase extends React.Component {
     }
   }
 
-  componentWillUnmount() {
-    clearTimeout(this.focusVisibleTimeout);
-  }
-
   getButtonNode() {
     return ReactDOM.findDOMNode(this.buttonRef.current);
   }
@@ -132,15 +115,6 @@ class ButtonBase extends React.Component {
     this.ripple = node;
   };
 
-  onFocusVisibleHandler = event => {
-    this.keyDown = false;
-    this.setState({ focusVisible: true });
-
-    if (this.props.onFocusVisible) {
-      this.props.onFocusVisible(event);
-    }
-  };
-
   static getDerivedStateFromProps(nextProps, prevState) {
     if (typeof prevState.focusVisible === 'undefined') {
       return {
@@ -224,10 +198,13 @@ class ButtonBase extends React.Component {
       this.buttonRef.current = event.currentTarget;
     }
 
-    event.persist();
-    detectFocusVisible(this, this.getButtonNode(), () => {
-      this.onFocusVisibleHandler(event);
-    });
+    if (isFocusVisible(event)) {
+      this.setState({ focusVisible: true });
+
+      if (this.props.onFocusVisible) {
+        this.props.onFocusVisible(event);
+      }
+    }
 
     if (this.props.onFocus) {
       this.props.onFocus(event);
diff --git a/packages/material-ui/src/ButtonBase/ButtonBase.test.js b/packages/material-ui/src/ButtonBase/ButtonBase.test.js
index df764f893f45e5..bb9b50e9847685 100644
--- a/packages/material-ui/src/ButtonBase/ButtonBase.test.js
+++ b/packages/material-ui/src/ButtonBase/ButtonBase.test.js
@@ -15,6 +15,17 @@ import ButtonBase from './ButtonBase';
 
 const ButtonBaseNaked = unwrap(ButtonBase);
 
+function focusVisible(element) {
+  element.ownerDocument.dispatchEvent(new window.Event('keydown'));
+  element.focus();
+}
+
+function simulatePointerDevice() {
+  // first focus on a page triggers focus visible until a pointer event
+  // has been dispatched
+  document.dispatchEvent(new window.Event('pointerdown'));
+}
+
 describe('<ButtonBase />', () => {
   let mount;
   let shallow;
@@ -270,55 +281,36 @@ describe('<ButtonBase />', () => {
       return;
     }
 
-    let wrapper;
-    let instance;
-    let button;
-    let clock;
     let rootElement;
 
     beforeEach(() => {
-      clock = useFakeTimers();
       rootElement = document.createElement('div');
       rootElement.tabIndex = 0;
       document.body.appendChild(rootElement);
       rootElement.attachShadow({ mode: 'open' });
-      wrapper = mount(<ButtonBase id="test-button">Hello</ButtonBase>, {
-        attachTo: rootElement.shadowRoot,
-      });
-      instance = wrapper.find('ButtonBase').instance();
-      button = rootElement.shadowRoot.getElementById('test-button');
-      if (!button) {
-        throw new Error('missing button');
-      }
-
-      button.focus();
-
-      if (document.activeElement !== rootElement) {
-        // Mock activeElement value and simulate host-retargeting in shadow root for
-        // jsdom@12.0.0 (https://github.com/jsdom/jsdom/issues/2343)
-        rootElement.focus();
-        rootElement.shadowRoot.activeElement = button;
-        wrapper.simulate('focus');
-      }
-
-      const event = new window.Event('keyup');
-      event.keyCode = 9; // Tab
-      window.dispatchEvent(event);
     });
 
     afterEach(() => {
-      clock.restore();
       ReactDOM.unmountComponentAtNode(rootElement.shadowRoot);
       document.body.removeChild(rootElement);
     });
 
     it('should set focus state for shadowRoot children', () => {
-      assert.strictEqual(wrapper.find(`.${classes.focusVisible}`).exists(), false);
+      mount(<ButtonBase id="test-button">Hello</ButtonBase>, {
+        attachTo: rootElement.shadowRoot,
+      });
+      simulatePointerDevice();
 
-      clock.tick(instance.focusVisibleCheckTime * instance.focusVisibleMaxCheckTimes);
-      wrapper.update();
+      const button = rootElement.shadowRoot.getElementById('test-button');
+      if (!button) {
+        throw new Error('missing button');
+      }
+
+      assert.strictEqual(button.classList.contains(classes.focusVisible), false);
+
+      focusVisible(button);
 
-      assert.strictEqual(wrapper.find(`.${classes.focusVisible}`).exists(), true);
+      assert.strictEqual(button.classList.contains(classes.focusVisible), true);
     });
   });
 
@@ -338,17 +330,20 @@ describe('<ButtonBase />', () => {
 
     beforeEach(() => {
       clock = useFakeTimers();
-      wrapper = mount(<ButtonBase>Hello</ButtonBase>);
+      wrapper = mount(
+        <ButtonBase
+          ref={element => {
+            button = element;
+          }}
+        >
+          Hello
+        </ButtonBase>,
+      );
+      simulatePointerDevice();
       instance = wrapper.find('ButtonBase').instance();
-      button = wrapper.find('button').getDOMNode();
       if (!button) {
         throw new Error('missing button');
       }
-      button.focus();
-
-      const event = new window.Event('keyup');
-      event.keyCode = 9; // Tab
-      window.dispatchEvent(event);
     });
 
     afterEach(() => {
@@ -357,25 +352,9 @@ describe('<ButtonBase />', () => {
 
     it('should detect the keyboard', () => {
       assert.strictEqual(getState().focusVisible, false);
-      clock.tick(instance.focusVisibleCheckTime * instance.focusVisibleMaxCheckTimes);
+      focusVisible(button);
       assert.strictEqual(getState().focusVisible, true);
     });
-
-    it('should ignore the keyboard after 1s', () => {
-      clock.tick(instance.focusVisibleCheckTime * instance.focusVisibleMaxCheckTimes);
-      assert.strictEqual(getState().focusVisible, true);
-      button.blur();
-      assert.strictEqual(getState().focusVisible, false);
-      button.focus();
-      clock.tick(instance.focusVisibleCheckTime * instance.focusVisibleMaxCheckTimes);
-      assert.strictEqual(getState().focusVisible, true);
-      clock.tick(1e3);
-      button.blur();
-      assert.strictEqual(getState().focusVisible, false);
-      button.focus();
-      clock.tick(instance.focusVisibleCheckTime * instance.focusVisibleMaxCheckTimes);
-      assert.strictEqual(getState().focusVisible, false);
-    });
   });
 
   describe('prop: disabled', () => {
@@ -450,17 +429,19 @@ describe('<ButtonBase />', () => {
     });
 
     it('onFocusVisibleHandler() should propagate call to onFocusVisible prop', () => {
-      const eventMock = 'woofButtonBase';
       const onFocusVisibleSpy = spy();
-      const wrapper = mount(
-        <ButtonBase component="span" onFocusVisible={onFocusVisibleSpy}>
+      const buttonRef = React.createRef();
+      mount(
+        <ButtonBase component="span" onFocusVisible={onFocusVisibleSpy} ref={buttonRef}>
           Hello
         </ButtonBase>,
       );
-      const instance = wrapper.find('ButtonBase').instance();
-      instance.onFocusVisibleHandler(eventMock);
+      simulatePointerDevice();
+
+      focusVisible(buttonRef.current);
+
       assert.strictEqual(onFocusVisibleSpy.callCount, 1);
-      assert.strictEqual(onFocusVisibleSpy.calledWith(eventMock), true);
+      assert.strictEqual(onFocusVisibleSpy.firstCall.args.length, 1);
     });
 
     it('should work with a functional component', () => {
diff --git a/packages/material-ui/src/ButtonBase/focusVisible.js b/packages/material-ui/src/ButtonBase/focusVisible.js
index 12fea08ef6214e..ed52aeb793c40d 100644
--- a/packages/material-ui/src/ButtonBase/focusVisible.js
+++ b/packages/material-ui/src/ButtonBase/focusVisible.js
@@ -1,75 +1,122 @@
-import warning from 'warning';
-import ownerDocument from '../utils/ownerDocument';
+// based on https://github.com/WICG/focus-visible/blob/v4.1.5/src/focus-visible.js
 
-const internal = {
-  focusKeyPressed: false,
-  keyUpEventTimeout: -1,
+let hadKeyboardEvent = true;
+let hadFocusVisibleRecently = false;
+let hadFocusVisibleRecentlyTimeout = null;
+
+const inputTypesWhitelist = {
+  text: true,
+  search: true,
+  url: true,
+  tel: true,
+  email: true,
+  password: true,
+  number: true,
+  date: true,
+  month: true,
+  week: true,
+  time: true,
+  datetime: true,
+  'datetime-local': true,
 };
 
-function findActiveElement(doc) {
-  let activeElement = doc.activeElement;
-  while (activeElement && activeElement.shadowRoot && activeElement.shadowRoot.activeElement) {
-    activeElement = activeElement.shadowRoot.activeElement;
+/**
+ * Computes whether the given element should automatically trigger the
+ * `focus-visible` class being added, i.e. whether it should always match
+ * `:focus-visible` when focused.
+ * @param {Element} node
+ * @return {boolean}
+ */
+function focusTriggersKeyboardModality(node) {
+  const { type, tagName } = node;
+
+  if (tagName === 'INPUT' && inputTypesWhitelist[type] && !node.readOnly) {
+    return true;
+  }
+
+  if (tagName === 'TEXTAREA' && !node.readOnly) {
+    return true;
   }
-  return activeElement;
+
+  if (node.isContentEditable) {
+    return true;
+  }
+
+  return false;
 }
 
-export function detectFocusVisible(instance, element, callback, attempt = 1) {
-  warning(instance.focusVisibleCheckTime, 'Material-UI: missing instance.focusVisibleCheckTime.');
-  warning(
-    instance.focusVisibleMaxCheckTimes,
-    'Material-UI: missing instance.focusVisibleMaxCheckTimes.',
-  );
+function handleKeyDown() {
+  hadKeyboardEvent = true;
+}
 
-  instance.focusVisibleTimeout = setTimeout(() => {
-    const doc = ownerDocument(element);
-    const activeElement = findActiveElement(doc);
+/**
+ * If at any point a user clicks with a pointing device, ensure that we change
+ * the modality away from keyboard.
+ * This avoids the situation where a user presses a key on an already focused
+ * element, and then clicks on a different element, focusing it with a
+ * pointing device, while we still think we're in keyboard modality.
+ * @param {Event} e
+ */
+function handlePointerDown() {
+  hadKeyboardEvent = false;
+}
 
-    if (
-      internal.focusKeyPressed &&
-      (activeElement === element || element.contains(activeElement))
-    ) {
-      callback();
-    } else if (attempt < instance.focusVisibleMaxCheckTimes) {
-      detectFocusVisible(instance, element, callback, attempt + 1);
+function handleVisibilityChange() {
+  if (this.visibilityState === 'hidden') {
+    // If the tab becomes active again, the browser will handle calling focus
+    // on the element (Safari actually calls it twice).
+    // If this tab change caused a blur on an element with focus-visible,
+    // re-apply the class when the user switches back to the tab.
+    if (hadFocusVisibleRecently) {
+      hadKeyboardEvent = true;
     }
-  }, instance.focusVisibleCheckTime);
+  }
 }
 
-// The keys that might change document.activeElement.
-const FOCUS_KEYS = [
-  9, // 'Tab',
-  13, // 'Enter',
-  27, // 'Escape',
-  32, // ' ',
-  36, // 'Home',
-  35, // 'End',
-  37, // 'ArrowLeft',
-  38, // 'ArrowUp',
-  39, // 'ArrowRight',
-  40, // 'ArrowDown',
-];
-
-function isFocusKey(event) {
-  // Use event.keyCode to support IE 11
-  return FOCUS_KEYS.indexOf(event.keyCode) > -1;
+export function prepare(ownerDocument) {
+  ownerDocument.addEventListener('keydown', handleKeyDown, true);
+  ownerDocument.addEventListener('mousedown', handlePointerDown, true);
+  ownerDocument.addEventListener('pointerdown', handlePointerDown, true);
+  ownerDocument.addEventListener('touchstart', handlePointerDown, true);
+  ownerDocument.addEventListener('visibilitychange', handleVisibilityChange, true);
 }
 
-const handleKeyUpEvent = event => {
-  if (isFocusKey(event)) {
-    internal.focusKeyPressed = true;
+export function teardown(ownerDocument) {
+  ownerDocument.removeEventListener('keydown', handleKeyDown, true);
+  ownerDocument.removeEventListener('mousedown', handlePointerDown, true);
+  ownerDocument.removeEventListener('pointerdown', handlePointerDown, true);
+  ownerDocument.removeEventListener('touchstart', handlePointerDown, true);
+  ownerDocument.removeEventListener('visibilitychange', handleVisibilityChange, true);
+}
 
-    // Let's consider that the user is using a keyboard during a window frame of 500ms.
-    clearTimeout(internal.keyUpEventTimeout);
-    internal.keyUpEventTimeout = setTimeout(() => {
-      internal.focusKeyPressed = false;
-    }, 500);
+export function isFocusVisible(event) {
+  const { target } = event;
+  try {
+    return target.matches(':focus-visible');
+  } catch (error) {
+    // browsers not implementing :focus-visible will throw a SyntaxError
+    // we use our own heuristic for those browsers
+    // rethrow might be better if it's not the expected error but do we really
+    // want to crash if focus-visible malfunctioned?
   }
-};
 
-export function listenForFocusKeys(win) {
-  // The event listener will only be added once per window.
-  // Duplicate event listeners will be ignored by addEventListener.
-  // Also, this logic is client side only, we don't need a teardown.
-  win.addEventListener('keyup', handleKeyUpEvent);
+  // no need for validFocusTarget check. the user does that by attaching it to
+  // focusable events only
+  return hadKeyboardEvent || focusTriggersKeyboardModality(target);
+}
+
+/**
+ * Should be called if a blur event is fired on a focus-visible element
+ */
+export function handleBlurVisible() {
+  // To detect a tab/window switch, we look for a blur event followed
+  // rapidly by a visibility change.
+  // If we don't see a visibility change within 100ms, it's probably a
+  // regular focus change.
+  hadFocusVisibleRecently = true;
+  window.clearTimeout(hadFocusVisibleRecentlyTimeout);
+  hadFocusVisibleRecentlyTimeout = window.setTimeout(() => {
+    hadFocusVisibleRecently = false;
+    window.clearTimeout(hadFocusVisibleRecentlyTimeout);
+  }, 100);
 }