diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6010039a3..37ea47a91 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -25,19 +25,17 @@ jobs:
         run: bundle exec rubocop
   irb:
     needs: ruby-versions
-    name: rake test ${{ matrix.ruby }} ${{ matrix.with_latest_reline && '(latest reline)' || '' }} ${{ matrix.with_tracer && '(with tracer)' || '' }}
+    name: rake test ${{ matrix.ruby }} ${{ matrix.with_latest_reline && '(latest reline)' || '' }}
     strategy:
       matrix:
         ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
         with_latest_reline: [true, false]
-        with_tracer: [true, false]
         exclude:
           - ruby: truffleruby
       fail-fast: false
     runs-on: ubuntu-latest
     env:
       WITH_LATEST_RELINE: ${{matrix.with_latest_reline}}
-      WITH_TRACER: ${{matrix.with_tracer}}
     timeout-minutes: 30
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
diff --git a/Gemfile b/Gemfile
index 149268561..3c2efa44d 100644
--- a/Gemfile
+++ b/Gemfile
@@ -19,7 +19,7 @@ gem "test-unit-ruby-core"
 
 gem "rubocop"
 
-gem "tracer" if ENV["WITH_TRACER"] == "true"
+gem "tracer" if !is_truffleruby
 gem "debug", github: "ruby/debug", platforms: [:mri, :mswin]
 
 if RUBY_VERSION >= "3.0.0" && !is_truffleruby
diff --git a/lib/irb/context.rb b/lib/irb/context.rb
index 5b8791c3b..e30125f46 100644
--- a/lib/irb/context.rb
+++ b/lib/irb/context.rb
@@ -61,7 +61,7 @@ def initialize(irb, workspace = nil, input_method = nil)
       @io = nil
 
       self.inspect_mode = IRB.conf[:INSPECT_MODE]
-      self.use_tracer = IRB.conf[:USE_TRACER] if IRB.conf[:USE_TRACER]
+      self.use_tracer = IRB.conf[:USE_TRACER]
       self.use_loader = IRB.conf[:USE_LOADER] if IRB.conf[:USE_LOADER]
       self.eval_history = IRB.conf[:EVAL_HISTORY] if IRB.conf[:EVAL_HISTORY]
 
@@ -162,8 +162,8 @@ def initialize(irb, workspace = nil, input_method = nil)
     private_constant :KEYWORD_ALIASES
 
     def use_tracer=(val)
-      require_relative "ext/tracer"
-      @use_tracer = val
+      require_relative "ext/tracer" if val
+      IRB.conf[:USE_TRACER] = val
     end
 
     private def build_completor
diff --git a/lib/irb/ext/tracer.rb b/lib/irb/ext/tracer.rb
index 53b2e6224..d498b5320 100644
--- a/lib/irb/ext/tracer.rb
+++ b/lib/irb/ext/tracer.rb
@@ -13,6 +13,13 @@
 end
 
 module IRB
+  class CallTracer < ::CallTracer
+    IRB_DIR = File.expand_path('../..', __dir__)
+
+    def skip?(tp)
+      super || tp.path.match?(IRB_DIR) || tp.path.match?('<internal:prelude>')
+    end
+  end
   class WorkSpace
     alias __evaluate__ evaluate
     # Evaluate the context of this workspace and use the Tracer library to
diff --git a/lib/irb/init.rb b/lib/irb/init.rb
index a434ab23e..8acfdea8e 100644
--- a/lib/irb/init.rb
+++ b/lib/irb/init.rb
@@ -62,9 +62,6 @@ def IRB.setup(ap_path, argv: ::ARGV)
 
   # @CONF default setting
   def IRB.init_config(ap_path)
-    # class instance variables
-    @TRACER_INITIALIZED = false
-
     # default configurations
     unless ap_path and @CONF[:AP_NAME]
       ap_path = File.join(File.dirname(File.dirname(__FILE__)), "irb.rb")
diff --git a/test/irb/test_tracer.rb b/test/irb/test_tracer.rb
index f8da066b6..199c9586b 100644
--- a/test/irb/test_tracer.rb
+++ b/test/irb/test_tracer.rb
@@ -10,7 +10,9 @@ class ContextWithTracerIntegrationTest < IntegrationTestCase
     def setup
       super
 
-      @envs.merge!("NO_COLOR" => "true", "RUBY_DEBUG_HISTORY_FILE" => '')
+      omit "Tracer gem is not available when running on TruffleRuby" if RUBY_ENGINE == "truffleruby"
+
+      @envs.merge!("NO_COLOR" => "true")
     end
 
     def example_ruby_file
@@ -29,54 +31,30 @@ def bar(obj)
       RUBY
     end
 
-    def test_use_tracer_is_disabled_by_default
+    def test_use_tracer_enabled_when_gem_is_unavailable
       write_rc <<~RUBY
-        IRB.conf[:USE_TRACER] = false
+        # Simulate the absence of the tracer gem
+        ::Kernel.send(:alias_method, :irb_original_require, :require)
+
+        ::Kernel.define_method(:require) do |name|
+          raise LoadError, "cannot load such file -- tracer (test)" if name.match?("tracer")
+          ::Kernel.send(:irb_original_require, name)
+        end
+
+        IRB.conf[:USE_TRACER] = true
       RUBY
 
       write_ruby example_ruby_file
 
       output = run_ruby_file do
         type "bar(Foo)"
-        type "exit!"
+        type "exit"
       end
 
-      assert_nil IRB.conf[:USER_TRACER]
-      assert_not_include(output, "#depth:")
-      assert_not_include(output, "Foo.foo")
-    end
-
-    def test_use_tracer_enabled_when_gem_is_unavailable
-      begin
-        gem 'tracer'
-        omit "Skipping because 'tracer' gem is available."
-      rescue Gem::LoadError
-        write_rc <<~RUBY
-          IRB.conf[:USE_TRACER] = true
-        RUBY
-
-        write_ruby example_ruby_file
-
-        output = run_ruby_file do
-          type "bar(Foo)"
-          type "exit!"
-        end
-
-        assert_include(output, "Tracer extension of IRB is enabled but tracer gem wasn't found.")
-      end
+      assert_include(output, "Tracer extension of IRB is enabled but tracer gem wasn't found.")
     end
 
     def test_use_tracer_enabled_when_gem_is_available
-      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('3.1.0')
-        omit "Ruby version before 3.1.0 does not support Tracer integration. Skipping this test."
-      end
-
-      begin
-        gem 'tracer'
-      rescue Gem::LoadError
-        omit "Skipping because 'tracer' gem is not available. Enable with WITH_TRACER=true."
-      end
-
       write_rc <<~RUBY
         IRB.conf[:USE_TRACER] = true
       RUBY
@@ -85,13 +63,29 @@ def test_use_tracer_enabled_when_gem_is_available
 
       output = run_ruby_file do
         type "bar(Foo)"
-        type "exit!"
+        type "exit"
       end
 
       assert_include(output, "Object#bar at")
       assert_include(output, "Foo.foo at")
       assert_include(output, "Foo.foo #=> 100")
       assert_include(output, "Object#bar #=> 100")
+
+      # Test that the tracer output does not include IRB's own files
+      assert_not_include(output, "irb/workspace.rb")
+    end
+
+    def test_use_tracer_is_disabled_by_default
+      write_ruby example_ruby_file
+
+      output = run_ruby_file do
+        type "bar(Foo)"
+        type "exit"
+      end
+
+      assert_not_include(output, "#depth:")
+      assert_not_include(output, "Foo.foo")
     end
+
   end
 end