diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4d302d3..b33ddc0 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -123,16 +123,27 @@ jobs:
         rubygems: latest
     - run: ruby -e "exit(Gem.rubygems_version > Gem::Version.new('3.0.3'))"
 
-  testFixedRubygemsVersion:
-    name: "Test rubygems input set to a fixed version upgrades RubyGems to that versoin"
+  testFixedRubygemsVersionUpgrades:
+    name: "Test rubygems input set to a fixed version upgrades RubyGems to that version if the default is older"
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
     - uses: ./
       with:
         ruby-version: 2.6
-        rubygems: 3.3.5
-    - run: gem --version | grep -F "3.3.5"
+        rubygems: 3.2.3
+    - run: gem --version | grep -F "3.2.3"
+
+  testFixedRubygemsVersionNoop:
+    name: "Test rubygems input set to a fixed version noops if the default is newer"
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ./
+      with:
+        ruby-version: 3.1
+        rubygems: 3.2.3
+    - run: gem --version | grep -F "3.3.3"
 
   testExactBundlerVersion:
     name: "Test with an exact Bundler version"
diff --git a/action.yml b/action.yml
index 23d1772..01a1b58 100644
--- a/action.yml
+++ b/action.yml
@@ -14,7 +14,7 @@ inputs:
       The version of RubyGems to use. Either 'default', 'latest', or a version number (e.g., 3.3.5).
       For 'default', no action is taken and the version of RubyGems that comes with Ruby by default is used.
       For 'latest', `gem update --system` is run to update to the latest RubyGems version.
-      Similarly, if a version number is given, `gem update --system <version>` is run to update to that version of RubyGems.
+      Similarly, if a version number is given, `gem update --system <version>` is run to update to that version of RubyGems, as long as that version is newer than the one provided by default.
       Defaults to 'default'.
   bundler:
     description: |
diff --git a/dist/index.js b/dist/index.js
index 9700cf5..2f38272 100644
--- a/dist/index.js
+++ b/dist/index.js
@@ -58965,12 +58965,31 @@ __nccwpck_require__.r(__webpack_exports__);
 /* harmony export */ });
 const path = __nccwpck_require__(5622)
 const exec = __nccwpck_require__(1514)
+const semver = __nccwpck_require__(5911)
 
 async function rubygemsUpdate(rubygemsVersionInput, rubyPrefix) {
   const gem = path.join(rubyPrefix, 'bin', 'gem')
-  const rubygemsVersion = (rubygemsVersionInput === 'latest') ? [] : [rubygemsVersionInput]
 
-  await exec.exec(gem, ['update', '--system', ...rubygemsVersion])
+  let gemVersion = ''
+
+  await exec.exec(gem, ['--version'], {
+    listeners: {
+      stdout: (data) => (gemVersion += data.toString()),
+    }
+  });
+
+  gemVersion = semver.coerce(gemVersion.trim())
+  console.log(`Default RubyGems version is ${gemVersion}`)
+
+  if (rubygemsVersionInput === 'latest') {
+    console.log('Updating RubyGems to latest version')
+    await exec.exec(gem, ['update', '--system'])
+  } else if (semver.gt(rubygemsVersionInput, gemVersion)) {
+    console.log(`Updating RubyGems to ${rubygemsVersionInput}`)
+    await exec.exec(gem, ['update', '--system', rubygemsVersionInput])
+  } else {
+    console.log(`Skipping RubyGems update because the given version (${rubygemsVersionInput}) is not newer than the default version (${gemVersion})`)
+  }
 
   return true
 }
diff --git a/package.json b/package.json
index 915464c..26783d1 100644
--- a/package.json
+++ b/package.json
@@ -26,7 +26,8 @@
     "@actions/core": "^1.4.0",
     "@actions/exec": "^1.1.0",
     "@actions/io": "^1.1.1",
-    "@actions/tool-cache": "^1.7.1"
+    "@actions/tool-cache": "^1.7.1",
+    "semver": "^6.1.0"
   },
   "devDependencies": {
     "@vercel/ncc": "^0.31.1"
diff --git a/rubygems.js b/rubygems.js
index 105f61d..8b77bd7 100644
--- a/rubygems.js
+++ b/rubygems.js
@@ -1,11 +1,30 @@
 const path = require('path')
 const exec = require('@actions/exec')
+const semver = require('semver')
 
 export async function rubygemsUpdate(rubygemsVersionInput, rubyPrefix) {
   const gem = path.join(rubyPrefix, 'bin', 'gem')
-  const rubygemsVersion = (rubygemsVersionInput === 'latest') ? [] : [rubygemsVersionInput]
 
-  await exec.exec(gem, ['update', '--system', ...rubygemsVersion])
+  let gemVersion = ''
+
+  await exec.exec(gem, ['--version'], {
+    listeners: {
+      stdout: (data) => (gemVersion += data.toString()),
+    }
+  });
+
+  gemVersion = semver.coerce(gemVersion.trim())
+  console.log(`Default RubyGems version is ${gemVersion}`)
+
+  if (rubygemsVersionInput === 'latest') {
+    console.log('Updating RubyGems to latest version')
+    await exec.exec(gem, ['update', '--system'])
+  } else if (semver.gt(rubygemsVersionInput, gemVersion)) {
+    console.log(`Updating RubyGems to ${rubygemsVersionInput}`)
+    await exec.exec(gem, ['update', '--system', rubygemsVersionInput])
+  } else {
+    console.log(`Skipping RubyGems update because the given version (${rubygemsVersionInput}) is not newer than the default version (${gemVersion})`)
+  }
 
   return true
 }