diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index fe4c5ee86..ab27d4b85 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -18,6 +18,8 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.7"
+          cache: "pip"
+          cache-dependency-path: pyproject.toml
       - uses: actions/cache@v3
         id: cache
         with:
diff --git a/.github/workflows/smokeshow.yml b/.github/workflows/smokeshow.yml
index 7559c24c0..55d64517f 100644
--- a/.github/workflows/smokeshow.yml
+++ b/.github/workflows/smokeshow.yml
@@ -17,6 +17,8 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
+          cache: "pip"
+          cache-dependency-path: pyproject.toml
 
       - run: pip install smokeshow
 
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 85779af18..ddc43c942 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,13 +19,17 @@ jobs:
       - uses: actions/checkout@v3
       - name: Set up Python
         uses: actions/setup-python@v4
-        id: setup-python
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
           cache-dependency-path: pyproject.toml
+      - uses: actions/cache@v3
+        id: cache
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-python-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-test-v03
       - name: Install Dependencies
-        if: steps.setup-python.outputs.cache-hit != 'true'
+        if: steps.cache.outputs.cache-hit != 'true'
         run: pip install -e .[all,dev,doc,test]
       - name: Lint
         run: bash scripts/lint.sh
@@ -50,6 +54,8 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: '3.8'
+          cache: "pip"
+          cache-dependency-path: pyproject.toml
 
       - name: Get coverage files
         uses: actions/download-artifact@v3
diff --git a/docs/en/docs/release-notes.md b/docs/en/docs/release-notes.md
index ea3e5ab43..f6fd36be0 100644
--- a/docs/en/docs/release-notes.md
+++ b/docs/en/docs/release-notes.md
@@ -2,6 +2,7 @@
 
 ## Latest Changes
 
+* 👷 Fix and tweak CI cache handling. PR [#5696](https://github.com/tiangolo/fastapi/pull/5696) by [@tiangolo](https://github.com/tiangolo).
 * 👷 Update `setup-python` action in tests to use new caching feature. PR [#5680](https://github.com/tiangolo/fastapi/pull/5680) by [@madkinsz](https://github.com/madkinsz).
 * ✏️ Fix typo in docs for `docs/en/docs/advanced/middleware.md`. PR [#5376](https://github.com/tiangolo/fastapi/pull/5376) by [@rifatrakib](https://github.com/rifatrakib).
 * ⬆ Bump black from 22.8.0 to 22.10.0. PR [#5569](https://github.com/tiangolo/fastapi/pull/5569) by [@dependabot[bot]](https://github.com/apps/dependabot).