diff --git a/fastapi/responses.py b/fastapi/responses.py
index 6cd793157..88dba96e8 100644
--- a/fastapi/responses.py
+++ b/fastapi/responses.py
@@ -31,4 +31,6 @@ class ORJSONResponse(JSONResponse):
 
     def render(self, content: Any) -> bytes:
         assert orjson is not None, "orjson must be installed to use ORJSONResponse"
-        return orjson.dumps(content)
+        return orjson.dumps(
+            content, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SERIALIZE_NUMPY
+        )
diff --git a/tests/test_orjson_response_class.py b/tests/test_orjson_response_class.py
new file mode 100644
index 000000000..6fe62daf9
--- /dev/null
+++ b/tests/test_orjson_response_class.py
@@ -0,0 +1,21 @@
+from fastapi import FastAPI
+from fastapi.responses import ORJSONResponse
+from fastapi.testclient import TestClient
+from sqlalchemy.sql.elements import quoted_name
+
+app = FastAPI(default_response_class=ORJSONResponse)
+
+
+@app.get("/orjson_non_str_keys")
+def get_orjson_non_str_keys():
+    key = quoted_name(value="msg", quote=False)
+    return {key: "Hello World", 1: 1}
+
+
+client = TestClient(app)
+
+
+def test_orjson_non_str_keys():
+    with client:
+        response = client.get("/orjson_non_str_keys")
+    assert response.json() == {"msg": "Hello World", "1": 1}