diff --git a/overlord/devicestate/crypto.go b/overlord/devicestate/crypto.go
index 8fdbdfcdf..5cddd27b6 100644
--- a/overlord/devicestate/crypto.go
+++ b/overlord/devicestate/crypto.go
@@ -34,19 +34,13 @@ import (
 )
 
 func generateRSAKey(keyLength int) (*rsa.PrivateKey, error) {
-       sshKeyFile       := filepath.Join(dirs.SnapRunDir, "snapd.key.tmp")
-       sshPublicKeyFile := sshKeyFile + ".pub"
+       tempDir, _ := ioutil.TempDir(dirs.SnapRunDir, "tmp")
+       sshKeyFile := filepath.Join(tempDir, "snapd.key")
 
        defer func() {
-               os.Remove(sshKeyFile)
-               os.Remove(sshPublicKeyFile)
+               os.Remove(tempDir)
        }()
 
-       os.MkdirAll(dirs.SnapRunDir, 0755)
-
-       os.Remove(sshKeyFile)
-       os.Remove(sshPublicKeyFile)
-
        cmd := exec.Command("ssh-keygen", "-t", "rsa", "-b", strconv.Itoa(keyLength), "-N", "", "-f", sshKeyFile)
        out, err := cmd.CombinedOutput()
        if err != nil {