From 22ffcb11df5567f7669016846a6386273365e29b Mon Sep 17 00:00:00 2001 From: David Christensen Date: Tue, 31 Oct 2023 15:19:15 -0400 Subject: [PATCH v3 1/5] TDE: doc updates --- doc/src/sgml/config.sgml | 127 +++++++++++++--- doc/src/sgml/database-encryption.sgml | 149 ++++++++++++++++++ doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/installation.sgml | 2 +- doc/src/sgml/postgres.sgml | 1 + doc/src/sgml/ref/allfiles.sgml | 1 + doc/src/sgml/ref/initdb.sgml | 47 ++++++ doc/src/sgml/ref/pg_alterckey.sgml | 210 ++++++++++++++++++++++++++ doc/src/sgml/ref/pg_ctl-ref.sgml | 14 ++ doc/src/sgml/ref/pgupgrade.sgml | 20 ++- doc/src/sgml/ref/postgres-ref.sgml | 13 ++ doc/src/sgml/reference.sgml | 1 + doc/src/sgml/storage.sgml | 5 + 13 files changed, 571 insertions(+), 20 deletions(-) create mode 100644 doc/src/sgml/database-encryption.sgml create mode 100644 doc/src/sgml/ref/pg_alterckey.sgml diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index bd70ff2e4b..c40cbc78e3 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1564,19 +1564,31 @@ include_dir 'conf.d' mechanism is used. - The command must print the passphrase to the standard output and exit - with code 0. In the parameter value, %p is - replaced by a prompt string. (Write %% for a - literal %.) Note that the prompt string will - probably contain whitespace, so be sure to quote adequately. A single - newline is stripped from the end of the output if present. + The command must print the passphrase to the standard output + and exit with code 0. It can prompt from the terminal if + is used. In the parameter + value, %R is replaced by a file descriptor + number opened to the terminal that started the server. A file + descriptor is only available if enabled at server start via + . If %R is specified and + no file descriptor is available, the server will not start. Value + %p is replaced by a pre-defined prompt string. + (Write %% for a literal %.) + Note that the prompt string will probably contain whitespace, + so be sure to quote its use adequately. Newlines are stripped + from the end of the output if present. + - The command does not actually have to prompt the user for a - passphrase. It can read it from a file, obtain it from a keychain - facility, or similar. It is up to the user to make sure the chosen - mechanism is adequately secure. + Sample scripts can be found in + $SHAREDIR/auth_commands, + where $SHAREDIR means the + PostgreSQL installation's shared-data + directory, often /usr/local/share/postgresql + (use pg_config --sharedir to determine it if + you're not sure). + This parameter can only be set in the postgresql.conf file or on the server command line. @@ -1598,10 +1610,12 @@ include_dir 'conf.d' parameter is off (the default), then ssl_passphrase_command will be ignored during a reload and the SSL configuration will not be reloaded if a passphrase - is needed. That setting is appropriate for a command that requires a - TTY for prompting, which might not be available when the server is - running. Setting this parameter to on might be appropriate if the - passphrase is obtained from a file, for example. + is needed. This setting is appropriate for a command that requires a + terminal for prompting, which will likely not be available when the server is + running. ( closes the terminal file + descriptor soon after server start.) Setting this parameter on + might be appropriate, for example, if the passphrase is obtained + from a file. This parameter can only be set in the postgresql.conf @@ -2773,7 +2787,9 @@ include_dir 'conf.d' max_wal_senders is non-zero. Note that changing wal_level to minimal makes previous base backups unusable - for point-in-time recovery and standby servers. + for point-in-time recovery and standby servers, which may + lead to data loss. Cluster file encryption also does not support + wal_level minimal. In logical level, the same information is logged as @@ -3126,9 +3142,10 @@ include_dir 'conf.d' - If data checksums are enabled, hint bit updates are always WAL-logged - and this setting is ignored. You can use this setting to test how much - extra WAL-logging would occur if your database had data checksums + If data checksums or cluster file encryption is enabled, + hint bit updates are always WAL-logged and this setting is + ignored. You can use this setting to test how much extra + WAL-logging would occur if your database had data checksums enabled. @@ -8520,6 +8537,64 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; + + Cluster File Encryption + + + + cluster_key_command (string) + + cluster_key_command configuration parameter + + + + + This option specifies an external command to obtain the cluster-level + key for cluster file encryption during server initialization and + server start. + + + The command must print the cluster key to the standard + output as 64 hexadecimal characters, and exit with code 0. + The command can prompt for the passphrase or PIN from the + terminal if is used. In the + parameter value, %R is replaced by a file + descriptor number opened to the terminal that started the server. + A file descriptor is only available if enabled at server start + via . If %R is specified + and no file descriptor is available, the server will not start. + Value %p is replaced by a pre-defined + prompt string. Value %d is replaced by the + directory containing the keys; this is useful if the command + must create files with the keys, e.g., to store a cluster-level + key encrypted by a key stored in a hardware security module. + (Write %% for a literal %.) + Note that the prompt string will probably contain whitespace, + so be sure to quote its use adequately. Newlines are stripped + from the end of the output if present. + + + + Sample script can be found in + $SHAREDIR/auth_commands, + where $SHAREDIR means the + PostgreSQL installation's shared-data + directory, often /usr/local/share/postgresql + (use pg_config --sharedir to determine it if + you're not sure). + + + + This parameter can only be set by + initdb, in the + postgresql.conf file, or on the server + command line. + + + + + + Client Connection Defaults @@ -10619,6 +10694,22 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir' + + file_encryption_method (boolean) + + Cluster file encryption method + + + + + Reports the cluster file + encryption method. See for more + information. + + + + data_directory_mode (integer) diff --git a/doc/src/sgml/database-encryption.sgml b/doc/src/sgml/database-encryption.sgml new file mode 100644 index 0000000000..ded406bb56 --- /dev/null +++ b/doc/src/sgml/database-encryption.sgml @@ -0,0 +1,149 @@ + + + + Cluster File Encryption + + + Cluster File Encryption + + + + The purpose of cluster file encryption is to prevent users with read + access on the directories used to store database files and write-ahead + log files from being able to access the data stored in those files. + For example, when using cluster file encryption, users who have read + access to the cluster directories for backup purposes will not be able + to decrypt the data stored in these files. Read-only access for a group + of users can be enabled using the initdb + option. Cluster file encryption + also provides data-at-rest security, protecting users from data loss + should the physical storage media be stolen or improperly erased before + disposal. + + + + Cluster file encryption does not protect against unauthorized file + system writes. Such writes can allow data decryption if used to weaken + the system's security and the weakened system is later supplied with + the externally-stored cluster encryption key. This also does not always + detect if users with write access remove or modify database files. + + + + This also does not protect against users who have read access to database + process memory because all in-memory data pages and data encryption keys + are stored unencrypted in memory. Therefore, an attacker who is able + to read memory can read the data encryption keys and decrypt the entire + cluster. The Postgres operating system user and the operating system + administrator, e.g., the root user, have such access. + + + + Keys + + + Cluster file encryption uses two levels of encryption — an upper + key which encrypts lower-level keys. The upper-level key is often + referred to as a Key Encryption Key (KEK). This key + is not stored in the file system, but provided at + initdb time and each time the server is started. This + key can be easily changed via pg_alterckey without + requiring any changes to the the data files or WAL + files. + + + + The lower level keys are data encryption keys, specifically for relations + and WAL. The relation key is used to encrypt database + heap and index files. The WAL key is used to encrypt write-ahead log + (WAL) files. Two different keys are used so that primary and standby + servers can use different relation keys, but the same WAL key, so that + these keys can (in a future release) be rotated by switching the + primary to the standby and then changing the WAL key. Eventually, + encryption will be able to added to non-encrypted clusters by creating + encrypted replicas and switching over to them. + + + + Postgres stores the data encryption (lower-level) keys in the data + directory encrypted (wrapped) by key encryption (upper-level) key. + Though the data encryption keys technically exist in the file system, + the key encryption key does not, so the data encryption keys are + securely stored. Data encryption keys are used to securely encrypt + other database files. + + + + + Initialization + + + Cluster file encryption is enabled when + PostgreSQL is built + with --with-openssl and is specified + during initdb. The cluster key + provided by the + option during initdb and the one generated + by in the + postgresql.conf must match for the database + cluster to start. Note that the cluster key command + passed to initdb must return a key of + 64 hexadecimal characters. For example: + +initdb -D dbname --cluster-key-command='ckey_passphrase.sh' + + Cluster file encryption does not support a wal_level + of minimal. + + + + + Operation + + + During the initdb process, if + is specified, two data-level + encryption keys are created. These two keys are then encrypted with + the key encryption key (KEK) supplied by the cluster key command before + being stored in the database directory. The key or passphrase that + derives the key must be supplied from the terminal or stored in a + trusted key store, such as key vault software or a hardware security + module. + + + + If the PostgreSQL server has + been initialized to require a cluster key, each time the + server starts the postgresql.conf + cluster_key_command command will be executed + and the cluster key retrieved. The data encryption keys in the + pg_cryptokeys directory will then be decrypted + using the supplied key and integrity-checked to ensure it matches the + initdb-supplied key. (If this check fails, the server will refuse + to start.) The cluster encryption key will then be removed from + system memory. The decrypted data encryption keys will remain in + shared memory until the server is stopped. + + + + The data encryption keys are randomly generated and can be 128, 192, + or 256-bits in length, depending on whether AES128, + AES192, or AES256 is specified. + They are encrypted by the key encryption key (KEK) using Advanced + Encryption Standard (AES256) encryption in Key + Wrap Padded Mode, which also provides KEK authentication; see RFC 5649. While + 128-bit encryption is sufficient for most sites, 256-bit encryption + is thought to be more immune to future quantum cryptographic attacks. + + + + If you prefer to create the random keys on your own, you can create + a empty directory with a pg_cryptokeys/live + subdirectory, generate the keys there using your tools. and use the + initdb + to copy those keys into the newly-created cluster. + + + diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 4c63a7e768..99bf98f6a5 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -50,6 +50,7 @@ + diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 9b25e9fdb1..3881c968d1 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -1019,7 +1019,7 @@ build-postgresql: Build with support for SSL (encrypted) - connections. The only LIBRARY + connections and cluster file encryption. The only LIBRARY supported is . This requires the OpenSSL package to be installed. configure will check for the required diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml index f629524be0..eac99fe54e 100644 --- a/doc/src/sgml/postgres.sgml +++ b/doc/src/sgml/postgres.sgml @@ -171,6 +171,7 @@ break is not needed in a wider output rendering. &wal; &logical-replication; &jit; + &database-encryption; ®ress; diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index 54b5f22d6e..36a591a91b 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -197,6 +197,7 @@ Complete list of usable sgml source files in this directory. + diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml index d43c91575c..777e8a3401 100644 --- a/doc/src/sgml/ref/initdb.sgml +++ b/doc/src/sgml/ref/initdb.sgml @@ -189,6 +189,18 @@ PostgreSQL documentation + + + + + + This option specifies an external command to obtain the cluster-level + key for cluster file encryption during server initialization and + server start; see for details. + + + + @@ -277,6 +289,18 @@ PostgreSQL documentation + + + + + + Specifies the cluster file encryption method. The + value values are AES128 (the default), AES192, and AES256. + + + + @@ -384,6 +408,17 @@ PostgreSQL documentation This option has no effect when is used. + + + + + + + + + Allows the command + to prompt for a passphrase or PIN. + @@ -412,6 +447,18 @@ PostgreSQL documentation + + + + + + Copies cluster file encryption keys from another cluster; required + when using pg_upgrade on a cluster + with cluster file encryption enabled. + + + + diff --git a/doc/src/sgml/ref/pg_alterckey.sgml b/doc/src/sgml/ref/pg_alterckey.sgml new file mode 100644 index 0000000000..867c439b8e --- /dev/null +++ b/doc/src/sgml/ref/pg_alterckey.sgml @@ -0,0 +1,210 @@ + + + + + pg_alterckey + + + + pg_alterckey + 1 + Application + + + + pg_alterckey + alter the PostgreSQL cluster key + + + + + pg_alterckey + + + + + + + + old_cluster_key_command + new_cluster_key_command + + + + + + + + datadir + + + + + + pg_alterckey + + + + + + + + + + + + + + + + + datadir + + + + + + + Description + + pg_alterckey alters the cluster key used + for cluster file encryption. The cluster key is initially set + during . The command can be run while the + server is running or stopped. The new password must be used the next + time the server is started. + + + + pg_alterckey changes the key encryption key + (KEK) which encrypts the data encryption keys; + it does not change the data encryption keys. It does this by + decrypting each data encryption key using the old_cluster_key_command, + re-encrypting it using the new_cluster_key_command, and then + writes the result back to the cluster directory. + + + + See the documentation for how to define + the old and new passphrase commands. You can use different executables + for these commands, or you can use the same executable with different + arguments to specify retrieval of the old or new key. + + + + pg_alterckey manages data encryption keys, + which are critical to allowing Postgres to access its decrypted + data. For this reason, it is very careful to preserve these + keys in most possible failure conditions, e.g., operating system + failure during cluster encryption key rotation. + + + + When started, pg_alterckey repairs any files that + remain from previous failures before altering the cluster encryption + key. During this repair phase, pg_alterckey will + either roll back the cluster key or roll forward the changes that + were previously requested. The server will not start if repair is + needed, though a running server is unaffected by an unrepaired cluster + key configuration. Therefore, if pg_alterckey + fails for any reason, it is recommended you run the command with + to simply roll back or forward any previous + changes. This will report if it rolled the cluster key back or forward, + and then run the command again to change the cluster key if needed. + + + + You can specify the data directory on the command line, or use + the environment variable PGDATA. + + + + + Options + + + + + + + + + Allows the and + commands + to prompt for a passphrase or PIN. + + + + + + + + Other options: + + + + + + + + Print the pg_alterckey version and exit. + + + + + + + + + + Show help about pg_alterckey command line + arguments, and exit. + + + + + + + + + + + Environment + + + + PGDATA + + + + Default data directory location + + + + + + PG_COLOR + + + Specifies whether to use color in diagnostic messages. Possible values + are always, auto and + never. + + + + + + + + See Also + + + + + + + diff --git a/doc/src/sgml/ref/pg_ctl-ref.sgml b/doc/src/sgml/ref/pg_ctl-ref.sgml index 46906966eb..e3705c724e 100644 --- a/doc/src/sgml/ref/pg_ctl-ref.sgml +++ b/doc/src/sgml/ref/pg_ctl-ref.sgml @@ -38,6 +38,7 @@ PostgreSQL documentation options path + @@ -72,6 +73,7 @@ PostgreSQL documentation seconds options + @@ -373,6 +375,18 @@ PostgreSQL documentation + + + + + + Allows or + to prompt for a passphrase + or PIN. + + + + diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index 46e8a0b746..12cab55fc2 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -182,6 +182,15 @@ PostgreSQL documentation + + + + allows or + to prompt for a passphrase + or PIN. + + + dir dir @@ -342,7 +351,9 @@ make prefix=/usr/local/pgsql.new install Again, use compatible initdb flags that match the old cluster. Many prebuilt installers do this step automatically. There is no need to - start the new cluster. + start the new cluster. If upgrading a cluster that uses + cluster file encryption, the initdb option + must be specified. @@ -947,6 +958,13 @@ psql --username=postgres --file=script.sql postgres is down. + + If the old cluster uses file encryption, the new cluster must use + the same keys, so pg_upgrade copies them to the + new cluster. It is necessary to initialize the new cluster with + the same cluster_key_command and the same + file encryption method. + diff --git a/doc/src/sgml/ref/postgres-ref.sgml b/doc/src/sgml/ref/postgres-ref.sgml index b13a16a117..1eb767865b 100644 --- a/doc/src/sgml/ref/postgres-ref.sgml +++ b/doc/src/sgml/ref/postgres-ref.sgml @@ -305,6 +305,19 @@ PostgreSQL documentation + + + + + Makes postgres prompt for a passphrase or PIN + from the specified open numeric file descriptor. The descriptor + is closed after the key is read. The file descriptor number + -1 duplicates standard error for the terminal; + this is useful for single-user mode. + + + + diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index e11b4b6130..4f6c9d1ca5 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -241,6 +241,7 @@ + &pgalterckey; &clusterdb; &createdb; &createuser; diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 3ea4e5526d..fc187342d5 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -77,6 +77,11 @@ Item Subdirectory containing transaction commit timestamp data + + pg_cryptokeys + Subdirectory containing file encryption keys + + pg_dynshmem Subdirectory containing files used by the dynamic shared memory -- 2.40.1